Merge pull request #3367 from confor/fix-deps

Add imghdr and cgi

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## v0.6.3
+ Released 26 May 2024
+
+Highlights:
+* Hotfix for searcher not returning results
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.2...v0.6.3).
+
+## v0.6.2
+ Released 26 May 2024
+
+Highlights:
+* Added soulseek support
+* Added bandcamp support
+* Changes and dependency updates to work with Python >= 3.12
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.1...v0.6.2).
+
+## v0.6.1
+R eleased 26 November 2023
+
+Highlights:
+* Dependency updates to work with > Python 3.11
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.0...v0.6.1).
+
+## v0.6.0
+Released 13 November 2022
+
+Highlights:
+* Updated to python 3
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.20...v0.6.0).
+
 ## v0.5.20
 Released 15 October 2021
 

Headphones.py

@@ -17,6 +17,10 @@
 import os
 import sys
 
+if sys.version_info <= (3, 6):
+    sys.stdout.write("Headphones requires Python >= 3.7\n")
+    sys.exit(1)
+
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib/'))
 

contrib/sni_test.py

@@ -6,7 +6,7 @@ import sys
 # Ensure that we use the Headphones provided libraries.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib"))
 
-import urlparse
+import urllib.parse
 
 
 def can_import(module):
@@ -89,7 +89,7 @@ def main():
         url = sys.argv[1]
 
     # Check if it is a HTTPS website.
-    parts = urlparse.urlparse(url)
+    parts = urllib.parse.urlparse(url)
 
     if parts.scheme.lower() != "https":
         sys.stderr.write(

data/interfaces/default/config.html

@@ -310,6 +310,16 @@
                                     <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="5">
                                 </div>
                             </fieldset>
+                            <fieldset title="Method for downloading Bandcamp.com files.">
+                                <legend>Bandcamp</legend>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can store raw downloads from Bandcamp.com.">
+                                        Bandcamp Directory
+                                    </label>
+                                    <input type="text" name="bandcamp_dir" value="${config['bandcamp_dir']}" size="50">
+                                    <small>Full path where raw MP3s will be stored, e.g. /Users/name/Downloads/bandcamp</small>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset title="Method for downloading torrent files.">
@@ -317,7 +327,7 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_blackhole" value="0" ${config['torrent_downloader_blackhole']}> Black Hole
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
-                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_qbittorrent" value="4" ${config['torrent_downloader_qbittorrent']}> QBitTorrent
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
@@ -438,6 +448,11 @@
                                     <input type="text" name="deluge_label" value="${config['deluge_label']}" size="30">
                                     <small>Labels shouldn't contain spaces (requires Label plugin)</small>
                                 </div>
+                                <div class="row">
+                                    <label>Download Directory</label>
+                                    <input type="text" name="deluge_download_directory" value="${config['deluge_download_directory']}" size="30">
+                                    <small>Directory where Deluge should download to</small>
+                                </div>
                                 <div class="row">
                                     <label>Move When Completed</label>
                                     <input type="text" name="deluge_done_directory" value="${config['deluge_done_directory']}" size="30">
@@ -467,7 +482,33 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
-                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>Soulseek
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>No Preference
+                                </div>
+                            </fieldset>
+                        </td>
+                        <td>
+                            <fieldset>
+                                <legend>Soulseek</legend>
+                                <div class="row">
+                                    <label>Soulseek API URL</label>
+                                    <input type="text" name="soulseek_api_url" value="${config['soulseek_api_url']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label>Soulseek API KEY</label>
+                                    <input type="text" name="soulseek_api_key" value="${config['soulseek_api_key']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_download_dir" value="${config['soulseek_download_dir']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Incomplete Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_incomplete_download_dir" value="${config['soulseek_incomplete_download_dir']}" size="50">
                                 </div>
                             </fieldset>
                         </td>
@@ -579,6 +620,19 @@
                                     </div>
                                 </div>
                             </fieldset>
+                            <fieldset>
+                                <legend>Other</legend>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_bandcamp" type="checkbox" class="bigcheck" name="use_bandcamp" value="1" ${config['use_bandcamp']} /><label for="use_bandcamp"><span class="option">Bandcamp</span></label>
+                                    </div>
+                                </fieldset>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_soulseek" type="checkbox" class="bigcheck" name="use_soulseek" value="1" ${config['use_soulseek']} /><label for="use_soulseek"><span class="option">Soulseek</span></label>
+                                    </div>
+                                </fieldset>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -601,26 +655,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_waffles" type="checkbox" class="bigcheck" name="use_waffles" value="1" ${config['use_waffles']} /><label for="use_waffles"><span class="option">Waffles.ch</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>UID Number</label>
-                                            <input type="text" name="waffles_uid" value="${config['waffles_uid']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Passkey</label>
-                                            <input type="password" name="waffles_passkey" value="${config['waffles_passkey']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="waffles_ratio" value="${config['waffles_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                                 <fieldset>
                                     <div class="row checkbox left">
                                         <input id="use_rutracker" type="checkbox" class="bigcheck" name="use_rutracker" value="1" ${config['use_rutracker']} /><label for="use_rutracker"><span class="option">rutracker.org</span></label>
@@ -674,6 +708,10 @@
                                         <input id="use_redacted" type="checkbox" class="bigcheck" name="use_redacted" value="1" ${config['use_redacted']} /><label for="use_redacted"><span class="option">Redacted</span></label>
                                     </div>
                                     <div class="config">
+                                        <div class="row">
+                                            <label>Api Key</label>
+                                            <input type="text" name="redacted_apikey" value="${config['redacted_apikey']}" size="36">
+                                        </div>
                                         <div class="row">
                                             <label>Username</label>
                                             <input type="text" name="redacted_username" value="${config['redacted_username']}" size="36">
@@ -695,7 +733,7 @@
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_torznab" type="checkbox" class="bigcheck" name="use_torznab" value="1" ${config['use_torznab']} /><label for="use_torznab"><span class="option">Jackett / Torznab Providers</span></label>
+                                        <input id="use_torznab" type="checkbox" class="bigcheck" name="use_torznab" value="1" ${config['use_torznab']} /><label for="use_torznab"><span class="option">Torznab Providers</span></label>
                                     </div>
                                     <div id="torznab_providers">
                                         <div class="config" id="torznab1">
@@ -754,22 +792,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_oldpiratebay" type="checkbox" class="bigcheck" name="use_oldpiratebay" value="1" ${config['use_oldpiratebay']} /><label for="use_oldpiratebay"><span class="option">Old Pirate Bay</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>URL</label>
-                                            <input type="text" name="oldpiratebay_url" value="${config['oldpiratebay_url']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="oldpiratebay_ratio" value="${config['oldpiratebay_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                             </fieldset>
                         </td>
                     </tr>
@@ -1274,7 +1296,7 @@
                                     <input type="checkbox" name="synoindex_enabled" id="synoindex" value="1" ${config['synoindex_enabled']} /><label for="synoindex"><span class="option">Synology NAS</span></label>
                                 </div>
                             </fieldset>
-
+<!--
                             <fieldset>
                                 <div class="row checkbox left">
                                     <input type="checkbox" class="bigcheck" name="twitter_enabled" id="twitter" value="1" ${config['twitter_enabled']} /><label for="twitter"><span class="option">Twitter</span></label>
@@ -1295,7 +1317,7 @@
                                     </div>
                                 </div>
                             </fieldset>
-
+-->
                             <fieldset>
                                 <div class="row checkbox left">
                                     <input type="checkbox" class="bigcheck" name="slack_enabled" id="slack" value="1" ${config['slack_enabled']} /><label for="slack"><span class="option">Slack</span></label>
@@ -1370,17 +1392,20 @@
                             <div class="row">
                                 <label>File Format</label>
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
-                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year. Put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
+                                <small>Use: In addition to the above, there is also $Title/$title (track title), $Track (track #), $Disc (disc #), $DiscTotal.</small>
                             </div>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
                             </div>
+                            <div class="checkbox row left clearfix nopad">
+                                <input type="checkbox" name="rename_single_disc_ignore" id="rename_single_disc_ignore" value="1" ${config['rename_single_disc_ignore']}/><label>Don't include disc# for single disc albums</label>
+                            </div>
                         </fieldset>
 
                         <fieldset>
                             <legend>Re-Encoding Options</legend>
                             <small class="heading"><i class="fa fa-info-circle"></i> Note: this option requires the lame, ffmpeg or xld encoder</small>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="music_encoder" id="music_encoder" value="1" ${config['music_encoder']}/><label>Re-encode downloads during postprocessing</label>
                             </div>
                             <div id="encoderoptions" class="row clearfix checkbox">
@@ -1651,6 +1676,16 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <legend>Last.fm</legend>
+                                <div id="lastfmoptions">
+                                    <div class="row">
+                                        <label>API Key</label>
+                                        <input type="text" name="lastfm_apikey" value="${config['lastfm_apikey']}" size="40" />
+                                    </div>
+                                </div>
+                            </fieldset>
+
                             <fieldset>
                                 <legend>Songkick</legend>
                                 <div class="row checkbox">
@@ -2551,8 +2586,6 @@
                 initConfigCheckbox("#use_omgwtfnzbs");
                 initConfigCheckbox("#use_torznab");
                 initConfigCheckbox("#use_piratebay");
-                initConfigCheckbox("#use_oldpiratebay");
-                initConfigCheckbox("#use_waffles");
                 initConfigCheckbox("#use_rutracker");
                 initConfigCheckbox("#use_orpheus");
                 initConfigCheckbox("#use_redacted");

data/interfaces/default/history.html

@@ -1,7 +1,7 @@
 <%inherit file="base.html"/>
 <%!
 	from headphones import helpers
-	import cgi
+	from html import escape as html_escape 
 %>
 
 <%def name="headerIncludes()">
@@ -56,17 +56,19 @@
                 fileid = 'torrent'
             if item['URL'].find('codeshy') != -1:
                 fileid = 'nzb'
+            if item['URL'].find('bandcamp') != -1:
+                fileid = 'bandcamp'
 
             folder = 'Folder: ' + item['FolderName']
 
         %>
 			<tr class="grade${grade}">
 				<td id="dateadded">${item['DateAdded']}</td>
-				<td id="filename">${cgi.escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
+				<td id="filename">${html_escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
 				<td id="size">${helpers.bytes_to_mb(item['Size'])}</td>
 				<td title="${folder}" id="status">${item['Status']}</td>
-				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${cgi.escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${cgi.escape(item['Title'], quote=True)}'">new</a>]</td>
-				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${cgi.escape(item['Title'], quote=True)}',$(this),'table')" data-success="${cgi.escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
+				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${html_escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${html_escape(item['Title'], quote=True)}'">new</a>]</td>
+				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${html_escape(item['Title'], quote=True)}',$(this),'table')" data-success="${html_escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
 			</tr>
 		%endfor
 		</tbody>

headphones/__init__.py

@@ -218,7 +218,7 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno)
 
     os.setsid()
@@ -232,10 +232,10 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno)
 
-    dev_null = file('/dev/null', 'r')
+    dev_null = open('/dev/null', 'r')
     os.dup2(dev_null.fileno(), sys.stdin.fileno())
 
     si = open('/dev/null', "r")
@@ -251,7 +251,7 @@ def daemonize():
 
     if CREATEPID:
         logger.info("Writing PID %d to %s", pid, PIDFILE)
-        with file(PIDFILE, 'w') as fp:
+        with open(PIDFILE, 'w') as fp:
             fp.write("%s\n" % pid)
 
 

headphones/albumart.py

@@ -28,11 +28,34 @@ def getAlbumArt(albumid):
 
     # CAA
     logger.info("Searching for artwork at CAA")
-    artwork_path = 'http://coverartarchive.org/release-group/%s/front' % albumid
-    artwork = getartwork(artwork_path)
-    if artwork:
-        logger.info("Artwork found at CAA")
-        return artwork_path, artwork
+    #artwork_path = 'https://coverartarchive.org/release-group/%s/front' % albumid
+    artwork_path = 'https://coverartarchive.org/release-group/%s' % albumid
+
+    data = request.request_json(artwork_path, timeout=20, whitelist_status_code=404)
+
+    image_url = None
+    if data:
+        for item in data.get("images", []):
+            try:
+                if "Front" not in item["types"]:
+                    continue
+
+                # Use desired size
+                image_url = item["image"]
+                if headphones.CONFIG.ALBUM_ART_MAX_WIDTH:
+                    if isinstance(item.get("thumbnails"), dict):
+                        image_url = item["thumbnails"].get(
+                            headphones.CONFIG.ALBUM_ART_MAX_WIDTH, image_url
+                        )
+                break
+            except KeyError:
+                pass
+
+    if image_url:
+        artwork = getartwork(image_url)
+        if artwork:
+            logger.info("Artwork found at CAA")
+            return artwork_path, artwork
 
     # Amazon
     logger.info("Searching for artwork at Amazon")
@@ -41,7 +64,7 @@ def getAlbumArt(albumid):
         'SELECT ArtistName, AlbumTitle, ReleaseID, AlbumASIN FROM albums WHERE AlbumID=?',
         [albumid]).fetchone()
     if dbalbum['AlbumASIN']:
-        artwork_path = 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % dbalbum['AlbumASIN']
+        artwork_path = 'https://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % dbalbum['AlbumASIN']
         artwork = getartwork(artwork_path)
         if artwork:
             logger.info("Artwork found at Amazon")
@@ -156,12 +179,21 @@ def getartwork(artwork_path):
                     break
                 elif maxwidth and img_width > maxwidth:
                     # Downsize using proxy service to max width
-                    artwork_path = '{0}?{1}'.format('http://images.weserv.nl/', urlencode({
-                        'url': artwork_path.replace('http://', ''),
-                        'w': maxwidth,
-                    }))
                     artwork = bytes()
-                    r = request.request_response(artwork_path, timeout=20, stream=True, whitelist_status_code=404)
+                    url = "https://images.weserv.nl"
+                    params = {
+                        "url": artwork_path,
+                        "w": maxwidth
+                    }
+                    headers = {"User-Agent": "Headphones"}
+                    r = request.request_response(
+                        url,
+                        params=params,
+                        timeout=20,
+                        stream=True,
+                        whitelist_status_code=404,
+                        headers=headers
+                    )
                     if r:
                         for chunk in r.iter_content(chunk_size=1024):
                             artwork += chunk
@@ -182,7 +214,7 @@ def getCachedArt(albumid):
     if not artwork_path:
         return
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith("http"):
         artwork = request.request_content(artwork_path, timeout=20)
 
         if not artwork:

headphones/api.py

@@ -86,7 +86,7 @@ class Api(object):
             methodToCall = getattr(self, "_" + self.cmd)
             methodToCall(**self.kwargs)
             if 'callback' not in self.kwargs:
-                if isinstance(self.data, basestring):
+                if isinstance(self.data, str):
                     return self.data
                 else:
                     return json.dumps(self.data)
@@ -106,7 +106,7 @@ class Api(object):
         rows_as_dic = []
 
         for row in rows:
-            row_as_dic = dict(zip(row.keys(), row))
+            row_as_dic = dict(list(zip(list(row.keys()), row)))
             rows_as_dic.append(row_as_dic)
 
         return rows_as_dic
@@ -474,17 +474,17 @@ class Api(object):
         # Handle situations where the torrent url contains arguments that are
         # parsed
         if kwargs:
-            import urllib
-            import urllib2
-            url = urllib2.quote(
-                url, safe=":?/=&") + '&' + urllib.urlencode(kwargs)
+            import urllib.request, urllib.parse, urllib.error
+            import urllib.request, urllib.error, urllib.parse
+            url = urllib.parse.quote(
+                url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
 
         try:
             result = [(title, int(size), url, provider, kind)]
         except ValueError:
             result = [(title, float(size), url, provider, kind)]
 
-        logger.info(u"Making sure we can download the chosen result")
+        logger.info("Making sure we can download the chosen result")
         (data, bestqual) = searcher.preprocess(result)
 
         if data and bestqual:

headphones/bandcamp.py

@@ -0,0 +1,166 @@
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>
+
+import headphones
+import json
+import os
+import re
+
+from headphones import logger, helpers, metadata, request
+from headphones.common import USER_AGENT
+from headphones.types import Result
+
+from mediafile import MediaFile, UnreadableFileError
+from bs4 import BeautifulSoup
+from bs4 import FeatureNotFound
+
+
+def search(album, albumlength=None, page=1, resultlist=None):
+    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ',
+           '"': '', ',': '', '*': '', '.': '', ':': ''}
+    if resultlist is None:
+        resultlist = []
+
+    cleanalbum = helpers.latinToAscii(
+        helpers.replace_all(album['AlbumTitle'], dic)
+        ).strip()
+    cleanartist = helpers.latinToAscii(
+        helpers.replace_all(album['ArtistName'], dic)
+        ).strip()
+
+    headers = {'User-Agent': USER_AGENT}
+    params = {
+        "page": page,
+        "q": cleanalbum,
+    }
+    logger.info("Looking up https://bandcamp.com/search with {}".format(
+        params))
+    content = request.request_content(
+        url='https://bandcamp.com/search',
+        params=params,
+        headers=headers
+        ).decode('utf8')
+    try:
+        soup = BeautifulSoup(content, "html5lib")
+    except FeatureNotFound:
+        soup = BeautifulSoup(content, "html.parser")
+
+    for item in soup.find_all("li", class_="searchresult"):
+        type = item.find('div', class_='itemtype').text.strip().lower()
+        if type == "album":
+            data = parse_album(item)
+
+            cleanartist_found = helpers.latinToAscii(data['artist'])
+            cleanalbum_found = helpers.latinToAscii(data['album'])
+
+            logger.debug(u"{} - {}".format(data['album'], cleanalbum_found))
+
+            logger.debug("Comparing {} to {}".format(
+                cleanalbum, cleanalbum_found))
+            if (cleanartist.lower() == cleanartist_found.lower() and
+                    cleanalbum.lower() == cleanalbum_found.lower()):
+                resultlist.append(Result(
+                    data['title'], data['size'], data['url'],
+                    'bandcamp', 'bandcamp', True))
+        else:
+            continue
+
+    if(soup.find('a', class_='next')):
+        page += 1
+        logger.debug("Calling next page ({})".format(page))
+        search(album, albumlength=albumlength,
+               page=page, resultlist=resultlist)
+
+    return resultlist
+
+
+def download(album, bestqual):
+    html = request.request_content(url=bestqual.url).decode('utf-8')
+    trackinfo = []
+    try:
+        trackinfo = json.loads(
+            re.search(r"trackinfo&quot;:(\[.*?\]),", html)
+            .group(1)
+            .replace('&quot;', '"'))
+    except ValueError as e:
+        logger.warn("Couldn't load json: {}".format(e))
+
+    directory = os.path.join(
+        headphones.CONFIG.BANDCAMP_DIR,
+        u'{} - {}'.format(
+            album['ArtistName'].replace('/', '_'),
+            album['AlbumTitle'].replace('/', '_')))
+    directory = helpers.latinToAscii(directory)
+
+    if not os.path.exists(directory):
+        try:
+            os.makedirs(directory)
+        except Exception as e:
+            logger.warn("Could not create directory ({})".format(e))
+
+    index = 1
+    for track in trackinfo:
+        filename = helpers.replace_illegal_chars(
+                    u'{:02d} - {}.mp3'.format(index, track['title']))
+        fullname = os.path.join(directory.encode('utf-8'),
+                                filename.encode('utf-8'))
+        logger.debug("Downloading to {}".format(fullname))
+
+        if 'file' in track and track['file'] != None and 'mp3-128' in track['file']:
+            content = request.request_content(track['file']['mp3-128'])
+            open(fullname, 'wb').write(content)
+            try:
+                f = MediaFile(fullname)
+                date, year = metadata._date_year(album)
+                f.update({
+                    'artist': album['ArtistName'].encode('utf-8'),
+                    'album': album['AlbumTitle'].encode('utf-8'),
+                    'title': track['title'].encode('utf-8'),
+                    'track': track['track_num'],
+                    'tracktotal': len(trackinfo),
+                    'year': year,
+                })
+                f.save()
+            except UnreadableFileError as ex:
+                logger.warn("MediaFile couldn't parse: %s (%s)",
+                            fullname,
+                            str(ex))
+
+        index += 1
+
+    return directory
+
+
+def parse_album(item):
+    album = item.find('div', class_='heading').text.strip()
+    artist = item.find('div', class_='subhead').text.strip().replace("by ", "")
+    released = item.find('div', class_='released').text.strip().replace(
+        "released ", "")
+    year = re.search(r"(\d{4})", released).group(1)
+
+    url = item.find('div', class_='heading').find('a')['href'].split("?")[0]
+
+    length = item.find('div', class_='length').text.strip()
+    tracks, minutes = length.split(",")
+    tracks = tracks.replace(" tracks", "").replace(" track", "").strip()
+    minutes = minutes.replace(" minutes", "").strip()
+    # bandcamp offers mp3 128b with should be 960KB/minute
+    size = int(minutes) * 983040
+
+    data = {"title": u'{} - {} [{}]'.format(artist, album, year),
+            "artist": artist, "album": album,
+            "url": url, "size": size}
+
+    return data

headphones/cache.py

@@ -240,7 +240,7 @@ class Cache(object):
 
             # fallback to 1st album cover if none of the above
             elif 'albums' in data:
-                for mbid, art in data.get('albums', dict()).items():
+                for mbid, art in list(data.get('albums', dict()).items()):
                     if 'albumcover' in art:
                         image_url = art['albumcover'][0]['url']
                         break
@@ -352,7 +352,7 @@ class Cache(object):
 
             # fallback to 1st album cover if none of the above
             elif 'albums' in data:
-                for mbid, art in data.get('albums', dict()).items():
+                for mbid, art in list(data.get('albums', dict()).items()):
                     if 'albumcover' in art:
                         image_url = art['albumcover'][0]['url']
                         break
@@ -540,12 +540,19 @@ class Cache(object):
                         artwork_thumb = None
                         if 'fanart' in thumb_url:
                             # Create thumb using image resizing service
-                            artwork_path = '{0}?{1}'.format('http://images.weserv.nl/', urlencode({
-                                'url': thumb_url.replace('http://', ''),
-                                'w': 300,
-                            }))
-                            artwork_thumb = request.request_content(artwork_path, timeout=20, whitelist_status_code=404)
-
+                            url = "https://images.weserv.nl"
+                            params = {
+                                "url": thumb_url,
+                                "w": 300
+                            }
+                            headers = {"User-Agent": "Headphones"}
+                            artwork_thumb = request.request_content(
+                                url,
+                                params=params,
+                                timeout=20,
+                                whitelist_status_code=404,
+                                headers=headers
+                            )
                         if artwork_thumb:
                             with open(thumb_path, 'wb') as f:
                                 f.write(artwork_thumb)

headphones/classes.py

@@ -18,12 +18,12 @@
 #######################################
 
 
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
-from common import USER_AGENT
+from .common import USER_AGENT
 
 
-class HeadphonesURLopener(urllib.FancyURLopener):
+class HeadphonesURLopener(urllib.request.FancyURLopener):
     version = USER_AGENT
 
 
@@ -44,7 +44,7 @@ class AuthURLOpener(HeadphonesURLopener):
         self.numTries = 0
 
         # call the base class
-        urllib.FancyURLopener.__init__(self)
+        urllib.request.FancyURLopener.__init__(self)
 
     def prompt_user_passwd(self, host, realm):
         """

headphones/common.py

@@ -24,6 +24,7 @@ import operator
 import os
 import re
 from headphones import version
+from functools import reduce
 
 
 # Identify Our Application
@@ -74,7 +75,7 @@ class Quality:
     @staticmethod
     def _getStatusStrings(status):
         toReturn = {}
-        for x in Quality.qualityStrings.keys():
+        for x in list(Quality.qualityStrings.keys()):
             toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + \
                                                            Quality.qualityStrings[x] + ")"
         return toReturn
@@ -93,7 +94,7 @@ class Quality:
     def splitQuality(quality):
         anyQualities = []
         bestQualities = []
-        for curQual in Quality.qualityStrings.keys():
+        for curQual in list(Quality.qualityStrings.keys()):
             if curQual & quality:
                 anyQualities.append(curQual)
             if curQual << 16 & quality:
@@ -101,36 +102,6 @@ class Quality:
 
         return (anyQualities, bestQualities)
 
-    @staticmethod
-    def nameQuality(name):
-
-        def checkName(list, func):
-            return func([re.search(x, name, re.I) for x in list])
-
-        name = os.path.basename(name)
-
-        # if we have our exact text then assume we put it there
-        for x in Quality.qualityStrings:
-            if x == Quality.UNKNOWN:
-                continue
-
-            regex = '\W' + Quality.qualityStrings[x].replace(' ', '\W') + '\W'
-            regex_match = re.search(regex, name, re.I)
-            if regex_match:
-                return x
-
-        # TODO: fix quality checking here
-        if checkName(["mp3", "192"], any) and not checkName(["flac"], all):
-            return Quality.B192
-        elif checkName(["mp3", "256"], any) and not checkName(["flac"], all):
-            return Quality.B256
-        elif checkName(["mp3", "vbr"], any) and not checkName(["flac"], all):
-            return Quality.VBR
-        elif checkName(["mp3", "320"], any) and not checkName(["flac"], all):
-            return Quality.B320
-        else:
-            return Quality.UNKNOWN
-
     @staticmethod
     def assumeQuality(name):
         if name.lower().endswith(".mp3"):
@@ -151,28 +122,21 @@ class Quality:
     @staticmethod
     def splitCompositeStatus(status):
         """Returns a tuple containing (status, quality)"""
-        for x in sorted(Quality.qualityStrings.keys(), reverse=True):
+        for x in sorted(list(Quality.qualityStrings.keys()), reverse=True):
             if status > x * 100:
                 return (status - x * 100, x)
 
         return (Quality.NONE, status)
 
-    @staticmethod
-    def statusFromName(name, assume=True):
-        quality = Quality.nameQuality(name)
-        if assume and quality == Quality.UNKNOWN:
-            quality = Quality.assumeQuality(name)
-        return Quality.compositeStatus(DOWNLOADED, quality)
-
     DOWNLOADED = None
     SNATCHED = None
     SNATCHED_PROPER = None
 
 
-Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in Quality.qualityStrings.keys()]
-Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in Quality.qualityStrings.keys()]
+Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in list(Quality.qualityStrings.keys())]
+Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in list(Quality.qualityStrings.keys())]
 Quality.SNATCHED_PROPER = [Quality.compositeStatus(SNATCHED_PROPER, x) for x in
-                           Quality.qualityStrings.keys()]
+                           list(Quality.qualityStrings.keys())]
 
 MP3 = Quality.combineQualities([Quality.B192, Quality.B256, Quality.B320, Quality.VBR], [])
 LOSSLESS = Quality.combineQualities([Quality.FLAC], [])

headphones/config.py

@@ -2,15 +2,16 @@ import itertools
 
 import os
 import re
+import ast
+from configparser import ConfigParser
 import headphones.logger
-from configobj import ConfigObj
 
 
 def bool_int(value):
     """
     Casts a config value into a 0 or 1
     """
-    if isinstance(value, basestring):
+    if isinstance(value, str):
         if value.lower() in ('', '0', 'false', 'f', 'no', 'n', 'off'):
             value = 0
     return int(bool(value))
@@ -79,6 +80,7 @@ _CONFIG_DEFINITIONS = {
     'DELUGE_PASSWORD': (str, 'Deluge', ''),
     'DELUGE_LABEL': (str, 'Deluge', ''),
     'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''),
+    'DELUGE_DOWNLOAD_DIRECTORY': (str, 'Deluge', ''),
     'DELUGE_PAUSED': (int, 'Deluge', 0),
     'DESTINATION_DIR': (str, 'General', ''),
     'DETECT_BITRATE': (int, 'General', 0),
@@ -154,9 +156,10 @@ _CONFIG_DEFINITIONS = {
     'KEEP_TORRENT_FILES': (int, 'General', 0),
     'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
+    'LASTFM_APIKEY': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
-    'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
+    'LIBRARYSCAN_INTERVAL': (int, 'General', 24),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
     'LOG_DIR': (path, 'General', ''),
@@ -200,9 +203,6 @@ _CONFIG_DEFINITIONS = {
     'PIRATEBAY': (int, 'Piratebay', 0),
     'PIRATEBAY_PROXY_URL': (str, 'Piratebay', ''),
     'PIRATEBAY_RATIO': (str, 'Piratebay', ''),
-    'OLDPIRATEBAY': (int, 'Old Piratebay', 0),
-    'OLDPIRATEBAY_URL': (str, 'Old Piratebay', ''),
-    'OLDPIRATEBAY_RATIO': (str, 'Old Piratebay', ''),
     'PLEX_CLIENT_HOST': (str, 'Plex', ''),
     'PLEX_ENABLED': (int, 'Plex', 0),
     'PLEX_NOTIFY': (int, 'Plex', 0),
@@ -239,6 +239,7 @@ _CONFIG_DEFINITIONS = {
     'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
     'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
+    'RENAME_SINGLE_DISC_IGNORE': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
     'REPLACE_EXISTING_FOLDERS': (int, 'General', 0),
@@ -266,6 +267,11 @@ _CONFIG_DEFINITIONS = {
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
+    'SOULSEEK_API_URL': (str, 'Soulseek', ''),
+    'SOULSEEK_API_KEY': (str, 'Soulseek', ''),
+    'SOULSEEK_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK_INCOMPLETE_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK': (int, 'Soulseek', 0),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
@@ -299,11 +305,8 @@ _CONFIG_DEFINITIONS = {
     'UTORRENT_USERNAME': (str, 'uTorrent', ''),
     'VERIFY_SSL_CERT': (bool_int, 'Advanced', 1),
     'WAIT_UNTIL_RELEASE_DATE': (int, 'General', 0),
-    'WAFFLES': (int, 'Waffles', 0),
-    'WAFFLES_PASSKEY': (str, 'Waffles', ''),
-    'WAFFLES_RATIO': (str, 'Waffles', ''),
-    'WAFFLES_UID': (str, 'Waffles', ''),
     'REDACTED': (int, 'Redacted', 0),
+    'REDACTED_APIKEY': (str, 'Redacted', ''),
     'REDACTED_USERNAME': (str, 'Redacted', ''),
     'REDACTED_PASSWORD': (str, 'Redacted', ''),
     'REDACTED_RATIO': (str, 'Redacted', ''),
@@ -314,7 +317,9 @@ _CONFIG_DEFINITIONS = {
     'XBMC_PASSWORD': (str, 'XBMC', ''),
     'XBMC_UPDATE': (int, 'XBMC', 0),
     'XBMC_USERNAME': (str, 'XBMC', ''),
-    'XLDPROFILE': (str, 'General', '')
+    'XLDPROFILE': (str, 'General', ''),
+    'BANDCAMP': (int, 'General', 0),
+    'BANDCAMP_DIR': (path, 'General', '')
 }
 
 
@@ -326,8 +331,9 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigObj(self._config_file, encoding='utf-8')
-        for key in _CONFIG_DEFINITIONS.keys():
+        self._config = ConfigParser(interpolation=None)
+        self._config.read(self._config_file)
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
         self.ENCODER_MULTICORE_COUNT = max(0, self.ENCODER_MULTICORE_COUNT)
         self._upgrade()
@@ -344,7 +350,7 @@ class Config(object):
 
     def check_section(self, section):
         """ Check if INI section exists, if not create it """
-        if section not in self._config:
+        if not self._config.has_section(section):
             self._config[section] = {}
             return True
         else:
@@ -354,28 +360,38 @@ class Config(object):
         """ Cast any value in the config to the right type or use the default """
         key, definition_type, section, ini_key, default = self._define(key)
         self.check_section(section)
+
+        # ConfigParser values are strings, so need to convert to actual list
+        if definition_type == list:
+            definition_type = ast.literal_eval
+
         try:
             my_val = definition_type(self._config[section][ini_key])
+            # ConfigParser interprets quotes in the config
+            # literally, so we need to sanitize it. It's not really
+            # a config upgrade, since a user can at any time put 
+            # some_key = 'some_val'
+            if type(my_val) in [str, path]:
+                my_val = my_val.strip('"').strip("'")
         except Exception:
-            my_val = definition_type(default)
-            self._config[section][ini_key] = my_val
+            my_val = default
+            self._config[section][ini_key] = str(my_val)
         return my_val
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigObj(encoding="UTF-8")
-        new_config.filename = self._config_file
+        new_config = ConfigParser(interpolation=None)
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data
-        for key, subkeys in self._config.items():
+        for key, subkeys in list(self._config.items()):
             if key not in new_config:
                 new_config[key] = {}
-            for subkey, value in subkeys.items():
+            for subkey, value in list(subkeys.items()):
                 new_config[key][subkey] = value
 
         # next make sure that everything we expect to have defined is so
-        for key in _CONFIG_DEFINITIONS.keys():
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             key, definition_type, section, ini_key, default = self._define(key)
             self.check_setting(key)
             if section not in new_config:
@@ -386,14 +402,15 @@ class Config(object):
         headphones.logger.info("Writing configuration to file")
 
         try:
-            new_config.write()
+            with open(self._config_file, 'w') as configfile:
+                new_config.write(configfile)
         except IOError as e:
             headphones.logger.error("Error writing configuration file: %s", e)
 
     def get_extra_newznabs(self):
         """ Return the extra newznab tuples """
         extra_newznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
+            zip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
                              for i in range(3)])
         )
         return extra_newznabs
@@ -412,7 +429,7 @@ class Config(object):
     def get_extra_torznabs(self):
         """ Return the extra torznab tuples """
         extra_torznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
+            zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
                              for i in range(4)])
         )
         return extra_torznabs
@@ -448,20 +465,21 @@ class Config(object):
             return value
         else:
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
             return self._config[section][ini_key]
 
     def process_kwargs(self, kwargs):
         """
         Given a big bunch of key value pairs, apply them to the ini.
         """
-        for name, value in kwargs.items():
+        for name, value in list(kwargs.items()):
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
 
     def _upgrade(self):
         """
-        Bring old configs up to date
+        Bring old configs up to date. Although this is kind of a dumb
+        way to do it because it doesn't handle multi-step upgrades
         """
         if self.CONFIG_VERSION == '2':
             # Update the config to use direct path to the encoder rather than the encoder folder
@@ -488,12 +506,12 @@ class Config(object):
             # Add Seed Ratio to Torznabs
             if self.EXTRA_TORZNABS:
                 extra_torznabs = list(
-                    itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+                    zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
                                      for i in range(3)])
                 )
                 new_torznabs = []
                 for torznab in extra_torznabs:
-                    new_torznabs.extend([torznab[0], torznab[1], u'', torznab[2]])
+                    new_torznabs.extend([torznab[0], torznab[1], '', torznab[2]])
                 if new_torznabs:
                     self.EXTRA_TORZNABS = new_torznabs
 

headphones/config_test.py

@@ -2,8 +2,8 @@ import mock
 from mock import MagicMock
 import headphones.config
 import re
-import unittestcompat
-from unittestcompat import TestCase, TestArgs
+from . import unittestcompat
+from .unittestcompat import TestCase, TestArgs
 
 
 class ConfigApiTest(TestCase):
@@ -101,7 +101,7 @@ class ConfigApiTest(TestCase):
         # call methods
         c = headphones.config.Config(path)
         # assertions:
-        with self.assertRaisesRegexp(KeyError, exc_regex):
+        with self.assertRaisesRegex(KeyError, exc_regex):
             c.check_setting(setting_name)
         pass
 

headphones/crier.py

@@ -21,7 +21,7 @@ def cry():
             main_thread = t
 
     # Loop over each thread's current frame, writing info about it
-    for tid, frame in sys._current_frames().iteritems():
+    for tid, frame in sys._current_frames().items():
         thread = tmap.get(tid, main_thread)
 
         lines = []

headphones/cuesplit.py

@@ -87,7 +87,7 @@ def check_splitter(command):
 
 def split_baby(split_file, split_cmd):
     '''Let's split baby'''
-    logger.info('Splitting %s...', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Splitting {split_file}...")
     logger.debug(subprocess.list2cmdline(split_cmd))
 
     # Prevent Windows from opening a terminal window
@@ -108,16 +108,16 @@ def split_baby(split_file, split_cmd):
 
     process = subprocess.Popen(split_cmd, startupinfo=startupinfo,
                                stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE, env=env)
+                               stderr=subprocess.PIPE, env=env, text=True)
     stdout, stderr = process.communicate()
 
     if process.returncode:
-        logger.error('Split failed for %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
-        out = stdout if stdout else stderr
-        logger.error('Error details: %s', out.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.error(f"Split failed for {split_file}")
+        out = stdout or stderr
+        logger.error(f"Error details: {out}")
         return False
     else:
-        logger.info('Split success %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.info(f"Split succeeded for {split_file}")
         return True
 
 
@@ -232,7 +232,7 @@ class Directory:
         for i in list_dir:
             if not check_match(i):
                 # music file
-                if os.path.splitext(i)[-1] in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+                if os.path.splitext(i)[-1] in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
                     track_nr = identify_track_number(i)
                     if track_nr:
                         self.content.append(WaveFile(self.path + os.sep + i, track_nr=track_nr))
@@ -378,7 +378,7 @@ class CueFile(File):
             except:
                 raise ValueError('Cant encode CUE Sheet.')
 
-        if self.content[0] == u'\ufeff':
+        if self.content[0] == '\ufeff':
             self.content = self.content[1:]
 
         header = header_parser()
@@ -581,7 +581,7 @@ def split(albumpath):
 
     # use xld profile to split cue
     if headphones.CONFIG.ENCODER == 'xld' and headphones.CONFIG.MUSIC_ENCODER and headphones.CONFIG.XLDPROFILE:
-        import getXldProfile
+        from . import getXldProfile
         xldprofile, xldformat, _ = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE)
         if not xldformat:
             raise ValueError(
@@ -601,7 +601,7 @@ def split(albumpath):
         raise ValueError('Command not found, ensure shntool or xld installed')
 
     # Determine if file can be split
-    if wave.name_ext not in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+    if wave.name_ext not in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
         raise ValueError('Cannot split, audio file has unsupported extension')
 
     # Split with xld

headphones/db.py

@@ -17,7 +17,7 @@
 # Stolen from Sick-Beard's db.py  #
 ###################################
 
-from __future__ import with_statement
+
 
 import time
 
@@ -116,8 +116,8 @@ class DBConnection:
 
                     break
 
-            except sqlite3.OperationalError, e:
-                if "unable to open database file" in e.message or "database is locked" in e.message:
+            except sqlite3.OperationalError as e:
+                if "unable to open database file" in str(e) or "database is locked" in str(e):
                     dberror = e
                     if args is None:
                         logger.debug('Database error: %s. Query: %s', e, query)
@@ -128,7 +128,7 @@ class DBConnection:
                 else:
                     logger.error('Database error: %s', e)
                     raise
-            except sqlite3.DatabaseError, e:
+            except sqlite3.DatabaseError as e:
                 logger.error('Fatal Error executing %s :: %s', query, e)
                 raise
 
@@ -156,14 +156,14 @@ class DBConnection:
         If the table is not updated then the 'WHERE changes' will be 0 and the table inserted
         """
         def genParams(myDict):
-            return [x + " = ?" for x in myDict.keys()]
+            return [x + " = ?" for x in list(myDict.keys())]
 
         update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
 
-        insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + " SELECT " + ", ".join(
-            ["?"] * len(valueDict.keys() + keyDict.keys())) + " WHERE changes()=0")
+        insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(list(valueDict.keys()) + list(keyDict.keys())) + ")" + " SELECT " + ", ".join(
+            ["?"] * len(list(valueDict.keys()) + list(keyDict.keys()))) + " WHERE changes()=0")
 
         try:
-            self.action(update_query, valueDict.values() + keyDict.values(), upsert_insert_qry=insert_query)
+            self.action(update_query, list(valueDict.values()) + list(keyDict.values()), upsert_insert_qry=insert_query)
         except sqlite3.IntegrityError:
             logger.info('Queries failed: %s and %s', update_query, insert_query)

headphones/deluge.py

@@ -34,7 +34,7 @@
 # You should have received a copy of the GNU General Public License
 # along with SickRage. If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import unicode_literals
+
 
 from headphones import logger
 
@@ -58,12 +58,12 @@ def _scrubber(text):
     if scrub_logs:
         try:
             # URL parameter values
-            text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text)
+            text = re.sub(r'=[0-9a-zA-Z]*', r'=REMOVED', text)
             # Local host with port
             # text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # just host
-            text = re.sub('\:\/\/.*\:[0-9]*', '://REMOVED:', text)
+            text = re.sub(r'\:\/\/.*\:[0-9]*', r'://REMOVED:', text)
             # Session cookie
-            text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text)
+            text = re.sub(r"_session_id'\: '.*'", r"_session_id': 'REMOVED'", text)
             # Local Windows user path
             if text.lower().startswith('c:\\users\\'):
                 k = text.split('\\')
@@ -84,7 +84,6 @@ def addTorrent(link, data=None, name=None):
         result = {}
         retid = False
         url_orpheus = ['https://orpheus.network/', 'http://orpheus.network/']
-        url_waffles = ['https://waffles.ch/', 'http://waffles.ch/']
 
         if link.lower().startswith('magnet:'):
             logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link))
@@ -94,9 +93,6 @@ def addTorrent(link, data=None, name=None):
 
         elif link.lower().startswith('http://') or link.lower().startswith('https://'):
             logger.debug('Deluge: Got a URL: %s' % _scrubber(link))
-            if link.lower().startswith(tuple(url_waffles)):
-                if 'rss=' not in link:
-                    link = link + '&rss=1'
             if link.lower().startswith(tuple(url_orpheus)):
                 logger.debug('Deluge: Using different User-Agent for this site')
                 user_agent = 'Headphones'
@@ -128,9 +124,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -160,9 +156,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -466,38 +462,36 @@ def _add_torrent_url(result):
 
 def _add_torrent_file(result):
     logger.debug('Deluge: Adding file')
+
+    options = {}
+
+    if headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY:
+        options['download_location'] = headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY
+
+    if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
+        options['move_completed'] = 1
+        if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
+            options['move_completed_path'] = headphones.CONFIG.DELUGE_DONE_DIRECTORY
+        else:
+            options['move_completed_path'] = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
+
+    if headphones.CONFIG.DELUGE_PAUSED:
+        options['add_paused'] = headphones.CONFIG.DELUGE_PAUSED
+
     if not any(delugeweb_auth):
         _get_auth()
     try:
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
                                 "params": [result['name'] + '.torrent',
-                                b64encode(result['content'].encode('utf8')), {}],
+                                b64encode(result['content']).decode(),
+                                options],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert, headers=headers)
+                                 verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
-    except UnicodeDecodeError:
-        try:
-            # content is torrent file contents that needs to be encoded to base64
-            # this time let's try leaving the encoding as is
-            logger.debug('Deluge: There was a decoding issue, let\'s try again')
-            post_data = json.dumps({"method": "core.add_torrent_file",
-                                    "params": [result['name'].decode('utf8') + '.torrent',
-                                    b64encode(result['content']), {}],
-                                    "id": 22})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-            result['hash'] = json.loads(response.text)['result']
-            logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
-            return json.loads(response.text)['result']
-        except Exception as e:
-            logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e))
-            formatted_lines = traceback.format_exc().splitlines()
-            logger.error('; '.join(formatted_lines))
-            return False
     except Exception as e:
         logger.error('Deluge: Adding torrent file failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -585,61 +579,3 @@ def setSeedRatio(result):
         return None
 
 
-def setTorrentPath(result):
-    logger.debug('Deluge: Setting download path')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
-            post_data = json.dumps({"method": "core.set_torrent_move_completed",
-                                    "params": [result['hash'], True],
-                                    "id": 7})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
-                move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
-            else:
-                move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
-
-            if not os.path.exists(move_to):
-                logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to)
-                os.makedirs(move_to)
-            post_data = json.dumps({"method": "core.set_torrent_move_completed_path",
-                                    "params": [result['hash'], move_to],
-                                    "id": 8})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent move-to directory failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None
-
-
-def setTorrentPause(result):
-    logger.debug('Deluge: Pausing torrent')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_PAUSED:
-            post_data = json.dumps({"method": "core.pause_torrent",
-                                    "params": [[result['hash']]],
-                                    "id": 9})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent paused failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None

headphones/getXldProfile.py

@@ -1,6 +1,6 @@
 import os.path
 
-import biplist
+import plistlib
 from headphones import logger
 
 
@@ -14,8 +14,9 @@ def getXldProfile(xldProfile):
 
     # Get xld preferences plist
     try:
-        preferences = biplist.readPlist(expanded)
-    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException), e:
+        with open(expanded, 'rb') as _f:
+            preferences = plistlib.load(_f)
+    except Exception as e:
         logger.error("Error reading xld preferences plist: %s", e)
         return (xldProfileNotFound, None, None)
 

headphones/helpers.py

@@ -14,23 +14,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from operator import itemgetter
-import unicodedata
-import datetime
+import os
+import re
 import shutil
-import time
 import sys
 import tempfile
-import glob
+import time
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, date
+from fnmatch import fnmatch
+from functools import cmp_to_key
+from glob import glob
+from operator import itemgetter
 
 from beets import logging as beetslogging
-import six
-from contextlib import contextmanager
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from six import text_type
+from unidecode import unidecode
 
-import fnmatch
-import re
-import os
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
 import headphones
 
 
@@ -40,6 +42,24 @@ RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.")
 RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)?", re.I)
 RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
 
+def cmp(x, y):
+    """
+    Replacement for built-in function cmp that was removed in Python 3
+
+    Compare the two objects x and y and return an integer according to
+    the outcome. The return value is negative if x < y, zero if x == y
+    and strictly positive if x > y.
+
+    https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function
+    """
+    if x is None and y is None:
+        return 0
+    elif x is None:
+        return -1
+    elif y is None:
+        return 1
+    else:
+        return (x > y) - (x < y)
 
 def multikeysort(items, columns):
     comparers = [
@@ -54,7 +74,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, cmp=comparer)
+    return sorted(items, key=cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -136,28 +156,25 @@ def convert_seconds(s):
 
 
 def today():
-    today = datetime.date.today()
-    yyyymmdd = datetime.date.isoformat(today)
-    return yyyymmdd
+    return date.isoformat(date.today())
 
 
 def now():
-    now = datetime.datetime.now()
+    now = datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def get_age(date):
-    try:
-        split_date = date.split('-')
-    except:
+def is_valid_date(d):
+    if not d:
         return False
+    else:
+        return bool(re.match(r'\d{4}-\d{2}-\d{2}', d))
 
-    try:
-        days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2])
-    except (IndexError, ValueError):
-        days_old = False
 
-    return days_old
+def age(d):
+    '''Requires a valid date'''
+    delta = date.today() - date.fromisoformat(d)
+    return delta.days
 
 
 def bytes_to_mb(bytes):
@@ -167,7 +184,7 @@ def bytes_to_mb(bytes):
 
 
 def mb_to_bytes(mb_str):
-    result = re.search('^(\d+(?:\.\d+)?)\s?(?:mb)?', mb_str, flags=re.I)
+    result = re.search(r"^(\d+(?:\.\d+)?)\s?(?:mb)?", mb_str, flags=re.I)
     if result:
         return int(float(result.group(1)) * 1048576)
 
@@ -210,7 +227,7 @@ def pattern_substitute(pattern, dic, normalize=False):
 
     if normalize:
         new_dic = {}
-        for i, j in dic.iteritems():
+        for i, j in dic.items():
             if j is not None:
                 try:
                     if sys.platform == 'darwin':
@@ -229,21 +246,21 @@ def replace_all(text, dic):
     if not text:
         return ''
 
-    for i, j in dic.iteritems():
+    for i, j in dic.items():
         text = text.replace(i, j)
     return text
 
 
 def replace_illegal_chars(string, type="file"):
     if type == "file":
-        string = re.sub('[\?"*:|<>/]', '_', string)
+        string = re.sub(r"[\?\"*:|<>/]", "_", string)
     if type == "folder":
-        string = re.sub('[:\?<>"|*]', '_', string)
+        string = re.sub(r"[:\?<>\"|*]", "_", string)
     return string
 
 
-_CN_RE1 = re.compile(ur'[^\w]+', re.UNICODE)
-_CN_RE2 = re.compile(ur'[\s_]+', re.UNICODE)
+_CN_RE1 = re.compile(r'[^\w]+', re.UNICODE)
+_CN_RE2 = re.compile(r'[\s_]+', re.UNICODE)
 
 
 _XLATE_GRAPHICAL_AND_DIACRITICAL = {
@@ -253,33 +270,33 @@ _XLATE_GRAPHICAL_AND_DIACRITICAL = {
     # ©ª«®²³¹»¼½¾ÆÐØÞßæðøþĐđĦħıIJijĸĿŀŁłŒœŦŧDŽDždžLJLjljNJNjnjǤǥDZDzdzȤȥ. This
     # includes also some graphical symbols which can be easily replaced and
     # usually are written by people who don't have appropriate keyboard layout.
-    u'©': '(C)', u'ª': 'a.', u'«': '<<', u'®': '(R)', u'²': '2', u'³': '3',
-    u'¹': '1', u'»': '>>', u'¼': ' 1/4 ', u'½': ' 1/2 ', u'¾': ' 3/4 ',
-    u'Æ': 'AE', u'Ð': 'D', u'Ø': 'O', u'Þ': 'Th', u'ß': 'ss', u'æ': 'ae',
-    u'ð': 'd', u'ø': 'o', u'þ': 'th', u'Đ': 'D', u'đ': 'd', u'Ħ': 'H',
-    u'ħ': 'h', u'ı': 'i', u'IJ': 'IJ', u'ij': 'ij', u'ĸ': 'q', u'Ŀ': 'L',
-    u'ŀ': 'l', u'Ł': 'L', u'ł': 'l', u'Œ': 'OE', u'œ': 'oe', u'Ŧ': 'T',
-    u'ŧ': 't', u'DŽ': 'DZ', u'Dž': 'Dz', u'LJ': 'LJ', u'Lj': 'Lj',
-    u'lj': 'lj', u'NJ': 'NJ', u'Nj': 'Nj', u'nj': 'nj',
-    u'Ǥ': 'G', u'ǥ': 'g', u'DZ': 'DZ', u'Dz': 'Dz', u'dz': 'dz',
-    u'Ȥ': 'Z', u'ȥ': 'z', u'№': 'No.',
-    u'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
+    '©': '(C)', 'ª': 'a.', '«': '<<', '®': '(R)', '²': '2', '³': '3',
+    '¹': '1', '»': '>>', '¼': ' 1/4 ', '½': ' 1/2 ', '¾': ' 3/4 ',
+    'Æ': 'AE', 'Ð': 'D', 'Ø': 'O', 'Þ': 'Th', 'ß': 'ss', 'æ': 'ae',
+    'ð': 'd', 'ø': 'o', 'þ': 'th', 'Đ': 'D', 'đ': 'd', 'Ħ': 'H',
+    'ħ': 'h', 'ı': 'i', 'IJ': 'IJ', 'ij': 'ij', 'ĸ': 'q', 'Ŀ': 'L',
+    'ŀ': 'l', 'Ł': 'L', 'ł': 'l', 'Œ': 'OE', 'œ': 'oe', 'Ŧ': 'T',
+    'ŧ': 't', 'DŽ': 'DZ', 'Dž': 'Dz', 'LJ': 'LJ', 'Lj': 'Lj',
+    'lj': 'lj', 'NJ': 'NJ', 'Nj': 'Nj', 'nj': 'nj',
+    'Ǥ': 'G', 'ǥ': 'g', 'DZ': 'DZ', 'Dz': 'Dz', 'dz': 'dz',
+    'Ȥ': 'Z', 'ȥ': 'z', '№': 'No.',
+    'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
                        # this is 'masculine ordering indicator', not degree
 }
 
 _XLATE_SPECIAL = {
     # Translation table.
     # Cover additional special characters processing normalization.
-    u"'": '',         # replace apostrophe with nothing
-    u"’": '',         # replace musicbrainz style apostrophe with nothing
-    u'&': ' and ',     # expand & to ' and '
+    "'": '',         # replace apostrophe with nothing
+    "’": '',         # replace musicbrainz style apostrophe with nothing
+    '&': ' and ',     # expand & to ' and '
 }
 
 _XLATE_MUSICBRAINZ = {
     # Translation table for Musicbrainz.
-    u"…": '...',     # HORIZONTAL ELLIPSIS (U+2026)
-    u"’": "'",       # APOSTROPHE (U+0027)
-    u"‐": "-",       # EN DASH (U+2013)
+    "…": '...',     # HORIZONTAL ELLIPSIS (U+2026)
+    "’": "'",       # APOSTROPHE (U+0027)
+    "‐": "-",       # EN DASH (U+2013)
 }
 
 
@@ -314,10 +331,10 @@ def _transliterate(u, xlate):
     Perform transliteration using the specified dictionary
     """
     u = unicodedata.normalize('NFD', u)
-    u = u''.join([u'' if _is_unicode_combining(x) else x for x in u])
+    u = ''.join(['' if _is_unicode_combining(x) else x for x in u])
     u = _translate(u, xlate)
     # at this point output is either unicode, or plain ascii
-    return unicode(u)
+    return str(u)
 
 
 def clean_name(s):
@@ -327,10 +344,10 @@ def clean_name(s):
     :param s: string to clean up, possibly unicode one.
     :return: cleaned-up version of input string.
     """
-    if not isinstance(s, unicode):
+    if not isinstance(s, str):
         # ignore extended chars if someone was dumb enough to pass non-ascii
         # narrow string here, use only unicode for meaningful texts
-        u = unicode(s, 'ascii', 'replace')
+        u = str(s, 'ascii', 'replace')
     else:
         u = s
     # 1. don't bother doing normalization NFKC, rather transliterate
@@ -341,9 +358,9 @@ def clean_name(s):
     # 3. translate spacials
     u = _translate(u, _XLATE_SPECIAL)
     # 4. replace any non-alphanumeric character sequences by spaces
-    u = _CN_RE1.sub(u' ', u)
+    u = _CN_RE1.sub(' ', u)
     # 5. coalesce interleaved space/underscore sequences
-    u = _CN_RE2.sub(u' ', u)
+    u = _CN_RE2.sub(' ', u)
     # 6. trim
     u = u.strip()
     # 7. lowercase
@@ -357,8 +374,8 @@ def clean_musicbrainz_name(s, return_as_string=True):
     :param s: string to clean up, probably unicode.
     :return: cleaned-up version of input string.
     """
-    if not isinstance(s, unicode):
-        u = unicode(s, 'ascii', 'replace')
+    if not isinstance(s, str):
+        u = str(s, 'ascii', 'replace')
     else:
         u = s
     u = _translate(u, _XLATE_MUSICBRAINZ)
@@ -369,7 +386,7 @@ def clean_musicbrainz_name(s, return_as_string=True):
 
 
 def cleanTitle(title):
-    title = re.sub('[\.\-\/\_]', ' ', title).lower()
+    title = re.sub(r"[\.\-\/\_]", " ", title).lower()
 
     # Strip out extra whitespace
     title = ' '.join(title.split())
@@ -452,8 +469,7 @@ def expand_subfolders(f):
 
     if difference > 0:
         logger.info(
-            "Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.",
-            len(media_folders), difference)
+            f"Found {len(media_folders)} media folders, but depth difference between lowest and deepest media folder is {difference} (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.")
 
         # While already failed, advice the user what he could try. We assume the
         # directory may contain separate CD's and maybe some extra's. The
@@ -465,8 +481,7 @@ def expand_subfolders(f):
             set([os.path.join(*media_folder) for media_folder in extra_media_folders]))
 
         logger.info(
-            "Please look at the following folder(s), since they cause the depth difference: %s",
-            extra_media_folders)
+            f"Please look at the following folder(s), since they cause the depth difference: {extra_media_folders}")
         return
 
     # Convert back to paths and remove duplicates, which may be there after
@@ -480,7 +495,7 @@ def expand_subfolders(f):
         logger.debug("Did not expand subfolder, as it resulted in one folder.")
         return
 
-    logger.debug("Expanded subfolders in folder: %s", media_folders)
+    logger.debug(f"Expanded subfolders in folder: {media_folders}")
     return media_folders
 
 
@@ -491,14 +506,14 @@ def path_match_patterns(path, patterns):
     """
 
     for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
+        if fnmatch(path, pattern):
             return True
 
     # No match
     return False
 
 
-def path_filter_patterns(paths, patterns, root=None):
+def path_filter_patterns(paths, patterns, root=''):
     """
     Scan for ignored paths based on glob patterns. Note that the whole path
     will be matched, therefore paths should only contain the relative paths.
@@ -512,8 +527,7 @@ def path_filter_patterns(paths, patterns, root=None):
 
     for path in paths[:]:
         if path_match_patterns(path, patterns):
-            logger.debug("Path ignored by pattern: %s",
-                         os.path.join(root or "", path))
+            logger.debug(f"Path ignored by pattern: {os.path.join(root, path)}")
 
             ignored += 1
             paths.remove(path)
@@ -595,7 +609,7 @@ def extract_metadata(f):
     count_ratio = 0.75
 
     if count < (count_ratio * len(results)):
-        logger.info("Counted %d media files, but only %d have tags, ignoring.", count, len(results))
+        logger.info(f"Counted {count} media files, but only {len(results)} have tags, ignoring.")
         return (None, None, None)
 
     # Count distinct values
@@ -613,8 +627,7 @@ def extract_metadata(f):
                 old_album = new_albums[index]
                 new_albums[index] = RE_CD_ALBUM.sub("", album).strip()
 
-                logger.debug("Stripped albumd number identifier: %s -> %s", old_album,
-                             new_albums[index])
+                logger.debug(f"Stripped album number identifier: {old_album} -> {new_albums[index]}")
 
         # Remove duplicates
         new_albums = list(set(new_albums))
@@ -632,7 +645,7 @@ def extract_metadata(f):
     if len(artists) > 1 and len(albums) == 1:
         split_artists = [RE_FEATURING.split(x) for x in artists]
         featurings = [len(split_artist) - 1 for split_artist in split_artists]
-        logger.info("Album seem to feature %d different artists", sum(featurings))
+        logger.info("Album seem to feature {sum(featurings)} different artists")
 
         if sum(featurings) > 0:
             # Find the artist of which the least splits have been generated.
@@ -644,9 +657,11 @@ def extract_metadata(f):
             return (artist, albums[0], years[0])
 
     # Not sure what to do here.
-    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists),
-                len(albums), len(years))
-    logger.debug("Artists: %s, Albums: %s, Years: %s", artists, albums, years)
+    logger.info(
+        f"Found {len(artists)} artists, {len(albums)} albums and "
+        f"{len(years)} years in metadata, so ignoring"
+    )
+    logger.debug("Artists: {artists}, Albums: {albums}, Years: {years}")
 
     return (None, None, None)
 
@@ -678,8 +693,7 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
     else:
         tempdir = tempfile.gettempdir()
 
-    logger.info("Preparing to copy to a temporary directory for post processing: " + albumpath.decode(
-            headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Preparing to copy to a temporary directory for post processing: {albumpath}")
 
     try:
         file_name = os.path.basename(os.path.normpath(albumpath))
@@ -689,8 +703,7 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
             prefix = "headphones_" + os.path.splitext(file_name)[0] + "_@hp@_"
         new_folder = tempfile.mkdtemp(prefix=prefix, dir=tempdir)
     except Exception as e:
-        logger.error("Cannot create temp directory: " + tempdir.decode(
-            headphones.SYS_ENCODING, 'replace') + ". Error: " + str(e))
+        logger.error(f"Cannot create temp directory: {tempdir}. Error: {e}")
         return None
 
     # Attempt to stop multiple temp dirs being created for the same albumpath
@@ -699,19 +712,23 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
             workdir = os.path.join(tempdir, prefix)
             workdir = re.sub(r'\[', '[[]', workdir)
             workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
-            if len(glob.glob(workdir + '*/')) >= 3:
+            if len(glob(workdir + '*/')) >= 3:
                 logger.error(
-                    "Looks like a temp directory has previously been created for this albumpath, not continuing " + workdir.decode(
-                        headphones.SYS_ENCODING, 'replace'))
+                    "Looks like a temp directory has previously been created "
+                    "for this albumpath, not continuing "
+                )
                 shutil.rmtree(new_folder)
                 return None
         except Exception as e:
-            logger.warn("Cannot determine if already copied/processed, will copy anyway: Warning: " + str(e))
+            logger.warn(
+                "Cannot determine if already copied/processed, will copy anyway. "
+                f"Warning: {e}"
+            )
 
     # Copy to temp dir
     try:
         subdir = os.path.join(new_folder, "headphones")
-        logger.info("Copying files to " + subdir.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.info(f"Copying files to {subdir}")
         if not single:
             shutil.copytree(albumpath, subdir)
         else:
@@ -720,9 +737,10 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
         # Update the album path with the new location
         return subdir
     except Exception as e:
-        logger.warn("Cannot copy/move files to temp directory: " + new_folder.decode(headphones.SYS_ENCODING,
-                                                                                  'replace') + ". Not continuing. Error: " + str(
-            e))
+        logger.warn(
+            f"Cannot copy/move files to temp directory: {new_folder}. "
+            f"Not continuing. Error: {e}"
+        )
         shutil.rmtree(new_folder)
         return None
 
@@ -767,7 +785,7 @@ def cue_split(albumpath, keep_original_folder=False):
                 cuesplit.split(cue_dir)
             except Exception as e:
                 os.chdir(cwd)
-                logger.warn("Cue not split: " + str(e))
+                logger.warn(f"Cue not split. Error: {e}")
                 return None
 
         os.chdir(cwd)
@@ -805,7 +823,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid default format", s)
+        logger.info(f"Couldn't parse {s} into a valid default format")
 
     # newzbin default format
     pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s\((?P<year>\d+?\))', re.VERBOSE)
@@ -816,7 +834,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid Newbin format", s)
+        logger.info(f"Couldn't parse {s} into a valid Newbin format")
         return (name, album, year)
 
 
@@ -829,7 +847,7 @@ def smartMove(src, dest, delete=True):
     dest_path = os.path.join(dest, filename)
 
     if os.path.isfile(dest_path):
-        logger.info('Destination file exists: %s', dest_path)
+        logger.info(f"Destination file exists: {dest_path}")
         title = os.path.splitext(filename)[0]
         ext = os.path.splitext(filename)[1]
         i = 1
@@ -838,13 +856,14 @@ def smartMove(src, dest, delete=True):
             if os.path.isfile(os.path.join(dest, newfile)):
                 i += 1
             else:
-                logger.info('Renaming to %s', newfile)
+                logger.info(f"Renaming to {newfile}")
                 try:
                     os.rename(src, os.path.join(source_dir, newfile))
                     filename = newfile
+                    source_path = os.path.join(source_dir, filename)
+                    dest_path = os.path.join(dest, filename)
                 except Exception as e:
-                    logger.warn('Error renaming %s: %s',
-                                src.decode(headphones.SYS_ENCODING, 'replace'), e)
+                    logger.warn(f"Error renaming {src}: {e}")
                 break
 
     if delete:
@@ -854,8 +873,9 @@ def smartMove(src, dest, delete=True):
         except Exception as e:
             exists = os.path.exists(dest_path)
             if exists and os.path.getsize(source_path) == os.path.getsize(dest_path):
-                logger.warn('Successfully moved file "%s", but something went wrong: %s',
-                    filename.decode(headphones.SYS_ENCODING, 'replace'), e)
+                logger.warn(
+                    f"Successfully moved {filename}, but something went wrong: {e}"
+                )
                 os.unlink(source_path)
             else:
                 # remove faultly copied file
@@ -864,12 +884,11 @@ def smartMove(src, dest, delete=True):
                 raise
     else:
         try:
-            logger.info('Copying "%s" to "%s"', source_path, dest_path)
+            logger.info(f"Copying {source_path} to {dest_path}")
             shutil.copy(source_path, dest_path)
             return True
         except Exception as e:
-            logger.warn('Error copying file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
-                        e)
+            logger.warn(f"Error copying {filename}: {e}") 
 
 
 def walk_directory(basedir, followlinks=True):
@@ -878,7 +897,7 @@ def walk_directory(basedir, followlinks=True):
     with care. In case a folder is already processed, don't traverse it again.
     """
 
-    import logger
+    from . import logger
 
     # Add the base path, because symlinks poiting to the basedir should not be
     # traversed again.
@@ -892,8 +911,10 @@ def walk_directory(basedir, followlinks=True):
                 real_path = os.path.abspath(os.readlink(path))
 
                 if real_path in traversed:
-                    logger.debug("Skipping '%s' since it is a symlink to "
-                                 "'%s', which is already visited.", path, real_path)
+                    logger.debug(
+                        f"Skipping {path} since it is a symlink to "
+                        f"{real_path}, which is already visited."
+                    )
                 else:
                     traversed.append(real_path)
 
@@ -935,22 +956,15 @@ def sab_sanitize_foldername(name):
     FL_ILLEGAL = CH_ILLEGAL + ':\x92"'
     FL_LEGAL = CH_LEGAL + "-''"
 
-    uFL_ILLEGAL = FL_ILLEGAL.decode('latin-1')
-    uFL_LEGAL = FL_LEGAL.decode('latin-1')
-
     if not name:
-        return name
-    if isinstance(name, unicode):
-        illegal = uFL_ILLEGAL
-        legal = uFL_LEGAL
-    else:
-        illegal = FL_ILLEGAL
-        legal = FL_LEGAL
+        return
+
+    name = unidecode(name)
 
     lst = []
     for ch in name.strip():
-        if ch in illegal:
-            ch = legal[illegal.find(ch)]
+        if ch in FL_ILLEGAL:
+            ch = FL_LEGAL[FL_ILLEGAL.find(ch)]
             lst.append(ch)
         else:
             lst.append(ch)
@@ -1006,7 +1020,7 @@ def create_https_certificates(ssl_cert, ssl_key):
         with open(ssl_cert, "w") as fp:
             fp.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
     except IOError as e:
-        logger.error("Error creating SSL key and certificate: %s", e)
+        logger.error(f"Error creating SSL key and certificate: e")
         return False
 
     return True
@@ -1019,7 +1033,7 @@ class BeetsLogCapture(beetslogging.Handler):
         self.messages = []
 
     def emit(self, record):
-        self.messages.append(six.text_type(record.msg))
+        self.messages.append(text_type(record.msg))
 
 
 @contextmanager
@@ -1031,3 +1045,17 @@ def capture_beets_log(logger='beets'):
         yield capture.messages
     finally:
         log.removeHandler(capture)
+
+def have_pct_have_total(db_artist):
+    have_tracks = db_artist['HaveTracks'] or 0
+    total_tracks = db_artist['TotalTracks'] or 0 
+    have_pct = have_tracks / total_tracks if total_tracks else 0
+    return (have_pct, total_tracks)
+
+
+def has_token(title, token):
+    return bool(
+            re.search(rf'(?:\W|^)+{token}(?:\W|$)+', 
+            title,
+            re.IGNORECASE | re.UNICODE)
+        )

headphones/helpers_test.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from unittestcompat import TestCase
-from headphones.helpers import clean_name
+from .unittestcompat import TestCase
+from headphones.helpers import clean_name, is_valid_date, age, has_token
 
 
 class HelpersTest(TestCase):
@@ -8,28 +8,28 @@ class HelpersTest(TestCase):
     def test_clean_name(self):
         """helpers: check correctness of clean_name() function"""
         cases = {
-            u' Weiße & rose ': 'Weisse and rose',
-            u'Multiple / spaces': 'Multiple spaces',
-            u'Kevin\'s m²': 'Kevins m2',
-            u'Symphonęy Nº9': 'Symphoney No.9',
-            u'ÆæßðÞIJij': u'AeaessdThIJıj',
-            u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
+            ' Weiße & rose ': 'Weisse and rose',
+            'Multiple / spaces': 'Multiple spaces',
+            'Kevin\'s m²': 'Kevins m2',
+            'Symphonęy Nº9': 'Symphoney No.9',
+            'ÆæßðÞIJij': 'AeaessdThIJıj',
+            'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
                                                     'apoplexy remix',
-            u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
+            'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
                                                     'zbojow',
-            u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
-            u'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
-            u'L\'Arc~en~Ciel': 'larc en ciel',
-            u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
-                u'Orquesta de la Luz オルケスタ デ ラ ルス'
+            'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
+            'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
+            'L\'Arc~en~Ciel': 'larc en ciel',
+            'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
+                'Orquesta de la Luz オルケスタ デ ラ ルス'
 
         }
-        for first, second in cases.iteritems():
+        for first, second in cases.items():
             nf = clean_name(first).lower()
             ns = clean_name(second).lower()
             self.assertEqual(
-                nf, ns, u"check cleaning of case (%s,"
-                        u"%s)" % (nf, ns)
+                nf, ns, "check cleaning of case (%s,"
+                        "%s)" % (nf, ns)
             )
 
     def test_clean_name_nonunicode(self):
@@ -46,3 +46,28 @@ class HelpersTest(TestCase):
         self.assertEqual(
             test, expected, "check clean_name() with narrow non-ascii input"
         )
+
+    def test_is_valid_date(date):
+        test_cases = [
+            ('2021-11-12', True, "check is_valid_date returns True for valid date"),
+            (None, False, "check is_valid_date returns False for None"),
+            ('2021-11', False, "check is_valid_date returns False for incomplete"),
+            ('2021', False, "check is_valid_date returns False for incomplete")
+        ]
+        for input, expected, desc in test_cases:
+            self.assertEqual(is_valid_date(input), expected, desc)
+
+    def test_has_token(self):
+        """helpers: has_token()"""
+        self.assertEqual(
+            has_token("a cat ran", "cat"), 
+            True, 
+            "return True if token is in string"
+        )
+        self.assertEqual(
+            has_token("acatran", "cat"),
+            False,
+            "return False if token is part of another word"
+        )
+
+

headphones/importer.py

@@ -16,7 +16,7 @@
 import time
 
 from headphones import logger, helpers, db, mb, lastfm, metacritic
-from beets.mediafile import MediaFile
+from mediafile import MediaFile
 import headphones
 
 blacklisted_special_artist_names = ['[anonymous]', '[data]', '[no artist]',
@@ -39,7 +39,7 @@ def is_exists(artistid):
 
     if any(artistid in x for x in artistlist):
         logger.info(artistlist[0][
-                        1] + u" is already in the database. Updating 'have tracks', but not artist information")
+                        1] + " is already in the database. Updating 'have tracks', but not artist information")
         return True
     else:
         return False
@@ -53,7 +53,7 @@ def artistlist_to_mbids(artistlist, forced=False):
 
         # If adding artists through Manage New Artists, they're coming through as non-unicode (utf-8?)
         # and screwing everything up
-        if not isinstance(artist, unicode):
+        if not isinstance(artist, str):
             try:
                 artist = artist.decode('utf-8', 'replace')
             except Exception:
@@ -102,12 +102,7 @@ def artistlist_to_mbids(artistlist, forced=False):
             myDB.action('DELETE from newartists WHERE ArtistName=?', [artist])
 
     # Update the similar artist tag cloud:
-    logger.info('Updating artist information from Last.fm')
-
-    try:
-        lastfm.getSimilar()
-    except Exception as e:
-        logger.warn('Failed to update artist information from Last.fm: %s' % e)
+    lastfm.getSimilar()
 
 
 def addArtistIDListToDB(artistidlist):
@@ -184,7 +179,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     else:
         sortname = artist['artist_name']
 
-    logger.info(u"Now adding/updating: " + artist['artist_name'])
+    logger.info("Now adding/updating: " + artist['artist_name'])
     controlValueDict = {"ArtistID": artistid}
     newValueDict = {"ArtistName": artist['artist_name'],
                     "ArtistSortName": sortname,
@@ -245,7 +240,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         rgid = rg['id']
         skip_log = 0
         # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
-        pause_delta = headphones.CONFIG.MB_IGNORE_AGE
+        ignore_age = headphones.CONFIG.MB_IGNORE_AGE
 
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
 
@@ -263,8 +258,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 new_releases = mb.get_new_releases(rgid, includeExtras)
 
             else:
-                if check_release_date is None or check_release_date == u"None":
-                    if headphones.CONFIG.MB_IGNORE_AGE_MISSING is not 1:
+                if check_release_date is None or check_release_date == "None":
+                    if not headphones.CONFIG.MB_IGNORE_AGE_MISSING:
                         logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
@@ -274,18 +269,18 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     if len(check_release_date) == 10:
                         release_date = check_release_date
                     elif len(check_release_date) == 7:
-                        release_date = check_release_date + "-31"
+                        release_date = check_release_date + "-27"
                     elif len(check_release_date) == 4:
-                        release_date = check_release_date + "-12-31"
+                        release_date = check_release_date + "-12-27"
                     else:
                         release_date = today
-                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                    if helpers.age(release_date) < ignore_age:
                         logger.info("[%s] Now updating: %s (Release Date <%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
                         logger.info("[%s] Skipping: %s (Release Date >%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         skip_log = 1
                         new_releases = 0
 
@@ -450,14 +445,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
                 if headphones.CONFIG.AUTOWANT_ALL:
                     newValueDict['Status'] = "Wanted"
-                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
-                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
-                # The first test just makes sure we have year-month-day
-                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
-                        today) - helpers.get_age(
-                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
+                elif headphones.CONFIG.AUTOWANT_UPCOMING:
+                    if helpers.is_valid_date(album['ReleaseDate']) and helpers.age(album['ReleaseDate']) < 21:
+                        newValueDict['Status'] = "Wanted"
                 else:
                     newValueDict['Status'] = "Skipped"
 
@@ -517,7 +507,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     marked_as_downloaded = True
 
             logger.info(
-                u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+                "[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
             try:
                 cache.getThumb(AlbumID=rg['id'])
             except Exception as e:
@@ -530,19 +520,19 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 album_searches.append(rg['id'])
         else:
             if skip_log == 0:
-                logger.info(u"[%s] No new releases, so no changes made to %s" % (
+                logger.info("[%s] No new releases, so no changes made to %s" % (
                             artist['artist_name'], rg['title']))
 
     time.sleep(3)
     finalize_update(artistid, artist['artist_name'], errors)
 
-    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
+    logger.info("Seeing if we need album art for: %s" % artist['artist_name'])
     try:
         cache.getThumb(ArtistID=artistid)
     except Exception as e:
         logger.error("Error getting album art: %s", e)
 
-    logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
+    logger.info("Fetching Metacritic reviews for: %s" % artist['artist_name'])
     try:
         metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
     except Exception as e:
@@ -554,7 +544,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 artist['artist_name'], artist['artist_name']))
     else:
         myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
-        logger.info(u"Updating complete for: %s" % artist['artist_name'])
+        logger.info("Updating complete for: %s" % artist['artist_name'])
 
     # Start searching for newly added albums
     if album_searches:
@@ -663,7 +653,7 @@ def addReleaseById(rid, rgid=None):
             sortname = release_dict['artist_name']
 
         logger.info(
-            u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
+            "Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
         controlValueDict = {"ArtistID": release_dict['artist_id']}
         newValueDict = {"ArtistName": release_dict['artist_name'],
                         "ArtistSortName": sortname,
@@ -696,7 +686,7 @@ def addReleaseById(rid, rgid=None):
 
     if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
         # but if it is this will fail
-        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
+        logger.info("Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
         controlValueDict = {"AlbumID": rgid}
         if status != 'Loading':
             status = 'Wanted'
@@ -772,7 +762,7 @@ def addReleaseById(rid, rgid=None):
 
         # Start a search for the album
         if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
-            import searcher
+            from . import searcher
             searcher.searchforalbum(rgid, False)
 
     elif not rg_exists and not release_dict:

headphones/lastfm.py

@@ -22,8 +22,8 @@ from headphones import db, logger, request
 
 TIMEOUT = 60.0  # seconds
 REQUEST_LIMIT = 1.0 / 5  # seconds
-ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
-API_KEY = "395e6ec6bb557382fc41fde867bce66f"
+ENTRY_POINT = "https://ws.audioscrobbler.com/2.0/"
+APP_API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
 # Required for API request limit
 lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
@@ -31,7 +31,7 @@ lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
 
 def request_lastfm(method, **kwargs):
     """
-    Call a Last.FM API method. Automatically sets the method and API key. Method
+    Call a Last.fm API method. Automatically sets the method and API key. Method
     will return the result if no error occured.
 
     By default, this method will request the JSON format, since it is more
@@ -40,35 +40,42 @@ def request_lastfm(method, **kwargs):
 
     # Prepare request
     kwargs["method"] = method
-    kwargs.setdefault("api_key", API_KEY)
+    kwargs.setdefault("api_key", headphones.CONFIG.LASTFM_APIKEY or APP_API_KEY)
     kwargs.setdefault("format", "json")
 
     # Send request
-    logger.debug("Calling Last.FM method: %s", method)
-    logger.debug("Last.FM call parameters: %s", kwargs)
+    logger.debug("Calling Last.fm method: %s", method)
+    logger.debug("Last.fm call parameters: %s", kwargs)
 
     data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs, lock=lastfm_lock)
 
     # Parse response and check for errors.
     if not data:
-        logger.error("Error calling Last.FM method: %s", method)
+        logger.error("Error calling Last.fm method: %s", method)
         return
 
     if "error" in data:
-        logger.debug("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.fm returned an error: %s", data["message"])
         return
 
     return data
 
 
 def getSimilar():
-    myDB = db.DBConnection()
-    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC")
+    if not headphones.CONFIG.LASTFM_APIKEY:
+        logger.info(
+            'To update the Similar Artists cloud tag, create a Last.fm application api key '
+            'and add it under the Advanced config tab'
+        )
+        return
 
-    logger.info("Fetching similar artists from Last.FM for tag cloud")
+    myDB = db.DBConnection()
+    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC LIMIT 10")
+
+    logger.info("Fetching similar artists from Last.fm for tag cloud")
     artistlist = []
 
-    for result in results[:12]:
+    for result in results:
         data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"])
 
         if data and "similarartists" in data:
@@ -85,13 +92,13 @@ def getSimilar():
                     artistlist.append((artist_name, artist_mbid))
 
     # Add new artists to tag cloud
-    logger.debug("Fetched %d artists from Last.FM", len(artistlist))
+    logger.debug("Fetched %d artists from Last.fm", len(artistlist))
     count = defaultdict(int)
 
     for artist, mbid in artistlist:
         count[artist, mbid] += 1
 
-    items = count.items()
+    items = list(count.items())
     top_list = sorted(items, key=lambda x: x[1], reverse=True)[:25]
 
     random.shuffle(top_list)
@@ -103,7 +110,7 @@ def getSimilar():
 
         myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count])
 
-    logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list))
+    logger.debug("Inserted %d artists into Last.fm tag cloud", len(top_list))
 
 
 def getArtists():
@@ -111,16 +118,16 @@ def getArtists():
     results = myDB.select("SELECT ArtistID from artists")
 
     if not headphones.CONFIG.LASTFM_USERNAME:
-        logger.warn("Last.FM username not set, not importing artists.")
+        logger.warn("Last.fm username not set, not importing artists.")
         return
 
-    logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME)
+    logger.info("Fetching artists from Last.fm for username: %s", headphones.CONFIG.LASTFM_USERNAME)
     data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME)
 
     if data and "artists" in data:
         artistlist = []
         artists = data["artists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             artist_mbid = artist["mbid"]
@@ -133,20 +140,20 @@ def getArtists():
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.info("Imported %d new artists from Last.FM", len(artistlist))
+        logger.info("Imported %d new artists from Last.fm", len(artistlist))
 
 
 def getTagTopArtists(tag, limit=50):
     myDB = db.DBConnection()
     results = myDB.select("SELECT ArtistID from artists")
 
-    logger.info("Fetching top artists from Last.FM for tag: %s", tag)
+    logger.info("Fetching top artists from Last.fm for tag: %s", tag)
     data = request_lastfm("tag.gettopartists", limit=limit, tag=tag)
 
     if data and "topartists" in data:
         artistlist = []
         artists = data["topartists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             try:
@@ -162,4 +169,4 @@ def getTagTopArtists(tag, limit=50):
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.debug("Added %d new artists from Last.FM", len(artistlist))
+        logger.debug("Added %d new artists from Last.fm", len(artistlist))

headphones/librarysync.py

@@ -17,7 +17,7 @@ import os
 import math
 
 import headphones
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from headphones import db, logger, helpers, importer, lastfm
 
 
@@ -30,72 +30,64 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     if not dir:
         if not headphones.CONFIG.MUSIC_DIR:
+            logger.info(
+                "No music directory configured. Add it under "
+                "Manage -> Scan Music Library"
+            )
             return
         else:
             dir = headphones.CONFIG.MUSIC_DIR
 
-    # If we're appending a dir, it's coming from the post processor which is
-    # already bytestring
-    if not append or artistScan:
-        dir = dir.encode(headphones.SYS_ENCODING)
-
     if not os.path.isdir(dir):
-        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING,
-                                                                           'replace'))
+        logger.warn(f"Cannot find music directory: {dir}")
         return
 
     myDB = db.DBConnection()
     new_artists = []
 
-    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Scanning music directory: {dir}")
 
     if not append:
 
         # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
         if ArtistID:
-            tracks = myDB.action(
+            dbtracks = myDB.action(
                 'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
                 'IS NOT NULL',
                 [ArtistID, ArtistID])
         else:
-            tracks = myDB.action(
+            dbtracks = myDB.action(
                 'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL')
 
-        locations = []
-        for track in tracks:
-            locations.append(track['Location'])
-        for location in locations:
-            encoded_track_string = location.encode(headphones.SYS_ENCODING, 'replace')
-            if not os.path.isfile(encoded_track_string):
+        for track in dbtracks:
+            track_location = track['Location']
+            if not os.path.isfile(track_location):
                 myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, location])
+                            [None, None, None, track_location])
                 myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, location])
+                            [None, None, None, track_location])
 
         if ArtistName:
             del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName])
         else:
             del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have')
 
-        locations = []
         for track in del_have_tracks:
-            locations.append([track['Location'], track['ArtistName']])
-        for location in locations:
-            encoded_track_string = location[0].encode(headphones.SYS_ENCODING, 'replace')
-            if not os.path.isfile(encoded_track_string):
-                if location[1]:
+            if not os.path.isfile(track['Location']):
+                if track['ArtistName']:
                     # Make sure deleted files get accounted for when updating artist track counts
-                    new_artists.append(location[1])
-                myDB.action('DELETE FROM have WHERE Location=?', [location[0]])
+                    new_artists.append(track['ArtistName'])
+                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                 logger.info(
-                    'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
-                        headphones.SYS_ENCODING, 'replace'))
+                    f"{track['Location']} removed from Headphones, as it "
+                    f"is no longer on disk"
+                )
 
     bitrates = []
-    song_list = []
+    track_list = []
     latest_subdirectory = []
 
-    new_song_count = 0
+    new_track_count = 0
     file_count = 0
 
     for r, d, f in helpers.walk_directory(dir):
@@ -110,32 +102,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 subdirectory = r.replace(dir, '')
                 latest_subdirectory.append(subdirectory)
 
-                if file_count == 0 and r.replace(dir, '') != '':
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-                elif latest_subdirectory[file_count] != latest_subdirectory[
-                            file_count - 1] and file_count != 0:
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-
-                song = os.path.join(r, files)
-
-                # We need the unicode path to use for logging, inserting into database
-                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')
+                track_path = os.path.join(r, files)
 
                 # Try to read the metadata
                 try:
-                    f = MediaFile(song)
+                    f = MediaFile(track_path)
                 except (FileTypeError, UnreadableFileError):
-                    logger.warning(
-                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
-                        unicode_song_path)
+                    logger.warning(f"Cannot read `{track_path}`. It may be corrupted or not a media file.")
                     continue
                 except IOError:
-                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?",
-                                   unicode_song_path)
+                    logger.warning(f"Cannnot read `{track_path}`. Does the file exists?")
                     continue
 
                 # Grab the bitrates for the auto detect bit rate option
@@ -150,15 +126,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 else:
                     f_artist = None
 
-                # Add the song to our song list -
-                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
+                # Add the track to our track list -
+                # TODO: skip adding tracks without the minimum requisite information (just a matter of putting together the right if statements)
 
                 if f_artist and f.album and f.title:
                     CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title)
                 else:
                     CleanName = None
 
-                controlValueDict = {'Location': unicode_song_path}
+                controlValueDict = {'Location': track_path}
 
                 newValueDict = {'TrackID': f.mb_trackid,
                                 # 'ReleaseID' : f.mb_albumid,
@@ -174,24 +150,24 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                                 'CleanName': CleanName
                                 }
 
-                # song_list.append(song_dict)
-                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?",
-                                               [unicode_song_path]).fetchone()
-                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
-                if not check_exist_song:
+                # track_list.append(track_dict)
+                check_exist_track = myDB.action("SELECT * FROM have WHERE Location=?",
+                                               [track_path]).fetchone()
+                # Only attempt to match tracks that are new, haven't yet been matched, or metadata has changed.
+                if not check_exist_track:
                     # This is a new track
                     if f_artist:
                         new_artists.append(f_artist)
                     myDB.upsert("have", newValueDict, controlValueDict)
-                    new_song_count += 1
+                    new_track_count += 1
                 else:
-                    if check_exist_song['ArtistName'] != f_artist or check_exist_song[
-                            'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                    if check_exist_track['ArtistName'] != f_artist or check_exist_track[
+                            'AlbumTitle'] != f.album or check_exist_track['TrackTitle'] != f.title:
                         # Important track metadata has been modified, need to run matcher again
-                        if f_artist and f_artist != check_exist_song['ArtistName']:
+                        if f_artist and f_artist != check_exist_track['ArtistName']:
                             new_artists.append(f_artist)
-                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
-                                        check_exist_song['Matched'] != "Ignored":
+                        elif f_artist and f_artist == check_exist_track['ArtistName'] and \
+                                        check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
                         else:
                             continue
@@ -200,51 +176,59 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                         myDB.upsert("have", newValueDict, controlValueDict)
                         myDB.action(
                             'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
+                            [None, None, None, track_path])
                         myDB.action(
                             'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
-                        new_song_count += 1
+                            [None, None, None, track_path])
+                        new_track_count += 1
                     else:
                         # This track information hasn't changed
-                        if f_artist and check_exist_song['Matched'] != "Ignored":
+                        if f_artist and check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
 
                 file_count += 1
 
     # Now we start track matching
-    logger.info("%s new/modified songs found and added to the database" % new_song_count)
-    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-                            [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
-    total_number_of_songs = \
-        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
-    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(
-        headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+    logger.info(f"{new_track_count} new/modified tracks found and added to the database")
+    dbtracks = myDB.action(
+            "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+            [f"{dir}%"]
+        )
+    dbtracks_count = myDB.action(
+            "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+            [f"{dir}%"]
+        ).fetchone()[0]
+    logger.info(f"Found {dbtracks_count} new/modified tracks in `{dir}`")
+    logger.info("Matching tracks to the appropriate releases....")
 
-    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
-    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
 
-    # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
-    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
 
-    # We'll use this to give a % completion, just because the track matching might take a while
-    song_count = 0
-    latest_artist = []
+    # Sort the track_list by most vague (e.g. no trackid or releaseid)
+    # to most specific (both trackid & releaseid)
+    # When we insert into the database, the tracks with the most 
+    # specific information will overwrite the more general matches
+
+    sorted_dbtracks = helpers.multikeysort(dbtracks, ['ArtistName', 'AlbumTitle'])
+
+
+    # We'll use this to give a % completion, just because the
+    # track matching might take a while
+    tracks_completed = 0
+    latest_artist = None
     last_completion_percentage = 0
     prev_artist_name = None
     artistid = None
 
-    for song in song_list:
+    for track in sorted_dbtracks:
 
-        latest_artist.append(song['ArtistName'])
-        if song_count == 0:
-            logger.info("Now matching songs by %s" % song['ArtistName'])
-        elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0:
-            logger.info("Now matching songs by %s" % song['ArtistName'])
+        if latest_artist != track['ArtistName']:
+            logger.info(f"Now matching tracks by {track['ArtistName']}")
+            latest_artist = track['ArtistName']
 
-        song_count += 1
-        completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10
+        tracks_completed += 1
+        completion_percentage = math.floor(
+                float(tracks_completed) / dbtracks_count * 1000
+            ) / 10
 
         if completion_percentage >= (last_completion_percentage + 10):
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
@@ -257,9 +241,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
         albumid = None
 
-        if song['ArtistName'] and song['CleanName']:
-            artist_name = song['ArtistName']
-            clean_name = song['CleanName']
+        if track['ArtistName'] and track['CleanName']:
+            artist_name = track['ArtistName']
+            clean_name = track['CleanName']
 
             # Only update if artist is in the db
             if artist_name != prev_artist_name:
@@ -297,12 +281,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 # matching on CleanName should be enough, ensure it's the same artist just in case
 
                 # Update tracks
-                track = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
-                if track:
-                    albumid = track['AlbumID']
+                dbtrack = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
+                if dbtrack:
+                    albumid = dbtrack['AlbumID']
                     myDB.action(
                         'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
-                        [song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
 
                 # Update alltracks
                 alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
@@ -310,26 +294,25 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                     albumid = alltrack['AlbumID']
                     myDB.action(
                         'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
-                        [song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
 
         # Update have
-        controlValueDict2 = {'Location': song['Location']}
+        controlValueDict2 = {'Location': track['Location']}
         if albumid:
             newValueDict2 = {'Matched': albumid}
         else:
             newValueDict2 = {'Matched': "Failed"}
         myDB.upsert("have", newValueDict2, controlValueDict2)
 
-        # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+        # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [track['ArtistName'], track['AlbumTitle'], track['TrackNumber'], track['TrackTitle'], track['TrackLength'], track['BitRate'], track['Genre'], track['Date'], track['TrackID'], track['Location'], CleanName, track['Format']])
 
-    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
-                                                                            'replace'))
+    logger.info(f"Completed matching tracks from `{dir}`")
 
     if not append or artistScan:
         logger.info('Updating scanned artist track counts')
 
         # Clean up the new artist list
-        unique_artists = {}.fromkeys(new_artists).keys()
+        unique_artists = list({}.fromkeys(new_artists).keys())
 
         # # Don't think we need to do this, check the db instead below
         #

headphones/lock.py

@@ -4,7 +4,7 @@ Locking-related classes
 
 import time
 import threading
-import Queue
+import queue
 
 import headphones.logger
 
@@ -29,7 +29,7 @@ class TimedLock(object):
         self.lock = threading.Lock()
         self.last_used = 0
         self.minimum_delta = minimum_delta
-        self.queue = Queue.Queue()
+        self.queue = queue.Queue()
 
     def __enter__(self):
         """
@@ -47,7 +47,7 @@ class TimedLock(object):
                 seconds = self.queue.get(False)
                 headphones.logger.debug('Sleeping %s (queued)', seconds)
                 time.sleep(seconds)
-            except Queue.Empty:
+            except queue.Empty:
                 continue
             self.queue.task_done()
 

headphones/logger.py

@@ -153,7 +153,8 @@ def initLogger(console=False, log_dir=False, verbose=False):
         file_formatter = logging.Formatter(
             '%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
         file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE,
-                                                    backupCount=MAX_FILES)
+                                                    backupCount=MAX_FILES,
+                                                    encoding='utf8')
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(file_formatter)
 

headphones/lyrics.py

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import htmlentitydefs
+import html.entities
 
 import re
 from headphones import logger, request
@@ -25,7 +25,7 @@ def getLyrics(artist, song):
               "fmt": 'xml'
               }
 
-    url = 'http://lyrics.wikia.com/api.php'
+    url = 'https://lyrics.wikia.com/api.php'
     data = request.request_minidom(url, params=params)
 
     if not data:
@@ -53,7 +53,7 @@ def getLyrics(artist, song):
             '''<div class='lyricbox'><span style="padding:1em"><a href="/Category:Instrumental" title="Instrumental">''').search(
             lyricspage)
         if m:
-            return u'(Instrumental)'
+            return '(Instrumental)'
         else:
             logger.warn('Cannot find lyrics on: %s' % lyricsurl)
             return
@@ -72,7 +72,7 @@ def convert_html_entities(s):
             name = hit[2:-1]
             try:
                 entnum = int(name)
-                s = s.replace(hit, unichr(entnum))
+                s = s.replace(hit, chr(entnum))
             except ValueError:
                 pass
 
@@ -83,7 +83,7 @@ def convert_html_entities(s):
         hits.remove(amp)
     for hit in hits:
         name = hit[1:-1]
-        if name in htmlentitydefs.name2codepoint:
-            s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name]))
+        if name in html.entities.name2codepoint:
+            s = s.replace(hit, chr(html.entities.name2codepoint[name]))
     s = s.replace(amp, "&")
     return s

headphones/mb.py

@@ -14,20 +14,14 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from headphones import logger, db, helpers
+from collections import OrderedDict
+
+import musicbrainzngs
 
 import headphones
-import musicbrainzngs
 import headphones.lock
+from headphones import logger, db, helpers
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
 
 mb_lock = headphones.lock.TimedLock(0)
 
@@ -97,7 +91,7 @@ def findArtist(name, limit=1):
         try:
             artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
         except ValueError as e:
-            if "at least one query term is required" in e.message:
+            if "at least one query term is required" in str(e):
                 logger.error(
                     "Tried to search without a term, or an empty one. Provided artist (probably emtpy): %s",
                     name)
@@ -112,9 +106,9 @@ def findArtist(name, limit=1):
         return False
     for result in artistResults:
         if 'disambiguation' in result:
-            uniquename = unicode(result['sort-name'] + " (" + result['disambiguation'] + ")")
+            uniquename = str(result['sort-name'] + " (" + result['disambiguation'] + ")")
         else:
-            uniquename = unicode(result['sort-name'])
+            uniquename = str(result['sort-name'])
         if result['name'] != uniquename and limit == 1:
             logger.info(
                 'Found an artist with a disambiguation: %s - doing an album based search' % name)
@@ -124,20 +118,16 @@ def findArtist(name, limit=1):
                     'Cannot determine the best match from an artist/album search. Using top match instead')
                 artistlist.append({
                     # Just need the artist id if the limit is 1
-                    #    'name':             unicode(result['sort-name']),
-                    #    'uniquename':        uniquename,
-                    'id': unicode(result['id']),
-                    #    'url':                 unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed
-                    #    'score':            int(result['ext:score'])
+                    'id': str(result['id']),
                 })
             else:
                 artistlist.append(artistdict)
         else:
             artistlist.append({
-                'name': unicode(result['sort-name']),
+                'name': str(result['sort-name']),
                 'uniquename': uniquename,
-                'id': unicode(result['id']),
-                'url': unicode("http://musicbrainz.org/artist/" + result['id']),
+                'id': str(result['id']),
+                'url': str("https://musicbrainz.org/artist/" + result['id']),
                 # probably needs to be changed
                 'score': int(result['ext:score'])
             })
@@ -187,7 +177,7 @@ def findRelease(name, limit=1, artist=None):
                     if tracks:
                         tracks += ' + '
                     tracks += str(medium['track-count'])
-            for format, count in format_dict.items():
+            for format, count in list(format_dict.items()):
                 if formats:
                     formats += ' + '
                 if count > 1:
@@ -203,22 +193,22 @@ def findRelease(name, limit=1, artist=None):
                     rg_type = secondary_type
 
         releaselist.append({
-            'uniquename': unicode(result['artist-credit'][0]['artist']['name']),
-            'title': unicode(title),
-            'id': unicode(result['artist-credit'][0]['artist']['id']),
-            'albumid': unicode(result['id']),
-            'url': unicode(
-                "http://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),
+            'uniquename': str(result['artist-credit'][0]['artist']['name']),
+            'title': str(title),
+            'id': str(result['artist-credit'][0]['artist']['id']),
+            'albumid': str(result['id']),
+            'url': str(
+                "https://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),
             # probably needs to be changed
-            'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),
+            'albumurl': str("https://musicbrainz.org/release/" + result['id']),
             # probably needs to be changed
             'score': int(result['ext:score']),
-            'date': unicode(result['date']) if 'date' in result else '',
-            'country': unicode(result['country']) if 'country' in result else '',
-            'formats': unicode(formats),
-            'tracks': unicode(tracks),
-            'rgid': unicode(result['release-group']['id']),
-            'rgtype': unicode(rg_type)
+            'date': str(result['date']) if 'date' in result else '',
+            'country': str(result['country']) if 'country' in result else '',
+            'formats': str(formats),
+            'tracks': str(tracks),
+            'rgid': str(result['release-group']['id']),
+            'rgtype': str(rg_type)
         })
     return releaselist
 
@@ -240,15 +230,15 @@ def findSeries(name, limit=1):
         return False
     for result in seriesResults:
         if 'disambiguation' in result:
-            uniquename = unicode(result['name'] + " (" + result['disambiguation'] + ")")
+            uniquename = str(result['name'] + " (" + result['disambiguation'] + ")")
         else:
-            uniquename = unicode(result['name'])
+            uniquename = str(result['name'])
         serieslist.append({
             'uniquename': uniquename,
-            'name': unicode(result['name']),
-            'type': unicode(result['type']),
-            'id': unicode(result['id']),
-            'url': unicode("http://musicbrainz.org/series/" + result['id']),
+            'name': str(result['name']),
+            'type': str(result['type']),
+            'id': str(result['id']),
+            'url': str("https://musicbrainz.org/series/" + result['id']),
             # probably needs to be changed
             'score': int(result['ext:score'])
         })
@@ -284,19 +274,19 @@ def getArtist(artistid, extrasonly=False):
     if not artist:
         return False
 
-    artist_dict['artist_name'] = unicode(artist['name'])
+    artist_dict['artist_name'] = str(artist['name'])
 
     releasegroups = []
 
     if not extrasonly:
         for rg in artist['release-group-list']:
-            if "secondary-type-list" in rg.keys():  # only add releases without a secondary type
+            if "secondary-type-list" in list(rg.keys()):  # only add releases without a secondary type
                 continue
             releasegroups.append({
-                'title': unicode(rg['title']),
-                'id': unicode(rg['id']),
-                'url': u"http://musicbrainz.org/release-group/" + rg['id'],
-                'type': unicode(rg['type'])
+                'title': str(rg['title']),
+                'id': str(rg['id']),
+                'url': "https://musicbrainz.org/release-group/" + rg['id'],
+                'type': str(rg['type'])
             })
 
     # See if we need to grab extras. Artist specific extras take precedence over global option
@@ -314,7 +304,7 @@ def getArtist(artistid, extrasonly=False):
 
         # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] (append new extras to end)
         if db_artist['Extras']:
-            extras = map(int, db_artist['Extras'].split(','))
+            extras = list(map(int, db_artist['Extras'].split(',')))
         else:
             extras = []
         extras_list = headphones.POSSIBLE_EXTRAS
@@ -354,10 +344,10 @@ def getArtist(artistid, extrasonly=False):
                         rg_type = secondary_type
 
                 releasegroups.append({
-                    'title': unicode(rg['title']),
-                    'id': unicode(rg['id']),
-                    'url': u"http://musicbrainz.org/release-group/" + rg['id'],
-                    'type': unicode(rg_type)
+                    'title': str(rg['title']),
+                    'id': str(rg['id']),
+                    'url': "https://musicbrainz.org/release-group/" + rg['id'],
+                    'type': str(rg_type)
                 })
     artist_dict['releasegroups'] = releasegroups
     return artist_dict
@@ -382,10 +372,10 @@ def getSeries(seriesid):
         return False
 
     if 'disambiguation' in series:
-        series_dict['artist_name'] = unicode(
-            series['name'] + " (" + unicode(series['disambiguation']) + ")")
+        series_dict['artist_name'] = str(
+            series['name'] + " (" + str(series['disambiguation']) + ")")
     else:
-        series_dict['artist_name'] = unicode(series['name'])
+        series_dict['artist_name'] = str(series['name'])
 
     releasegroups = []
 
@@ -448,42 +438,42 @@ def getRelease(releaseid, include_artist_info=True):
     if not results:
         return False
 
-    release['title'] = unicode(results['title'])
-    release['id'] = unicode(results['id'])
-    release['asin'] = unicode(results['asin']) if 'asin' in results else None
-    release['date'] = unicode(results['date']) if 'date' in results else None
+    release['title'] = str(results['title'])
+    release['id'] = str(results['id'])
+    release['asin'] = str(results['asin']) if 'asin' in results else None
+    release['date'] = str(results['date']) if 'date' in results else None
     try:
-        release['format'] = unicode(results['medium-list'][0]['format'])
+        release['format'] = str(results['medium-list'][0]['format'])
     except:
-        release['format'] = u'Unknown'
+        release['format'] = 'Unknown'
 
     try:
-        release['country'] = unicode(results['country'])
+        release['country'] = str(results['country'])
     except:
-        release['country'] = u'Unknown'
+        release['country'] = 'Unknown'
 
     if include_artist_info:
 
         if 'release-group' in results:
-            release['rgid'] = unicode(results['release-group']['id'])
-            release['rg_title'] = unicode(results['release-group']['title'])
+            release['rgid'] = str(results['release-group']['id'])
+            release['rg_title'] = str(results['release-group']['title'])
             try:
-                release['rg_type'] = unicode(results['release-group']['type'])
+                release['rg_type'] = str(results['release-group']['type'])
 
                 if release['rg_type'] == 'Album' and 'secondary-type-list' in results[
                         'release-group']:
-                    secondary_type = unicode(results['release-group']['secondary-type-list'][0])
+                    secondary_type = str(results['release-group']['secondary-type-list'][0])
                     if secondary_type != release['rg_type']:
                         release['rg_type'] = secondary_type
 
             except KeyError:
-                release['rg_type'] = u'Unknown'
+                release['rg_type'] = 'Unknown'
 
         else:
             logger.warn("Release " + releaseid + "had no ReleaseGroup associated")
 
-        release['artist_name'] = unicode(results['artist-credit'][0]['artist']['name'])
-        release['artist_id'] = unicode(results['artist-credit'][0]['artist']['id'])
+        release['artist_name'] = str(results['artist-credit'][0]['artist']['name'])
+        release['artist_id'] = str(results['artist-credit'][0]['artist']['id'])
 
     release['tracks'] = getTracksFromRelease(results)
 
@@ -529,7 +519,7 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
     force_repackage1 = 0
     if len(results) != 0:
         for release_mark in results:
-            release_list.append(unicode(release_mark['id']))
+            release_list.append(str(release_mark['id']))
             release_title = release_mark['title']
         remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?",
                                               [rgid])
@@ -561,31 +551,31 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
             # DELETE all references to this release since we're updating it anyway.
             myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
             myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
-            release['AlbumTitle'] = unicode(releasedata['title'])
-            release['AlbumID'] = unicode(rgid)
-            release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
-            release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
+            release['AlbumTitle'] = str(releasedata['title'])
+            release['AlbumID'] = str(rgid)
+            release['AlbumASIN'] = str(releasedata['asin']) if 'asin' in releasedata else None
+            release['ReleaseDate'] = str(releasedata['date']) if 'date' in releasedata else None
             release['ReleaseID'] = releasedata['id']
             if 'release-group' not in releasedata:
                 raise Exception('No release group associated with release id ' + releasedata[
                     'id'] + ' album id' + rgid)
-            release['Type'] = unicode(releasedata['release-group']['type'])
+            release['Type'] = str(releasedata['release-group']['type'])
 
             if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']:
-                secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0])
+                secondary_type = str(releasedata['release-group']['secondary-type-list'][0])
                 if secondary_type != release['Type']:
                     release['Type'] = secondary_type
 
             # making the assumption that the most important artist will be first in the list
             if 'artist-credit' in releasedata:
-                release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
-                release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
+                release['ArtistID'] = str(releasedata['artist-credit'][0]['artist']['id'])
+                release['ArtistName'] = str(releasedata['artist-credit-phrase'])
             else:
                 logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
                 return False
 
-            release['ReleaseCountry'] = unicode(
-                releasedata['country']) if 'country' in releasedata else u'Unknown'
+            release['ReleaseCountry'] = str(
+                releasedata['country']) if 'country' in releasedata else 'Unknown'
             # assuming that the list will contain media and that the format will be consistent
             try:
                 additional_medium = ''
@@ -600,9 +590,9 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                     disc_number = str(medium_count) + 'x'
                 packaged_medium = disc_number + releasedata['medium-list'][0][
                     'format'] + additional_medium
-                release['ReleaseFormat'] = unicode(packaged_medium)
+                release['ReleaseFormat'] = str(packaged_medium)
             except:
-                release['ReleaseFormat'] = u'Unknown'
+                release['ReleaseFormat'] = 'Unknown'
 
             release['Tracks'] = getTracksFromRelease(releasedata)
 
@@ -684,14 +674,14 @@ def getTracksFromRelease(release):
     for medium in release['medium-list']:
         for track in medium['track-list']:
             try:
-                track_title = unicode(track['title'])
+                track_title = str(track['title'])
             except:
-                track_title = unicode(track['recording']['title'])
+                track_title = str(track['recording']['title'])
             tracks.append({
                 'number': totalTracks,
                 'title': track_title,
-                'id': unicode(track['recording']['id']),
-                'url': u"http://musicbrainz.org/track/" + track['recording']['id'],
+                'id': str(track['recording']['id']),
+                'url': "https://musicbrainz.org/track/" + track['recording']['id'],
                 'duration': int(track['length']) if 'length' in track else 0
             })
             totalTracks += 1
@@ -733,15 +723,7 @@ def findArtistbyAlbum(name):
     for releaseGroup in results:
         newArtist = releaseGroup['artist-credit'][0]['artist']
         # Only need the artist ID if we're doing an artist+album lookup
-        # if 'disambiguation' in newArtist:
-        #    uniquename = unicode(newArtist['sort-name'] + " (" + newArtist['disambiguation'] + ")")
-        # else:
-        #    uniquename = unicode(newArtist['sort-name'])
-        # artist_dict['name'] = unicode(newArtist['sort-name'])
-        # artist_dict['uniquename'] = uniquename
-        artist_dict['id'] = unicode(newArtist['id'])
-        # artist_dict['url'] = u'http://musicbrainz.org/artist/' + newArtist['id']
-        # artist_dict['score'] = int(releaseGroup['ext:score'])
+        artist_dict['id'] = str(newArtist['id'])
 
     return artist_dict
 
@@ -768,7 +750,7 @@ def findAlbumID(artist=None, album=None):
 
     if len(results) < 1:
         return False
-    rgid = unicode(results[0]['id'])
+    rgid = str(results[0]['id'])
     return rgid
 
 

headphones/metacritic.py

@@ -35,7 +35,7 @@ def update(artistid, artist_name, release_groups):
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'}
 
-    url = "http://www.metacritic.com/person/" + mc_artist_name + "?filter-options=music&sort_options=date&num_items=100"
+    url = "https://www.metacritic.com/person/" + mc_artist_name + "?filter-options=music&sort_options=date&num_items=100"
 
     res = request.request_soup(url, headers=headers, whitelist_status_code=404)
 

headphones/metadata.py

@@ -17,8 +17,8 @@
 Track/album metadata handling routines.
 """
 
-from __future__ import print_function
-from beets.mediafile import MediaFile, UnreadableFileError
+
+from mediafile import MediaFile, UnreadableFileError
 import headphones
 from headphones import logger
 import os.path
@@ -60,7 +60,7 @@ class MetadataDict(dict):
             self._lower = {}
             if seq is not None:
                 try:
-                    self.add_items(seq.iteritems())
+                    self.add_items(iter(seq.items()))
                 except KeyError:
                     self.add_items(seq)
 
@@ -79,6 +79,7 @@ class Vars:
     Metadata $variable names (only ones set explicitly by headphones).
     """
     DISC = '$Disc'
+    DISC_TOTAL = '$DiscTotal'
     TRACK = '$Track'
     TITLE = '$Title'
     ARTIST = '$Artist'
@@ -103,11 +104,11 @@ def _verify_var_type(val):
     """
     Check if type of value is allowed as a variable in pathname substitution.
     """
-    return isinstance(val, (basestring, int, float, datetime.date))
+    return isinstance(val, (str, int, float, datetime.date))
 
 
 def _as_str(val):
-    if isinstance(val, basestring):
+    if isinstance(val, str):
         return val
     else:
         return str(val)
@@ -134,7 +135,7 @@ def _row_to_dict(row, d):
     """
     Populate dict with database row fields.
     """
-    for fld in row.keys():
+    for fld in list(row.keys()):
         val = row[fld]
         if val is None:
             val = ''
@@ -171,7 +172,7 @@ def _lower(s):
     return None
 
 
-def file_metadata(path, release):
+def file_metadata(path, release, single_disc_ignore=False):
     # type: (str,sqlite3.Row)->Tuple[Mapping[str,str],bool]
     """
     Prepare metadata dictionary for path substitution, based on file name,
@@ -184,9 +185,7 @@ def file_metadata(path, release):
     try:
         f = MediaFile(path)
     except UnreadableFileError as ex:
-        logger.info("MediaFile couldn't parse: %s (%s)",
-                    path.decode(headphones.SYS_ENCODING, 'replace'),
-                    str(ex))
+        logger.info(f"MediaFile couldn't parse {path}: {e}")
         return None, None
 
     res = MetadataDict()
@@ -196,7 +195,13 @@ def file_metadata(path, release):
     _row_to_dict(release, res)
 
     date, year = _date_year(release)
-    if not f.disc:
+
+    if not f.disctotal or (f.disctotal == 1 and single_disc_ignore):
+        disc_total = ''
+    else:
+        disc_total = '%d' % f.disctotal
+
+    if not f.disc or (f.disctotal == 1 and single_disc_ignore):
         disc_number = ''
     else:
         disc_number = '%d' % f.disc
@@ -207,8 +212,7 @@ def file_metadata(path, release):
         track_number = '%02d' % f.track
 
     if not f.title:
-        basename = os.path.basename(
-            path.decode(headphones.SYS_ENCODING, 'replace'))
+        basename = os.path.basename(path)
         title = os.path.splitext(basename)[0]
         from_metadata = False
     else:
@@ -229,6 +233,7 @@ def file_metadata(path, release):
     album_title = release['AlbumTitle']
     override_values = {
         Vars.DISC: disc_number,
+        Vars.DISC_TOTAL: disc_total,
         Vars.TRACK: track_number,
         Vars.TITLE: title,
         Vars.ARTIST: artist_name,
@@ -242,7 +247,7 @@ def file_metadata(path, release):
         Vars.SORT_ARTIST_LOWER: _lower(sort_name),
         Vars.ALBUM_LOWER: _lower(album_title),
     }
-    res.add_items(override_values.iteritems())
+    res.add_items(iter(override_values.items()))
     return res, from_metadata
 
 
@@ -252,7 +257,7 @@ def _intersect(d1, d2):
     Create intersection (common part) of two dictionaries.
     """
     res = {}
-    for key, val in d1.iteritems():
+    for key, val in d1.items():
         if key in d2 and d2[key] == val:
             res[key] = val
     return res
@@ -284,21 +289,19 @@ def album_metadata(path, release, common_tags):
         sort_name = artist
 
     if not sort_name or sort_name[0].isdigit():
-        first_char = u'0-9'
+        first_char = '0-9'
     else:
         first_char = sort_name[0]
 
-    orig_folder = u''
+    orig_folder = ''
 
     # Get from temp path
     if "_@hp@_" in path:
         orig_folder = path.rsplit("headphones_", 1)[1].split("_@hp@_")[0]
-        orig_folder = orig_folder.decode(headphones.SYS_ENCODING, 'replace')
     else:
         for r, d, f in os.walk(path):
             try:
-                orig_folder = os.path.basename(
-                    os.path.normpath(r).decode(headphones.SYS_ENCODING, 'replace'))
+                orig_folder = os.path.basename(os.path.normpath(r))
                 break
             except:
                 pass
@@ -320,7 +323,7 @@ def album_metadata(path, release, common_tags):
         Vars.ORIGINAL_FOLDER_LOWER: _lower(orig_folder)
     }
     res = MetadataDict(common_tags)
-    res.add_items(override_values.iteritems())
+    res.add_items(iter(override_values.items()))
     return res
 
 
@@ -345,7 +348,7 @@ def albumart_metadata(release, common_tags):
         Vars.ALBUM_LOWER: _lower(album)
     }
     res = MetadataDict(common_tags)
-    res.add_items(override_values.iteritems())
+    res.add_items(iter(override_values.items()))
     return res
 
 

headphones/metadata_test.py

@@ -22,7 +22,7 @@ import headphones.helpers as _hp
 from headphones.metadata import MetadataDict
 import datetime
 
-from unittestcompat import TestCase
+from .unittestcompat import TestCase
 
 
 __author__ = "Andrzej Ciarkowski <andrzej.ciarkowski@gmail.com>"
@@ -50,7 +50,7 @@ class _MockDatabaseRow(object):
         self._dict = dict(d)
 
     def keys(self):
-        return self._dict.iterkeys()
+        return iter(self._dict.keys())
 
     def __getitem__(self, item):
         return self._dict[item]
@@ -63,9 +63,9 @@ class MetadataTest(TestCase):
 
     def test_metadata_dict_ci(self):
         """MetadataDict: case-insensitive lookup"""
-        expected = u'naïve'
+        expected = 'naïve'
         key_var = '$TitlE'
-        m = MetadataDict({key_var.lower(): u'naïve'})
+        m = MetadataDict({key_var.lower(): 'naïve'})
         self.assertFalse('$track' in m)
         self.assertTrue('$tITLe' in m, "cross-case lookup with 'in'")
         self.assertEqual(m[key_var], expected, "cross-case lookup success")
@@ -74,7 +74,7 @@ class MetadataTest(TestCase):
 
     def test_metadata_dict_cs(self):
         """MetadataDice: case-preserving lookup"""
-        expected_var = u'NaïVe'
+        expected_var = 'NaïVe'
         key_var = '$TitlE'
         m = MetadataDict({
             key_var.lower(): expected_var.lower(),
@@ -171,5 +171,5 @@ class MetadataTest(TestCase):
         res = _hp.pattern_substitute(
             "/music/$First/$Artist/$Artist - $Album{ [$Year]}", md, True)
 
-        self.assertEqual(res, u"/music/A/artist/artist - Album",
+        self.assertEqual(res, "/music/A/artist/artist - Album",
                          "check correct rendering of None via pattern_substitute()")

headphones/music_encoder.py

@@ -14,6 +14,7 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import time
+import datetime
 import shutil
 import subprocess
 import multiprocessing
@@ -21,11 +22,11 @@ import multiprocessing
 import os
 import headphones
 from headphones import logger
-from beets.mediafile import MediaFile
+from mediafile import MediaFile
 
 
 # xld
-import getXldProfile
+from . import getXldProfile
 
 
 def encode(albumPath):
@@ -63,8 +64,7 @@ def encode(albumPath):
         for music in f:
             if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
                 if not use_xld:
-                    encoderFormat = headphones.CONFIG.ENCODEROUTPUTFORMAT.encode(
-                        headphones.SYS_ENCODING)
+                    encoderFormat = headphones.CONFIG.ENCODEROUTPUTFORMAT
                 else:
                     xldMusicFile = os.path.join(r, music)
                     xldInfoMusic = MediaFile(xldMusicFile)
@@ -86,7 +86,7 @@ def encode(albumPath):
                     musicTempFiles.append(os.path.join(tempDirEncode, musicTemp))
 
     if headphones.CONFIG.ENCODER_PATH:
-        encoder = headphones.CONFIG.ENCODER_PATH.encode(headphones.SYS_ENCODING)
+        encoder = headphones.CONFIG.ENCODER_PATH
     else:
         if use_xld:
             encoder = os.path.join('/Applications', 'xld')
@@ -117,18 +117,17 @@ def encode(albumPath):
 
         if use_xld:
             if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate):
-                logger.info('%s has bitrate <= %skb, will not be re-encoded',
-                            music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)
+                logger.info(f"{music} has bitrate <= {xldBitrate}kb, will not be re-encoded")
             else:
                 encode = True
         elif headphones.CONFIG.ENCODER == 'lame':
             if not any(
-                    music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x
+                    music.lower().endswith('.' + x) for x
                     in ["mp3", "wav"]):
                 logger.warn('Lame cannot encode %s format for %s, use ffmpeg',
                             os.path.splitext(music)[1], music)
             else:
-                if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (
+                if music.lower().endswith('.mp3') and (
                         int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
                     logger.info('%s has bitrate <= %skb, will not be re-encoded', music,
                                 headphones.CONFIG.BITRATE)
@@ -136,13 +135,12 @@ def encode(albumPath):
                     encode = True
         else:
             if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
-                if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'):
-                    logger.warn('Cannot re-encode .ogg %s',
-                                music.decode(headphones.SYS_ENCODING, 'replace'))
+                if music.lower().endswith('.ogg'):
+                    logger.warn(f"Cannot re-encode .ogg {music}")
                 else:
                     encode = True
             else:
-                if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + headphones.CONFIG.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
+                if music.lower().endswith('.' + headphones.CONFIG.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
                     logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.CONFIG.BITRATE)
                 else:
                     encode = True
@@ -185,13 +183,13 @@ def encode(albumPath):
                 # Retrieve the results
                 results = results.get()
         else:
-            results = map(command_map, jobs)
+            results = list(map(command_map, jobs))
 
         # The results are either True or False, so determine if one is False
         encoder_failed = not all(results)
 
-    musicFiles = filter(None, musicFiles)
-    musicTempFiles = filter(None, musicTempFiles)
+    musicFiles = [_f for _f in musicFiles if _f]
+    musicTempFiles = [_f for _f in musicTempFiles if _f]
 
     # check all files to be encoded now exist in temp directory
     if not encoder_failed and musicTempFiles:
@@ -352,36 +350,31 @@ def command(encoder, musicSource, musicDest, albumPath, xldProfile):
             startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
 
     # Encode
-    logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
+    logger.info(f"Encoding {musicSource}")
     logger.debug(subprocess.list2cmdline(cmd))
 
     process = subprocess.Popen(cmd, startupinfo=startupinfo,
                                stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE)
+                               stderr=subprocess.PIPE, text=True)
     stdout, stderr = process.communicate(headphones.CONFIG.ENCODER)
 
     # Error if return code not zero
     if process.returncode:
-        logger.error(
-            'Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
-        out = stdout if stdout else stderr
-        out = out.decode(headphones.SYS_ENCODING, 'replace')
+        logger.error(f"Encoding failed for {musicSource}")
+        out = stdout or stderr
         outlast2lines = '\n'.join(out.splitlines()[-2:])
-        logger.error('%s error details: %s' % (headphones.CONFIG.ENCODER, outlast2lines))
+        logger.error(f"{headphones.CONFIG.ENCODER} error details: {outlast2lines}")
         out = out.rstrip("\n")
         logger.debug(out)
         encoded = False
     else:
-        logger.info('%s encoded in %s', musicSource, getTimeEncode(startMusicTime))
+        logger.info(f"{musicSource} encoded in {getTimeEncode(startMusicTime)}")
         encoded = True
 
     return encoded
 
 
 def getTimeEncode(start):
-    seconds = int(time.time() - start)
-    hours = seconds / 3600
-    seconds -= 3600 * hours
-    minutes = seconds / 60
-    seconds -= 60 * minutes
-    return "%02d:%02d:%02d" % (hours, minutes, seconds)
+    finish = time.time()
+    seconds = int(finish - start)
+    return datetime.timedelta(seconds=seconds)

headphones/notifiers.py

@@ -1,28 +1,13 @@
-#  This file is part of Headphones.
-#
-#  Headphones is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  Headphones is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
-
-from urllib import urlencode, quote_plus
-import urllib
+from urllib.parse import urlencode, quote_plus
+import urllib.request, urllib.parse, urllib.error
 import subprocess
 import json
 from email.mime.text import MIMEText
 import smtplib
 import email.utils
-from httplib import HTTPSConnection
-from urlparse import parse_qsl
-import urllib2
+from http.client import HTTPSConnection
+from urllib.parse import parse_qsl
+import urllib.request, urllib.error, urllib.parse
 import requests as requests
 
 import os.path
@@ -31,8 +16,8 @@ from pynma import pynma
 import cherrypy
 import headphones
 import gntp.notifier
-import oauth2 as oauth
-import pythontwitter as twitter
+#import oauth2 as oauth
+import twitter 
 
 
 class GROWL(object):
@@ -81,10 +66,10 @@ class GROWL(object):
         try:
             growl.register()
         except gntp.notifier.errors.NetworkError:
-            logger.warning(u'Growl notification failed: network error')
+            logger.warning('Growl notification failed: network error')
             return
         except gntp.notifier.errors.AuthError:
-            logger.warning(u'Growl notification failed: authentication error')
+            logger.warning('Growl notification failed: authentication error')
             return
 
         # Fix message
@@ -105,10 +90,10 @@ class GROWL(object):
                 icon=image
             )
         except gntp.notifier.errors.NetworkError:
-            logger.warning(u'Growl notification failed: network error')
+            logger.warning('Growl notification failed: network error')
             return
 
-        logger.info(u"Growl notifications sent.")
+        logger.info("Growl notifications sent.")
 
     def updateLibrary(self):
         # For uniformity reasons not removed
@@ -157,13 +142,13 @@ class PROWL(object):
         request_status = response.status
 
         if request_status == 200:
-            logger.info(u"Prowl notifications sent.")
+            logger.info("Prowl notifications sent.")
             return True
         elif request_status == 401:
-            logger.info(u"Prowl auth failed: %s" % response.reason)
+            logger.info("Prowl auth failed: %s" % response.reason)
             return False
         else:
-            logger.info(u"Prowl notification failed.")
+            logger.info("Prowl notification failed.")
             return False
 
     def updateLibrary(self):
@@ -202,7 +187,7 @@ class XBMC(object):
         self.password = headphones.CONFIG.XBMC_PASSWORD
 
     def _sendhttp(self, host, command):
-        url_command = urllib.urlencode(command)
+        url_command = urllib.parse.urlencode(command)
         url = host + '/xbmcCmds/xbmcHttp/?' + url_command
 
         if self.password:
@@ -295,10 +280,10 @@ class LMS(object):
 
         content = {'Content-Type': 'application/json'}
 
-        req = urllib2.Request(host + '/jsonrpc.js', data, content)
+        req = urllib.request.Request(host + '/jsonrpc.js', data, content)
 
         try:
-            handle = urllib2.urlopen(req)
+            handle = urllib.request.urlopen(req)
         except Exception as e:
             logger.warn('Error opening LMS url: %s' % e)
             return
@@ -424,7 +409,7 @@ class Plex(object):
                 sections = r.getElementsByTagName('Directory')
 
                 if not sections:
-                    logger.info(u"Plex Media Server not running on: " + host)
+                    logger.info("Plex Media Server not running on: " + host)
                     return False
 
                 for s in sections:
@@ -483,9 +468,9 @@ class NMA(object):
         api = headphones.CONFIG.NMA_APIKEY
         nma_priority = headphones.CONFIG.NMA_PRIORITY
 
-        logger.debug(u"NMA title: " + title)
-        logger.debug(u"NMA API: " + api)
-        logger.debug(u"NMA Priority: " + str(nma_priority))
+        logger.debug("NMA title: " + title)
+        logger.debug("NMA API: " + api)
+        logger.debug("NMA Priority: " + str(nma_priority))
 
         if snatched:
             event = snatched + " snatched!"
@@ -495,8 +480,8 @@ class NMA(object):
             message = "Headphones has downloaded and postprocessed: " + \
                       artist + ' [' + album + ']'
 
-        logger.debug(u"NMA event: " + event)
-        logger.debug(u"NMA message: " + message)
+        logger.debug("NMA event: " + event)
+        logger.debug("NMA message: " + message)
 
         batch = False
 
@@ -510,8 +495,8 @@ class NMA(object):
         response = p.push(title, event, message, priority=nma_priority,
                           batch_mode=batch)
 
-        if not response[api][u'code'] == u'200':
-            logger.error(u'Could not send notification to NotifyMyAndroid')
+        if not response[api]['code'] == '200':
+            logger.error('Could not send notification to NotifyMyAndroid')
             return False
         else:
             return True
@@ -543,10 +528,10 @@ class PUSHBULLET(object):
                                         data=json.dumps(data))
 
         if response:
-            logger.info(u"PushBullet notifications sent.")
+            logger.info("PushBullet notifications sent.")
             return True
         else:
-            logger.info(u"PushBullet notification failed.")
+            logger.info("PushBullet notification failed.")
             return False
 
 
@@ -557,9 +542,9 @@ class PUSHALOT(object):
 
         pushalot_authorizationtoken = headphones.CONFIG.PUSHALOT_APIKEY
 
-        logger.debug(u"Pushalot event: " + event)
-        logger.debug(u"Pushalot message: " + message)
-        logger.debug(u"Pushalot api: " + pushalot_authorizationtoken)
+        logger.debug("Pushalot event: " + event)
+        logger.debug("Pushalot message: " + message)
+        logger.debug("Pushalot api: " + pushalot_authorizationtoken)
 
         http_handler = HTTPSConnection("pushalot.com")
 
@@ -576,18 +561,18 @@ class PUSHALOT(object):
         response = http_handler.getresponse()
         request_status = response.status
 
-        logger.debug(u"Pushalot response status: %r" % request_status)
-        logger.debug(u"Pushalot response headers: %r" % response.getheaders())
-        logger.debug(u"Pushalot response body: %r" % response.read())
+        logger.debug("Pushalot response status: %r" % request_status)
+        logger.debug("Pushalot response headers: %r" % response.getheaders())
+        logger.debug("Pushalot response body: %r" % response.read())
 
         if request_status == 200:
-            logger.info(u"Pushalot notifications sent.")
+            logger.info("Pushalot notifications sent.")
             return True
         elif request_status == 410:
-            logger.info(u"Pushalot auth failed: %s" % response.reason)
+            logger.info("Pushalot auth failed: %s" % response.reason)
             return False
         else:
-            logger.info(u"Pushalot notification failed.")
+            logger.info("Pushalot notification failed.")
             return False
 
 
@@ -618,7 +603,7 @@ class JOIN(object):
         else:
             self.url += '&deviceId={deviceid}'
 
-        response = urllib2.urlopen(self.url.format(apikey=self.apikey,
+        response = urllib.request.urlopen(self.url.format(apikey=self.apikey,
                                                    title=quote_plus(event),
                                                    text=quote_plus(
                                                        message.encode(
@@ -627,10 +612,10 @@ class JOIN(object):
                                                    deviceid=self.deviceid))
 
         if response:
-            logger.info(u"Join notifications sent.")
+            logger.info("Join notifications sent.")
             return True
         else:
-            logger.error(u"Join notification failed.")
+            logger.error("Join notification failed.")
             return False
 
 
@@ -669,7 +654,7 @@ class Synoindex(object):
             out, error = p.communicate()
             # synoindex never returns any codes other than '0',
             #  highly irritating
-        except OSError, e:
+        except OSError as e:
             logger.warn("Error sending notification: %s" % str(e))
 
     def notify_multiple(self, path_list):
@@ -710,10 +695,10 @@ class PUSHOVER(object):
                                             headers=headers, data=data)
 
         if response:
-            logger.info(u"Pushover notifications sent.")
+            logger.info("Pushover notifications sent.")
             return True
         else:
-            logger.error(u"Pushover notification failed.")
+            logger.error("Pushover notification failed.")
             return False
 
     def updateLibrary(self):
@@ -832,7 +817,7 @@ class TwitterNotifier(object):
         access_token_key = headphones.CONFIG.TWITTER_USERNAME
         access_token_secret = headphones.CONFIG.TWITTER_PASSWORD
 
-        logger.info(u"Sending tweet: " + message)
+        logger.info("Sending tweet: " + message)
 
         api = twitter.Api(username, password, access_token_key,
                           access_token_secret)
@@ -840,7 +825,7 @@ class TwitterNotifier(object):
         try:
             api.PostUpdate(message)
         except Exception as e:
-            logger.info(u"Error Sending Tweet: %s" % e)
+            logger.info("Error Sending Tweet: %s" % e)
             return False
 
         return True
@@ -935,10 +920,10 @@ class BOXCAR(object):
     def notify(self, title, message, rgid=None):
         try:
             if rgid:
-                message += '<br></br><a href="http://musicbrainz.org/' \
+                message += '<br></br><a href="https://musicbrainz.org/' \
                            'release-group/%s">MusicBrainz</a>' % rgid
 
-            data = urllib.urlencode({
+            data = urllib.parse.urlencode({
                 'user_credentials': headphones.CONFIG.BOXCAR_TOKEN,
                 'notification[title]': title.encode('utf-8'),
                 'notification[long_message]': message.encode('utf-8'),
@@ -947,12 +932,12 @@ class BOXCAR(object):
                                           "/headphoneslogo.png"
             })
 
-            req = urllib2.Request(self.url)
-            handle = urllib2.urlopen(req, data)
+            req = urllib.request.Request(self.url)
+            handle = urllib.request.urlopen(req, data)
             handle.close()
             return True
 
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
             logger.warn('Error sending Boxcar2 Notification: %s' % e)
             return False
 
@@ -1011,7 +996,7 @@ class Email(object):
             mailserver.quit()
             return True
 
-        except Exception, e:
+        except Exception as e:
             logger.warn('Error sending Email: %s' % e)
             return False
 
@@ -1034,7 +1019,7 @@ class TELEGRAM(object):
 
         # MusicBrainz link
         if rgid:
-            message += '\n\n <a href="http://musicbrainz.org/' \
+            message += '\n\n <a href="https://musicbrainz.org/' \
                       'release-group/%s">MusicBrainz</a>' % rgid
 
         # Send image
@@ -1044,15 +1029,15 @@ class TELEGRAM(object):
             payload = {'chat_id': userid, 'parse_mode': "HTML", 'caption': status + message}
             try:
                 response = requests.post(TELEGRAM_API % (token, "sendPhoto"), data=payload, files=image_file)
-            except Exception, e:
-                logger.info(u'Telegram notify failed: ' + str(e))
+            except Exception as e:
+                logger.info('Telegram notify failed: ' + str(e))
         # Sent text
         else:
             payload = {'chat_id': userid, 'parse_mode': "HTML", 'text': status + message}
             try:
                 response = requests.post(TELEGRAM_API % (token, "sendMessage"), data=payload)
-            except Exception, e:
-                logger.info(u'Telegram notify failed: ' + str(e))
+            except Exception as e:
+                logger.info('Telegram notify failed: ' + str(e))
 
         # Error logging
         sent_successfuly = True
@@ -1060,7 +1045,7 @@ class TELEGRAM(object):
             logger.info("Could not send notification to TelegramBot (token=%s). Response: [%s]", token, response.text)
             sent_successfuly = False
 
-        logger.info(u"Telegram notifications sent.")
+        logger.info("Telegram notifications sent.")
         return sent_successfuly
 
 
@@ -1080,15 +1065,15 @@ class SLACK(object):
 
         try:
             response = requests.post(SLACK_URL, json=payload)
-        except Exception, e:
-            logger.info(u'Slack notify failed: ' + str(e))
+        except Exception as e:
+            logger.info('Slack notify failed: ' + str(e))
 
         sent_successfuly = True
         if not response.status_code == 200:
             logger.info(
-                u'Could not send notification to Slack. Response: [%s]',
+                'Could not send notification to Slack. Response: [%s]',
                 (response.text))
             sent_successfuly = False
 
-        logger.info(u"Slack notifications sent.")
+        logger.info("Slack notifications sent.")
         return sent_successfuly

headphones/nzbget.py

@@ -20,8 +20,8 @@
 
 
 from base64 import standard_b64encode
-import httplib
-import xmlrpclib
+import http.client
+import xmlrpc.client
 
 import headphones
 from headphones import logger
@@ -32,7 +32,7 @@ def sendNZB(nzb):
     nzbgetXMLrpc = "%(protocol)s://%(username)s:%(password)s@%(host)s/xmlrpc"
 
     if not headphones.CONFIG.NZBGET_HOST:
-        logger.error(u"No NZBget host found in configuration. Please configure it.")
+        logger.error("No NZBget host found in configuration. Please configure it.")
         return False
 
     if headphones.CONFIG.NZBGET_HOST.startswith('https://'):
@@ -46,34 +46,35 @@ def sendNZB(nzb):
                           "username": headphones.CONFIG.NZBGET_USERNAME,
                           "password": headphones.CONFIG.NZBGET_PASSWORD}
 
-    nzbGetRPC = xmlrpclib.ServerProxy(url)
+    nzbGetRPC = xmlrpc.client.ServerProxy(url)
     try:
         if nzbGetRPC.writelog("INFO", "headphones connected to drop of %s any moment now." % (
                 nzb.name + ".nzb")):
-            logger.debug(u"Successfully connected to NZBget")
+            logger.debug("Successfully connected to NZBget")
         else:
-            logger.info(u"Successfully connected to NZBget, but unable to send a message" % (
+            logger.info("Successfully connected to NZBget, but unable to send a message" % (
                 nzb.name + ".nzb"))
 
-    except httplib.socket.error:
+    except http.client.socket.error:
         logger.error(
-            u"Please check your NZBget host and port (if it is running). NZBget is not responding to this combination")
+            "Please check your NZBget host and port (if it is running). NZBget is not responding to this combination")
         return False
 
-    except xmlrpclib.ProtocolError, e:
+    except xmlrpc.client.ProtocolError as e:
         if e.errmsg == "Unauthorized":
-            logger.error(u"NZBget password is incorrect.")
+            logger.error("NZBget password is incorrect.")
         else:
-            logger.error(u"Protocol Error: " + e.errmsg)
+            logger.error("Protocol Error: " + e.errmsg)
         return False
 
     nzbcontent64 = None
     if nzb.resultType == "nzbdata":
         data = nzb.extraInfo[0]
-        nzbcontent64 = standard_b64encode(data)
+        # NZBGet needs a string, not bytes
+        nzbcontent64 = standard_b64encode(data).decode("utf-8")
 
-    logger.info(u"Sending NZB to NZBget")
-    logger.debug(u"URL: " + url)
+    logger.info("Sending NZB to NZBget")
+    logger.debug("URL: " + url)
 
     dupekey = ""
     dupescore = 0
@@ -131,12 +132,12 @@ def sendNZB(nzb):
                                                     nzb.url)
 
         if nzbget_result:
-            logger.debug(u"NZB sent to NZBget successfully")
+            logger.debug("NZB sent to NZBget successfully")
             return True
         else:
-            logger.error(u"NZBget could not add %s to the queue" % (nzb.name + ".nzb"))
+            logger.error("NZBget could not add %s to the queue" % (nzb.name + ".nzb"))
             return False
     except:
         logger.error(
-            u"Connect Error to NZBget: could not add %s to the queue" % (nzb.name + ".nzb"))
+            "Connect Error to NZBget: could not add %s to the queue" % (nzb.name + ".nzb"))
         return False

headphones/pathrender.py

@@ -30,7 +30,7 @@ syntax elements are supported:
   nonempty value only if any variable or optional inside returned
   nonempty value, ignoring literals (like {'{'$That'}'}).
 """
-from __future__ import print_function
+
 from enum import Enum
 
 __author__ = "Andrzej Ciarkowski <andrzej.ciarkowski@gmail.com>"
@@ -111,9 +111,9 @@ class _OptionalBlock(_Generator):
         # type: (Mapping[str,str]) -> str
         res = [(isinstance(x, _Generator), x.render(replacement)) for x in self._scope]
         if any((t[0] and t[1] is not None and len(t[1]) != 0) for t in res):
-            return u"".join(t[1] for t in res)
+            return "".join(t[1] for t in res)
         else:
-            return u""
+            return ""
 
     def __eq__(self, other):
         """
@@ -122,15 +122,15 @@ class _OptionalBlock(_Generator):
         return isinstance(other, _OptionalBlock) and self._scope == other._scope
 
 
-_OPTIONAL_START = u'{'
-_OPTIONAL_END = u'}'
-_ESCAPE_CHAR = u'\''
-_REPLACEMENT_START = u'$'
+_OPTIONAL_START = '{'
+_OPTIONAL_END = '}'
+_ESCAPE_CHAR = '\''
+_REPLACEMENT_START = '$'
 
 
 def _is_replacement_valid(c):
     # type: (str) -> bool
-    return c.isalnum() or c == u'_'
+    return c.isalnum() or c == '_'
 
 
 class _State(Enum):
@@ -243,7 +243,7 @@ class Pattern(object):
     def __call__(self, replacement):
         # type: (Mapping[str,str]) -> str
         '''Execute path rendering/substitution based on replacement dictionary.'''
-        return u"".join(p.render(replacement) for p in self._pattern)
+        return "".join(p.render(replacement) for p in self._pattern)
 
     def _get_warnings(self):
         # type: () -> str
@@ -262,6 +262,6 @@ def render(pattern, replacement):
 
 if __name__ == "__main__":
     # primitive test ;)
-    p = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]}")
-    d = {'$Disc': '', '$Track': '05', '$Artist': u'Grzegżółka', '$Title': u'Błona kapłona', '$Year': '2019'}
-    assert p(d) == u"05 - Grzegżółka - Błona kapłona [2019]"
+    p = Pattern("{$Disc.}$Track - $Artist - $Title{ [$Year]}")
+    d = {'$Disc': '', '$Track': '05', '$Artist': 'Grzegżółka', '$Title': 'Błona kapłona', '$Year': '2019'}
+    assert p(d) == "05 - Grzegżółka - Błona kapłona [2019]"

headphones/pathrender_test.py

@@ -19,7 +19,7 @@ Test module for pathrender.
 import headphones.pathrender as _pr
 from headphones.pathrender import Pattern, Warnings
 
-from unittestcompat import TestCase
+from .unittestcompat import TestCase
 
 
 __author__ = "Andrzej Ciarkowski <andrzej.ciarkowski@gmail.com>"
@@ -32,21 +32,21 @@ class PathRenderTest(TestCase):
 
     def test_parsing(self):
         """pathrender: pattern parsing"""
-        pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]}")
+        pattern = Pattern("{$Disc.}$Track - $Artist - $Title{ [$Year]}")
         expected = [
             _pr._OptionalBlock([
-                _pr._Replacement(u"$Disc"),
-                _pr._LiteralText(u".")
+                _pr._Replacement("$Disc"),
+                _pr._LiteralText(".")
             ]),
-            _pr._Replacement(u"$Track"),
-            _pr._LiteralText(u" - "),
-            _pr._Replacement(u"$Artist"),
-            _pr._LiteralText(u" - "),
-            _pr._Replacement(u"$Title"),
+            _pr._Replacement("$Track"),
+            _pr._LiteralText(" - "),
+            _pr._Replacement("$Artist"),
+            _pr._LiteralText(" - "),
+            _pr._Replacement("$Title"),
             _pr._OptionalBlock([
-                _pr._LiteralText(u" ["),
-                _pr._Replacement(u"$Year"),
-                _pr._LiteralText(u"]")
+                _pr._LiteralText(" ["),
+                _pr._Replacement("$Year"),
+                _pr._LiteralText("]")
             ])
         ]
         self.assertEqual(expected, pattern._pattern)
@@ -54,27 +54,27 @@ class PathRenderTest(TestCase):
 
     def test_parsing_warnings(self):
         """pathrender: pattern parsing with warnings"""
-        pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]")
+        pattern = Pattern("{$Disc.}$Track - $Artist - $Title{ [$Year]")
         self.assertEqual(set([Warnings.UNCLOSED_OPTIONAL]), pattern.warnings)
-        pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]'}")
+        pattern = Pattern("{$Disc.}$Track - $Artist - $Title{ [$Year]'}")
         self.assertEqual(set([Warnings.UNCLOSED_ESCAPE, Warnings.UNCLOSED_OPTIONAL]), pattern.warnings)
 
     def test_replacement(self):
         """pathrender: _Replacement variable substitution"""
-        r = _pr._Replacement(u"$Title")
+        r = _pr._Replacement("$Title")
         subst = {'$Title': 'foo', '$Track': 'bar'}
         res = r.render(subst)
-        self.assertEqual(res, u'foo', 'check valid replacement')
+        self.assertEqual(res, 'foo', 'check valid replacement')
         subst = {}
         res = r.render(subst)
-        self.assertEqual(res, u'$Title', 'check missing replacement')
+        self.assertEqual(res, '$Title', 'check missing replacement')
         subst = {'$Title': None}
         res = r.render(subst)
         self.assertEqual(res, '', 'check render() works with None')
 
     def test_literal(self):
         """pathrender: _Literal text rendering"""
-        l = _pr._LiteralText(u"foo")
+        l = _pr._LiteralText("foo")
         subst = {'$foo': 'bar'}
         res = l.render(subst)
         self.assertEqual(res, 'foo')
@@ -82,12 +82,12 @@ class PathRenderTest(TestCase):
     def test_optional(self):
         """pathrender: _OptionalBlock element processing"""
         o = _pr._OptionalBlock([
-            _pr._Replacement(u"$Title"),
-            _pr._LiteralText(u".foobar")
+            _pr._Replacement("$Title"),
+            _pr._LiteralText(".foobar")
         ])
         subst = {'$Title': 'foo', '$Track': 'bar'}
         res = o.render(subst)
-        self.assertEqual(res, u'foo.foobar', 'check non-empty replacement')
+        self.assertEqual(res, 'foo.foobar', 'check non-empty replacement')
         subst = {'$Title': ''}
         res = o.render(subst)
         self.assertEqual(res, '', 'check empty replacement')

headphones/postprocessor.py

@@ -25,9 +25,9 @@ import headphones
 from beets import autotag
 from beets import config as beetsconfig
 from beets import logging as beetslogging
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from beetsplug import lyrics as beetslyrics
-from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
+from headphones import notifiers, utorrent, transmission, deluge, qbittorrent, soulseek
 from headphones import db, albumart, librarysync
 from headphones import logger, helpers, mb, music_encoder
 from headphones import metadata
@@ -36,18 +36,45 @@ postprocessor_lock = threading.Lock()
 
 
 def checkFolder():
-    logger.debug("Checking download folder for completed downloads (only snatched ones).")
+    logger.info("Checking download folder for completed downloads (only snatched ones).")
 
     with postprocessor_lock:
         myDB = db.DBConnection()
         snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"')
-
         for album in snatched:
             if album['FolderName']:
                 folder_name = album['FolderName']
                 single = False
-                if album['Kind'] == 'nzb':
-                    download_dir = headphones.CONFIG.DOWNLOAD_DIR
+
+                # Soulseek, check download complete or errored
+                if album['Kind'] == 'soulseek':
+                    match = re.search(r'\{(.*?)\}(.*?)$', folder_name)    # get soulseek user from folder_name
+                    user_name = match.group(1)
+                    folder_name = match.group(2)
+                    completed, errored = soulseek.download_completed_album(user_name, folder_name)
+                    if errored:
+                        # If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted.
+                        logger.info(f"Soulseek: Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.")
+                        myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],))
+                        myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE AlbumID=?', (album['AlbumID'],))
+                        
+                        # Folder will be removed from configured complete and Incomplete directory
+                        complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name)
+                        incomplete_path = os.path.join(headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR, folder_name)
+                        for path in [complete_path, incomplete_path]:
+                            try:
+                                shutil.rmtree(path)
+                            except Exception as e:
+                                pass
+                        continue
+                    elif completed:
+                        download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR
+                    else:
+                        continue
+                elif album['Kind'] == 'nzb':
+                    download_dir = headphones.CONFIG.DOWNLOAD_DIR   
+                elif album['Kind'] == 'bandcamp':
+                    download_dir = headphones.CONFIG.BANDCAMP_DIR 
                 else:
                     if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
                         download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -65,8 +92,7 @@ def checkFolder():
                             folder_name = torrent_folder_name
 
                 if folder_name:
-                    album_path = os.path.join(download_dir, folder_name).encode(
-                        headphones.SYS_ENCODING, 'replace')
+                    album_path = os.path.join(download_dir, folder_name)
                     logger.debug("Checking if %s exists" % album_path)
 
                     if os.path.exists(album_path):
@@ -135,11 +161,11 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
                     if headphones.CONFIG.RENAME_FROZEN:
                         renameUnprocessedFolder(albumpath, tag="Frozen")
                     else:
-                        logger.warn(u"Won't rename %s to mark as 'Frozen', because it is disabled.",
-                                    albumpath.decode(headphones.SYS_ENCODING, 'replace'))
+                        logger.warn("Won't rename %s to mark as 'Frozen', because it is disabled.",
+                                    albumpath)
                 return
 
-        logger.info(u"Now adding/updating artist: " + release_dict['artist_name'])
+        logger.info("Now adding/updating artist: " + release_dict['artist_name'])
 
         if release_dict['artist_name'].startswith('The '):
             sortname = release_dict['artist_name'][4:]
@@ -161,7 +187,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
 
         myDB.upsert("artists", newValueDict, controlValueDict)
 
-        logger.info(u"Now adding album: " + release_dict['title'])
+        logger.info("Now adding album: " + release_dict['title'])
         controlValueDict = {"AlbumID": albumid}
 
         newValueDict = {"ArtistID": release_dict['artist_id'],
@@ -202,7 +228,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
         newValueDict = {"Status": "Paused"}
 
         myDB.upsert("artists", newValueDict, controlValueDict)
-        logger.info(u"Addition complete for: " + release_dict['title'] + " - " + release_dict[
+        logger.info("Addition complete for: " + release_dict['title'] + " - " + release_dict[
             'artist_name'])
 
         release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
@@ -211,17 +237,18 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
     downloaded_track_list = []
     downloaded_cuecount = 0
 
-    for r, d, f in os.walk(albumpath):
-        for files in f:
-            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
-                downloaded_track_list.append(os.path.join(r, files))
-            elif files.lower().endswith('.cue'):
+    media_extensions = tuple(map(lambda x: '.' + x, headphones.MEDIA_FORMATS))
+
+    for root, dirs, files in os.walk(albumpath):
+        for file in files:
+            if file.endswith(media_extensions):
+                downloaded_track_list.append(os.path.join(root, file))
+            elif file.endswith('.cue'):
                 downloaded_cuecount += 1
             # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though
-            elif files.lower().endswith(('.part', '.utpart')) and not forced:
+            elif file.endswith(('.part', '.utpart')) and not forced:
                 logger.info(
-                    "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING,
-                                                                       'replace') + " isn't complete yet. Will try again on the next run")
+                    "Looks like " + os.path.basename(albumpath) + " isn't complete yet. Will try again on the next run")
                 return
 
     # Force single file through
@@ -264,10 +291,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
         try:
             f = MediaFile(downloaded_track)
         except Exception as e:
-            logger.info(
-                u"Exception from MediaFile for: " + downloaded_track.decode(headphones.SYS_ENCODING,
-                                                                            'replace') + u" : " + unicode(
-                    e))
+            logger.info(f"Exception from MediaFile for {downloaded_track}: {e}")
             continue
 
         if not f.artist:
@@ -275,10 +299,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
         if not f.album:
             continue
 
-        metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8')
-        dbartist = helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8')
-        metaalbum = helpers.latinToAscii(f.album.lower()).encode('UTF-8')
-        dbalbum = helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8')
+        metaartist = helpers.latinToAscii(f.artist.lower())
+        dbartist = helpers.latinToAscii(release['ArtistName'].lower())
+        metaalbum = helpers.latinToAscii(f.album.lower())
+        dbalbum = helpers.latinToAscii(release['AlbumTitle'].lower())
 
         logger.debug('Matching metadata artist: %s with artist name: %s' % (metaartist, dbartist))
         logger.debug('Matching metadata album: %s with album name: %s' % (metaalbum, dbalbum))
@@ -292,14 +316,14 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
     logger.debug('Metadata check failed. Verifying filenames...')
     for downloaded_track in downloaded_track_list:
         track_name = os.path.splitext(downloaded_track)[0]
-        split_track_name = re.sub('[\.\-\_]', ' ', track_name).lower()
+        split_track_name = re.sub(r'[\.\-\_]', r' ', track_name).lower()
         for track in tracks:
 
             if not track['TrackTitle']:
                 continue
 
-            dbtrack = helpers.latinToAscii(track['TrackTitle'].lower()).encode('UTF-8')
-            filetrack = helpers.latinToAscii(split_track_name).encode('UTF-8')
+            dbtrack = helpers.latinToAscii(track['TrackTitle'].lower())
+            filetrack = helpers.latinToAscii(split_track_name)
             logger.debug('Checking if track title: %s is in file name: %s' % (dbtrack, filetrack))
 
             if dbtrack in filetrack:
@@ -340,11 +364,9 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
                                  keep_original_folder, forced, single)
                 return
 
-    logger.warn(u'Could not identify album: %s. It may not be the intended album.',
-                albumpath.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.warn(f"Could not identify {albumpath}. It may not be the intended album")
     markAsUnprocessed(albumid, albumpath, keep_original_folder)
 
-
 def markAsUnprocessed(albumid, albumpath, keep_original_folder=False):
     myDB = db.DBConnection()
     myDB.action(
@@ -354,13 +376,19 @@ def markAsUnprocessed(albumid, albumpath, keep_original_folder=False):
         if headphones.CONFIG.RENAME_UNPROCESSED and not keep_original_folder:
             renameUnprocessedFolder(albumpath, tag="Unprocessed")
         else:
-            logger.warn(u"Won't rename %s to mark as 'Unprocessed', because it is disabled or folder is being kept.",
-                        albumpath.decode(headphones.SYS_ENCODING, 'replace'))
+            logger.warn(
+                f"Won't rename {albumpath} to mark as 'Unprocessed', "
+                f"because it is disabled or folder is being kept."
+            )
+            return
 
 
 def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind=None,
                      keep_original_folder=False, forced=False, single=False):
-    logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle']))
+    logger.info(
+        f"Starting post-processing for: {release['ArtistName']} - "
+        f"{release['AlbumTitle']}"
+    )
     new_folder = None
 
     # Preserve the torrent dir
@@ -393,12 +421,10 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
             f = MediaFile(downloaded_track)
             builder.add_media_file(f)
         except (FileTypeError, UnreadableFileError):
-            logger.error("Track file is not a valid media file: %s. Not continuing.",
-                         downloaded_track.decode(headphones.SYS_ENCODING, "replace"))
+            logger.error(f"`{downloaded_track}` is not a valid media file. Not continuing.")
             return
         except IOError:
-            logger.error("Unable to find media file: %s. Not continuing.", downloaded_track.decode(
-                headphones.SYS_ENCODING, "replace"))
+            logger.error(f"Unable to find `{downloaded_track}`. Not continuing.")
             if new_folder:
                 shutil.rmtree(new_folder)
             return
@@ -416,9 +442,10 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                     fp.seek(0)
             except IOError as e:
                 logger.debug("Write check exact error: %s", e)
-                logger.error("Track file is not writable. This is required "
-                             "for some post processing steps: %s. Not continuing.",
-                             downloaded_track.decode(headphones.SYS_ENCODING, "replace"))
+                logger.error(
+                    f"`{downloaded_track}` is not writable. This is required "
+                     "for some post processing steps. Not continuing."
+                )
                 if new_folder:
                     shutil.rmtree(new_folder)
                 return
@@ -475,7 +502,8 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
     else:
         albumpaths = [albumpath]
 
-    updateFilePermissions(albumpaths)
+    if headphones.CONFIG.FILE_PERMISSIONS_ENABLED:
+        updateFilePermissions(albumpaths)
 
     myDB = db.DBConnection()
     myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
@@ -491,7 +519,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         if seed_snatched:
             hash = seed_snatched['TorrentHash']
             torrent_removed = False
-            logger.info(u'%s - %s. Checking if torrent has finished seeding and can be removed' % (
+            logger.info('%s - %s. Checking if torrent has finished seeding and can be removed' % (
                 release['ArtistName'], release['AlbumTitle']))
             if headphones.CONFIG.TORRENT_DOWNLOADER == 1:
                 torrent_removed = transmission.removeTorrent(hash, True)
@@ -517,18 +545,18 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                                 ArtistName=release['ArtistName'])
 
     logger.info(
-        u'Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle']))
+        'Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle']))
 
     pushmessage = release['ArtistName'] + ' - ' + release['AlbumTitle']
     statusmessage = "Download and Postprocessing completed"
 
     if headphones.CONFIG.GROWL_ENABLED:
-        logger.info(u"Growl request")
+        logger.info("Growl request")
         growl = notifiers.GROWL()
         growl.notify(pushmessage, statusmessage)
 
     if headphones.CONFIG.PROWL_ENABLED:
-        logger.info(u"Prowl request")
+        logger.info("Prowl request")
         prowl = notifiers.PROWL()
         prowl.notify(pushmessage, statusmessage)
 
@@ -559,7 +587,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         nma.notify(release['ArtistName'], release['AlbumTitle'])
 
     if headphones.CONFIG.PUSHALOT_ENABLED:
-        logger.info(u"Pushalot request")
+        logger.info("Pushalot request")
         pushalot = notifiers.PUSHALOT()
         pushalot.notify(pushmessage, statusmessage)
 
@@ -569,35 +597,36 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
             syno.notify(albumpath)
 
     if headphones.CONFIG.PUSHOVER_ENABLED:
-        logger.info(u"Pushover request")
+        logger.info("Pushover request")
         pushover = notifiers.PUSHOVER()
         pushover.notify(pushmessage, "Headphones")
 
     if headphones.CONFIG.PUSHBULLET_ENABLED:
-        logger.info(u"PushBullet request")
+        logger.info("PushBullet request")
         pushbullet = notifiers.PUSHBULLET()
         pushbullet.notify(pushmessage, statusmessage)
 
     if headphones.CONFIG.JOIN_ENABLED:
-        logger.info(u"Join request")
+        logger.info("Join request")
         join = notifiers.JOIN()
         join.notify(pushmessage, statusmessage)
 
     if headphones.CONFIG.TELEGRAM_ENABLED:
-        logger.info(u"Telegram request")
+        logger.info("Telegram request")
         telegram = notifiers.TELEGRAM()
         telegram.notify(statusmessage, pushmessage)
 
     if headphones.CONFIG.TWITTER_ENABLED:
-        logger.info(u"Sending Twitter notification")
-        twitter = notifiers.TwitterNotifier()
-        twitter.notify_download(pushmessage)
+        logger.info("Twitter notifications temporarily disabled")
+        #logger.info("Sending Twitter notification")
+        #twitter = notifiers.TwitterNotifier()
+        #twitter.notify_download(pushmessage)
 
     if headphones.CONFIG.OSX_NOTIFY_ENABLED:
         from headphones import cache
         c = cache.Cache()
         album_art = c.get_artwork_from_cache(None, release['AlbumID'])
-        logger.info(u"Sending OS X notification")
+        logger.info("Sending OS X notification")
         osx_notify = notifiers.OSX_NOTIFY()
         osx_notify.notify(release['ArtistName'],
                           release['AlbumTitle'],
@@ -605,13 +634,13 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                           image=album_art)
 
     if headphones.CONFIG.BOXCAR_ENABLED:
-        logger.info(u"Sending Boxcar2 notification")
+        logger.info("Sending Boxcar2 notification")
         boxcar = notifiers.BOXCAR()
         boxcar.notify('Headphones processed: ' + pushmessage,
                       statusmessage, release['AlbumID'])
 
     if headphones.CONFIG.SUBSONIC_ENABLED:
-        logger.info(u"Sending Subsonic update")
+        logger.info("Sending Subsonic update")
         subsonic = notifiers.SubSonicNotifier()
         subsonic.notify(albumpaths)
 
@@ -620,7 +649,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         mpc.notify()
 
     if headphones.CONFIG.EMAIL_ENABLED:
-        logger.info(u"Sending Email notification")
+        logger.info("Sending Email notification")
         email = notifiers.Email()
         subject = release['ArtistName'] + ' - ' + release['AlbumTitle']
         email.notify(subject, "Download and Postprocessing completed")
@@ -636,23 +665,21 @@ def embedAlbumArt(artwork, downloaded_track_list):
         try:
             f = MediaFile(downloaded_track)
         except:
-            logger.error(u'Could not read %s. Not adding album art' % downloaded_track.decode(
-                headphones.SYS_ENCODING, 'replace'))
+            logger.error(f"Could not read {downloaded_track}. Not adding album art")
             continue
 
-        logger.debug('Adding album art to: %s' % downloaded_track)
+        logger.debug(f"Adding album art to `{downloaded_track}`")
 
         try:
             f.art = artwork
             f.save()
         except Exception as e:
-            logger.error(u'Error embedding album art to: %s. Error: %s' % (
-                downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e)))
+            logger.error(f"Error embedding album art to `{downloaded_track}`: {e}")
             continue
 
 
 def addAlbumArt(artwork, albumpath, release, metadata_dict):
-    logger.info('Adding album art to folder')
+    logger.info(f"Adding album art to `{albumpath}`")
     md = metadata.album_metadata(albumpath, release, metadata_dict)
 
     ext = ".jpg"
@@ -663,8 +690,7 @@ def addAlbumArt(artwork, albumpath, release, metadata_dict):
     album_art_name = helpers.pattern_substitute(
         headphones.CONFIG.ALBUM_ART_FORMAT.strip(), md) + ext
 
-    album_art_name = helpers.replace_illegal_chars(album_art_name).encode(
-        headphones.SYS_ENCODING, 'replace')
+    album_art_name = helpers.replace_illegal_chars(album_art_name)
 
     if headphones.CONFIG.FILE_UNDERSCORES:
         album_art_name = album_art_name.replace(' ', '_')
@@ -684,14 +710,13 @@ def cleanupFiles(albumpath):
     logger.info('Cleaning up files')
 
     for r, d, f in os.walk(albumpath):
-        for files in f:
-            if not any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
-                logger.debug('Removing: %s' % files)
+        for file in f:
+            if not any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
+                logger.debug('Removing: %s' % file)
                 try:
-                    os.remove(os.path.join(r, files))
+                    os.remove(os.path.join(r, file))
                 except Exception as e:
-                    logger.error(u'Could not remove file: %s. Error: %s' % (
-                        files.decode(headphones.SYS_ENCODING, 'replace'), e))
+                    logger.error('Could not remove file: %s. Error: %s' % (file, e))
 
 
 def renameNFO(albumpath):
@@ -701,19 +726,16 @@ def renameNFO(albumpath):
         for file in f:
             if file.lower().endswith('.nfo'):
                 if not file.lower().endswith('.orig.nfo'):
-                    logger.debug('Renaming: "%s" to "%s"' % (
-                        file.decode(headphones.SYS_ENCODING, 'replace'),
-                        file.decode(headphones.SYS_ENCODING, 'replace') + '-orig'))
                     try:
                         new_file_name = os.path.join(r, file)[:-3] + 'orig.nfo'
+                        logger.debug(f"Renaming `{file}` to `{new_file_name}`")
                         os.rename(os.path.join(r, file), new_file_name)
                     except Exception as e:
-                        logger.error(u'Could not rename file: %s. Error: %s' % (
-                            os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e))
+                        logger.error(f"Could not rename {file}: {e}")
 
 
 def moveFiles(albumpath, release, metadata_dict):
-    logger.info("Moving files: %s" % albumpath)
+    logger.info(f"Moving files: `{albumpath}`")
 
     md = metadata.album_metadata(albumpath, release, metadata_dict)
     folder = helpers.pattern_substitute(
@@ -750,12 +772,8 @@ def moveFiles(albumpath, release, metadata_dict):
     make_lossy_folder = False
     make_lossless_folder = False
 
-    lossy_destination_path = os.path.normpath(
-        os.path.join(headphones.CONFIG.DESTINATION_DIR, folder)).encode(headphones.SYS_ENCODING,
-                                                                        'replace')
-    lossless_destination_path = os.path.normpath(
-        os.path.join(headphones.CONFIG.LOSSLESS_DESTINATION_DIR, folder)).encode(
-        headphones.SYS_ENCODING, 'replace')
+    lossy_destination_path = os.path.join(headphones.CONFIG.DESTINATION_DIR, folder)
+    lossless_destination_path = os.path.join(headphones.CONFIG.LOSSLESS_DESTINATION_DIR, folder)
 
     # If they set a destination dir for lossless media, only create the lossy folder if there is lossy media
     if headphones.CONFIG.LOSSLESS_DESTINATION_DIR:
@@ -780,8 +798,9 @@ def moveFiles(albumpath, release, metadata_dict):
                     shutil.rmtree(lossless_destination_path)
                 except Exception as e:
                     logger.error(
-                        "Error deleting existing folder: %s. Creating duplicate folder. Error: %s" % (
-                            lossless_destination_path.decode(headphones.SYS_ENCODING, 'replace'), e))
+                        f"Error deleting `{lossless_destination_path}`. "
+                        f"Creating duplicate folder. Error: {e}"
+                    )
                     create_duplicate_folder = True
 
             if not headphones.CONFIG.REPLACE_EXISTING_FOLDERS or create_duplicate_folder:
@@ -791,8 +810,11 @@ def moveFiles(albumpath, release, metadata_dict):
                 while True:
                     newfolder = temp_folder + '[%i]' % i
                     lossless_destination_path = os.path.normpath(
-                        os.path.join(headphones.CONFIG.LOSSLESS_DESTINATION_DIR, newfolder)).encode(
-                        headphones.SYS_ENCODING, 'replace')
+                        os.path.join(
+                            headphones.CONFIG.LOSSLESS_DESTINATION_DIR, 
+                            newfolder
+                        )
+                    )
                     if os.path.exists(lossless_destination_path):
                         i += 1
                     else:
@@ -818,8 +840,9 @@ def moveFiles(albumpath, release, metadata_dict):
                     shutil.rmtree(lossy_destination_path)
                 except Exception as e:
                     logger.error(
-                        "Error deleting existing folder: %s. Creating duplicate folder. Error: %s" % (
-                            lossy_destination_path.decode(headphones.SYS_ENCODING, 'replace'), e))
+                        f"Error deleting `{lossy_destination_path}`. "
+                        f"Creating duplicate folder. Error: {e}"
+                    )
                     create_duplicate_folder = True
 
             if not headphones.CONFIG.REPLACE_EXISTING_FOLDERS or create_duplicate_folder:
@@ -829,8 +852,11 @@ def moveFiles(albumpath, release, metadata_dict):
                 while True:
                     newfolder = temp_folder + '[%i]' % i
                     lossy_destination_path = os.path.normpath(
-                        os.path.join(headphones.CONFIG.DESTINATION_DIR, newfolder)).encode(
-                        headphones.SYS_ENCODING, 'replace')
+                        os.path.join(
+                            headphones.CONFIG.DESTINATION_DIR, 
+                            newfolder
+                        )
+                    )
                     if os.path.exists(lossy_destination_path):
                         i += 1
                     else:
@@ -876,12 +902,11 @@ def moveFiles(albumpath, release, metadata_dict):
                         os.remove(file_to_move)
                     except Exception as e:
                         logger.error(
-                            "Error deleting file '" + file_to_move.decode(headphones.SYS_ENCODING,
-                                                                          'replace') + "' from source directory")
+                            f"Error deleting `{file_to_move}` from source directory") 
                 else:
-                    logger.error("Error copying '" + file_to_move.decode(headphones.SYS_ENCODING,
-                                                                         'replace') + "'. Not deleting from download directory")
-
+                    logger.error(
+                        f"Error copying `{file_to_move}`. "
+                        f"Not deleting from download directory")
     elif make_lossless_folder and not make_lossy_folder:
 
         for file_to_move in files_to_move:
@@ -910,20 +935,20 @@ def moveFiles(albumpath, release, metadata_dict):
 
             if headphones.CONFIG.FOLDER_PERMISSIONS_ENABLED:
                 try:
-                    os.chmod(os.path.normpath(temp_f).encode(headphones.SYS_ENCODING, 'replace'),
+                    os.chmod(os.path.normpath(temp_f),
                              int(headphones.CONFIG.FOLDER_PERMISSIONS, 8))
                 except Exception as e:
-                    logger.error("Error trying to change permissions on folder: %s. %s",
-                                 temp_f.decode(headphones.SYS_ENCODING, 'replace'), e)
+                    logger.error(f"Error trying to change permissions on `{temp_f}`: {e}")
             else:
-                logger.debug("Not changing folder permissions, since it is disabled: %s",
-                             temp_f.decode(headphones.SYS_ENCODING, 'replace'))
+                logger.debug(
+                    f"Not changing permissions on `{temp_f}`, "
+                    "since it is disabled")
 
     # If we failed to move all the files out of the directory, this will fail too
     try:
         shutil.rmtree(albumpath)
     except Exception as e:
-        logger.error('Could not remove directory: %s. %s', albumpath, e)
+        logger.error(f"Could not remove `{albumpath}`: {e}")
 
     destination_paths = []
 
@@ -952,11 +977,15 @@ def correctMetadata(albumid, release, downloaded_track_list):
                      headphones.LOSSY_MEDIA_FORMATS):
                 lossy_items.append(beets.library.Item.from_path(downloaded_track))
             else:
-                logger.warn("Skipping: %s because it is not a mutagen friendly file format",
-                            downloaded_track.decode(headphones.SYS_ENCODING, 'replace'))
+                logger.warn(
+                    f"Skipping `{downloaded_track}` because it is "
+                    f"not a mutagen friendly file format"
+                )
+                continue
         except Exception as e:
-            logger.error("Beets couldn't create an Item from: %s - not a media file? %s",
-                         downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e))
+            logger.error(
+                f"Beets couldn't create an Item from `{downloaded_track}`: {e}")
+            continue
 
     for items in [lossy_items, lossless_items]:
 
@@ -1018,11 +1047,9 @@ def correctMetadata(albumid, release, downloaded_track_list):
         for item in items:
             try:
                 item.write()
-                logger.info("Successfully applied metadata to: %s",
-                            item.path.decode(headphones.SYS_ENCODING, 'replace'))
+                logger.info(f"Successfully applied metadata to `{item.path}`")
             except Exception as e:
-                logger.warn("Error writing metadata to '%s': %s",
-                            item.path.decode(headphones.SYS_ENCODING, 'replace'), str(e))
+                logger.warn(f"Error writing metadata to `{item.path}: {e}")
                 return False
 
         return True
@@ -1048,11 +1075,11 @@ def embedLyrics(downloaded_track_list):
                      headphones.LOSSY_MEDIA_FORMATS):
                 lossy_items.append(beets.library.Item.from_path(downloaded_track))
             else:
-                logger.warn("Skipping: %s because it is not a mutagen friendly file format",
-                            downloaded_track.decode(headphones.SYS_ENCODING, 'replace'))
+                logger.warn(
+                    f"Skipping `{downloaded_track}` because it is "
+                    f"not a mutagen friendly file format")
         except Exception as e:
-            logger.error("Beets couldn't create an Item from: %s - not a media file? %s",
-                         downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e))
+            logger.error(f"Beets couldn't create an Item from `{downloaded_track}`: {e}")
 
     for items in [lossy_items, lossless_items]:
 
@@ -1067,7 +1094,7 @@ def embedLyrics(downloaded_track_list):
                 if any(lyrics):
                     break
 
-            lyrics = u"\n\n---\n\n".join([l for l in lyrics if l])
+            lyrics = "\n\n---\n\n".join([l for l in lyrics if l])
 
             if lyrics:
                 logger.debug('Adding lyrics to: %s', item.title)
@@ -1085,7 +1112,11 @@ def renameFiles(albumpath, downloaded_track_list, release):
     # Until tagging works better I'm going to rely on the already provided metadata
 
     for downloaded_track in downloaded_track_list:
-        md, from_metadata = metadata.file_metadata(downloaded_track, release)
+        md, from_metadata = metadata.file_metadata(
+            downloaded_track,
+            release,
+            headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE
+        )
         if md is None:
             # unable to parse media file, skip file
             continue
@@ -1099,8 +1130,7 @@ def renameFiles(albumpath, downloaded_track_list, release):
                 headphones.CONFIG.FILE_FORMAT.strip(), md
             ).replace('/', '_') + ext
 
-        new_file_name = helpers.replace_illegal_chars(new_file_name).encode(
-            headphones.SYS_ENCODING, 'replace')
+        new_file_name = helpers.replace_illegal_chars(new_file_name)
 
         if headphones.CONFIG.FILE_UNDERSCORES:
             new_file_name = new_file_name.replace(' ', '_')
@@ -1111,37 +1141,28 @@ def renameFiles(albumpath, downloaded_track_list, release):
         new_file = os.path.join(albumpath, new_file_name)
 
         if downloaded_track == new_file_name:
-            logger.debug("Renaming for: " + downloaded_track.decode(
-                headphones.SYS_ENCODING, 'replace') + " is not neccessary")
+            logger.debug(f"Renaming for {downloaded_track} is not neccessary")
             continue
 
-        logger.debug('Renaming %s ---> %s',
-                     downloaded_track.decode(headphones.SYS_ENCODING, 'replace'),
-                     new_file_name.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.debug(f"Renaming {downloaded_track} ---> {new_file_name}")
         try:
             os.rename(downloaded_track, new_file)
         except Exception as e:
-            logger.error('Error renaming file: %s. Error: %s',
-                         downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), e)
+            logger.error(f"Error renaming {downloaded_track}: {e}")
             continue
 
 
 def updateFilePermissions(albumpaths):
     for folder in albumpaths:
-        logger.info("Updating file permissions in %s", folder)
+        logger.info(f"Updating file permissions in `{folder}`")
         for r, d, f in os.walk(folder):
             for files in f:
                 full_path = os.path.join(r, files)
-                if headphones.CONFIG.FILE_PERMISSIONS_ENABLED:
-                    try:
-                        os.chmod(full_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8))
-                    except:
-                        logger.error("Could not change permissions for file: %s", full_path)
-                        continue
-                else:
-                    logger.debug("Not changing file permissions, since it is disabled: %s",
-                                 full_path.decode(headphones.SYS_ENCODING, 'replace'))
-
+                try:
+                    os.chmod(full_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8))
+                except:
+                    logger.error(f"Could not change permissions for `{full_path}`")
+                    continue
 
 def renameUnprocessedFolder(path, tag):
     """
@@ -1168,18 +1189,21 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
     ignored = 0
 
     if album_dir:
-        folders = [album_dir.encode(headphones.SYS_ENCODING, 'replace')]
+        folders = [album_dir]
     else:
         download_dirs = []
 
         if dir:
-            download_dirs.append(dir.encode(headphones.SYS_ENCODING, 'replace'))
-        if headphones.CONFIG.DOWNLOAD_DIR and not dir:
-            download_dirs.append(
-                headphones.CONFIG.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
-        if headphones.CONFIG.DOWNLOAD_TORRENT_DIR and not dir:
-            download_dirs.append(
-                headphones.CONFIG.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
+            download_dirs.append(dir)
+        else:
+            if headphones.CONFIG.DOWNLOAD_DIR:
+                download_dirs.append(headphones.CONFIG.DOWNLOAD_DIR)
+            if headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR:
+                download_dirs.append(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR)
+            if headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
+                download_dirs.append(headphones.CONFIG.DOWNLOAD_TORRENT_DIR)
+            if headphones.CONFIG.BANDCAMP:
+                download_dirs.append(headphones.CONFIG.BANDCAMP_DIR)
 
         # If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
         download_dirs = list(set(download_dirs))
@@ -1189,6 +1213,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
         folders = []
 
         for download_dir in download_dirs:
+            download_dir = download_dir.encode(headphones.SYS_ENCODING, 'replace')
             if not os.path.isdir(download_dir):
                 logger.warn('Directory %s does not exist. Skipping', download_dir)
                 continue
@@ -1206,9 +1231,9 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                     subfolders = helpers.expand_subfolders(path_to_folder)
 
                     if expand_subfolders and subfolders is not None:
-                        folders.extend(subfolders)
+                        folders.extend(subfolders.decode(headphones.SYS_ENCODING, 'replace'))
                     else:
-                        folders.append(path_to_folder)
+                        folders.append(path_to_folder.decode(headphones.SYS_ENCODING, 'replace'))
 
     # Log number of folders
     if folders:
@@ -1223,7 +1248,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
     myDB = db.DBConnection()
 
     for folder in folders:
-        folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
+        folder_basename = os.path.basename(folder)
         logger.info('Processing: %s', folder_basename)
 
         # Attempt 1: First try to see if there's a match in the snatched table,

headphones/qbittorrent.py

@@ -13,9 +13,9 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import urllib
-import urllib2
-import cookielib
+import urllib.request, urllib.parse, urllib.error
+import urllib.request, urllib.error, urllib.parse
+import http.cookiejar
 import json
 import time
 import mimetypes
@@ -61,23 +61,23 @@ class qbittorrentclient(object):
             self.version = 2
         except Exception as e:
             logger.warning("Error with qBittorrent v2 api, check settings or update, will try v1: %s" % e)
-            self.cookiejar = cookielib.CookieJar()
+            self.cookiejar = http.cookiejar.CookieJar()
             self.opener = self._make_opener()
             self._get_sid(self.base_url, self.username, self.password)
             self.version = 1
 
     def _make_opener(self):
         # create opener with cookie handler to carry QBitTorrent SID cookie
-        cookie_handler = urllib2.HTTPCookieProcessor(self.cookiejar)
+        cookie_handler = urllib.request.HTTPCookieProcessor(self.cookiejar)
         handlers = [cookie_handler]
-        return urllib2.build_opener(*handlers)
+        return urllib.request.build_opener(*handlers)
 
     def _get_sid(self, base_url, username, password):
         # login so we can capture SID cookie
-        login_data = urllib.urlencode({'username': username, 'password': password})
+        login_data = urllib.parse.urlencode({'username': username, 'password': password})
         try:
             self.opener.open(base_url + '/login', login_data)
-        except urllib2.URLError as err:
+        except urllib.error.URLError as err:
             logger.debug('Error getting SID. qBittorrent responded with error: ' + str(err.reason))
             return
         for cookie in self.cookiejar:
@@ -95,14 +95,14 @@ class qbittorrentclient(object):
             data, headers = encode_multipart(args, files)
         else:
             if args:
-                data = urllib.urlencode(args)
+                data = urllib.parse.urlencode(args)
             if content_type:
                 headers['Content-Type'] = content_type
 
         logger.debug('%s' % json.dumps(headers, indent=4))
         logger.debug('%s' % data)
 
-        request = urllib2.Request(url, data, headers)
+        request = urllib.request.Request(url, data, headers)
         try:
             response = self.opener.open(request)
             info = response.info()
@@ -117,7 +117,7 @@ class qbittorrentclient(object):
                         return response.code, json.loads(resp)
             logger.debug('response code: %s' % str(response.code))
             return response.code, None
-        except urllib2.URLError as err:
+        except urllib.error.URLError as err:
             logger.debug('Failed URL: %s' % url)
             logger.debug('QBitTorrent webUI raised the following error: %s' % str(err))
             return None, None
@@ -319,7 +319,7 @@ def encode_multipart(args, files, boundary=None):
     lines = []
 
     if args:
-        for name, value in args.items():
+        for name, value in list(args.items()):
             lines.extend((
                 '--{0}'.format(boundary),
                 'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
@@ -329,7 +329,7 @@ def encode_multipart(args, files, boundary=None):
     logger.debug(''.join(lines))
 
     if files:
-        for name, value in files.items():
+        for name, value in list(files.items()):
             filename = value['filename']
             if 'mimetype' in value:
                 mimetype = value['mimetype']

headphones/request.py

@@ -23,6 +23,9 @@ from headphones import logger
 import feedparser
 import headphones
 import headphones.lock
+from bs4.builder import XMLParsedAsHTMLWarning
+import warnings
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
 
 
 # Disable SSL certificate warnings. We have our own handling
@@ -138,7 +141,7 @@ def request_soup(url, **kwargs):
     no exceptions are raised.
     """
 
-    parser = kwargs.pop("parser", "html5lib")
+    parser = kwargs.pop("parser", "html.parser")
     response = request_response(url, **kwargs)
 
     if response is not None:
@@ -222,7 +225,7 @@ def server_message(response):
     if response.headers.get("content-type") and \
                     "text/html" in response.headers.get("content-type"):
         try:
-            soup = BeautifulSoup(response.content, "html5lib")
+            soup = BeautifulSoup(response.content, "html.parser")
         except Exception:
             pass
 
@@ -245,7 +248,9 @@ def server_message(response):
 
     if message:
         # Truncate message if it is too long.
-        if len(message) > 150:
-            message = message[:150] + "..."
+        if len(message) > 200:
+            if not type(message) == str:
+                message = message.decode(headphones.SYS_ENCODING, 'replace')
+            message = message[:200] + "..."
 
         logger.debug("Server responded with message: %s", message)

headphones/rutracker.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import time
-from urlparse import urlparse
+from urllib.parse import urlparse
 import re
 
 import requests as requests
@@ -11,6 +11,7 @@ from bs4 import BeautifulSoup
 
 import headphones
 from headphones import logger
+from headphones.types import Result
 
 
 class Rutracker(object):
@@ -19,13 +20,13 @@ class Rutracker(object):
         self.timeout = 60
         self.loggedin = False
         self.maxsize = 0
-        self.search_referer = 'http://rutracker.org/forum/tracker.php'
+        self.search_referer = 'https://rutracker.org/forum/tracker.php'
 
     def logged_in(self):
         return self.loggedin
 
     def still_logged_in(self, html):
-        if not html or "action=\"http://rutracker.org/forum/login.php\">" in html:
+        if not html or "action=\"https://rutracker.org/forum/login.php\">" in html:
             return False
         else:
             return True
@@ -35,25 +36,28 @@ class Rutracker(object):
         Logs in user
         """
 
-        loginpage = 'http://rutracker.org/forum/login.php'
+        loginpage = 'https://rutracker.org/forum/login.php'
         post_params = {
             'login_username': headphones.CONFIG.RUTRACKER_USER,
             'login_password': headphones.CONFIG.RUTRACKER_PASSWORD,
             'login': b'\xc2\xf5\xee\xe4'  # '%C2%F5%EE%E4'
         }
+        headers = {
+            'User-Agent' : 'Headphones'   
+        }
 
         logger.info("Attempting to log in to rutracker...")
 
         try:
-            r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
+            r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers)
             # try again
             if not self.has_bb_session_cookie(r):
                 time.sleep(10)
                 if headphones.CONFIG.RUTRACKER_COOKIE:
                     logger.info("Attempting to log in using predefined cookie...")
-                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE})
+                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE})
                 else:
-                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
+                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers)
             if self.has_bb_session_cookie(r):
                 self.loggedin = True
                 logger.info("Successfully logged in to rutracker")
@@ -68,10 +72,10 @@ class Rutracker(object):
             return self.loggedin
 
     def has_bb_session_cookie(self, response):
-        if 'bb_session' in response.cookies.keys():
+        if 'bb_session' in list(response.cookies.keys()):
             return True
         # Rutracker randomly send a 302 redirect code, cookie may be present in response history
-        return next(('bb_session' in r.cookies.keys() for r in response.history), False)
+        return next(('bb_session' in list(r.cookies.keys()) for r in response.history), False)
 
     def searchurl(self, artist, album, year, format):
         """
@@ -99,10 +103,10 @@ class Rutracker(object):
         # sort by size, descending.
         sort = '&o=7&s=2'
         try:
-            searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort)
+            searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.parse.quote(searchterm), format, sort)
         except:
             searchterm = searchterm.encode('utf-8')
-            searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort)
+            searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.parse.quote(searchterm), format, sort)
         logger.info("Searching rutracker using term: %s", searchterm)
 
         return searchurl
@@ -112,9 +116,12 @@ class Rutracker(object):
         Parse the search results and return valid torrent list
         """
         try:
-            headers = {'Referer': self.search_referer}
+            headers = {
+                'Referer': self.search_referer,
+                'User-Agent' : 'Headphones'   
+            }
             r = self.session.get(url=searchurl, headers=headers, timeout=self.timeout)
-            soup = BeautifulSoup(r.content, 'html5lib')
+            soup = BeautifulSoup(r.content, 'html.parser')
 
             # Debug
             # logger.debug (soup.prettify())
@@ -123,7 +130,7 @@ class Rutracker(object):
             if not self.still_logged_in(soup):
                 self.login()
                 r = self.session.get(url=searchurl, timeout=self.timeout)
-                soup = BeautifulSoup(r.content, 'html5lib')
+                soup = BeautifulSoup(r.content, 'html.parser')
                 if not self.still_logged_in(soup):
                     logger.error("Error getting rutracker data")
                     return None
@@ -159,8 +166,8 @@ class Rutracker(object):
                     # Torrent topic page
                     torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])[
                         't']
-                    topicurl = 'http://rutracker.org/forum/viewtopic.php?t=' + torrent_id
-                    rulist.append((title, size, topicurl, 'rutracker.org', 'torrent', True))
+                    topicurl = 'https://rutracker.org/forum/viewtopic.php?t=' + torrent_id
+                    rulist.append(Result(title, size, url, 'rutracker.org', 'torrent', True))
                 else:
                     logger.info("%s is larger than the maxsize or has too little seeders for this category, "
                                 "skipping. (Size: %i bytes, Seeders: %i)" % (title, size, int(seeds)))
@@ -179,10 +186,13 @@ class Rutracker(object):
         return the .torrent data
         """
         torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
-        downloadurl = 'http://rutracker.org/forum/dl.php?t=' + torrent_id
+        downloadurl = 'https://rutracker.org/forum/dl.php?t=' + torrent_id
         cookie = {'bb_dl': torrent_id}
         try:
-            headers = {'Referer': url}
+            headers = {
+                'Referer': url,
+                'User-Agent' : 'Headphones'   
+            }
             r = self.session.post(url=downloadurl, cookies=cookie, headers=headers,
                                   timeout=self.timeout)
             return r.content

headphones/sab.py

@@ -17,7 +17,7 @@
 # Stolen from Sick-Beard's sab.py #
 ###################################
 
-import cookielib
+import http.cookiejar
 
 import headphones
 from headphones.common import USER_AGENT
@@ -74,29 +74,28 @@ def sendNZB(nzb):
 
     # if we get a raw data result we want to upload it to SAB
     elif nzb.resultType == "nzbdata":
-        # Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
-        nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
+        nzbdata = nzb.extraInfo[0]
         params['mode'] = 'addfile'
-        files = {"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)}
+        files = {"nzbfile": (nzb.name + ".nzb", nzbdata)}
         headers = {'User-Agent': USER_AGENT}
 
     logger.info("Attempting to connect to SABnzbd on url: %s" % headphones.CONFIG.SAB_HOST)
     if nzb.resultType == "nzb":
         response = sab_api_call('send_nzb', params=params)
     elif nzb.resultType == "nzbdata":
-        cookies = cookielib.CookieJar()
+        cookies = http.cookiejar.CookieJar()
         response = sab_api_call('send_nzb', params=params, method="post", files=files,
                                 cookies=cookies, headers=headers)
 
     if not response:
-        logger.info(u"No data returned from SABnzbd, NZB not sent")
+        logger.info("No data returned from SABnzbd, NZB not sent")
         return False
 
     if response['status']:
-        logger.info(u"NZB sent to SABnzbd successfully")
+        logger.info("NZB sent to SABnzbd successfully")
         return True
     else:
-        logger.error(u"Error sending NZB to SABnzbd: %s" % response['error'])
+        logger.error("Error sending NZB to SABnzbd: %s" % response['error'])
         return False
 
 

headphones/softchroot_test.py

@@ -38,8 +38,8 @@ class SoftChrootTest(TestCase):
             cf = SoftChroot(path)
         self.assertIsNone(cf)
 
-        self.assertRegexpMatches(str(exc.exception), r'No such directory')
-        self.assertRegexpMatches(str(exc.exception), path)
+        self.assertRegex(str(exc.exception), r'No such directory')
+        self.assertRegex(str(exc.exception), path)
 
     @mock.patch('headphones.softchroot.os', wrap=os, name='OsMock')
     def test_create_on_file(self, os_mock):
@@ -57,8 +57,8 @@ class SoftChrootTest(TestCase):
 
         self.assertTrue(os_mock.path.isdir.called)
 
-        self.assertRegexpMatches(str(exc.exception), r'No such directory')
-        self.assertRegexpMatches(str(exc.exception), path)
+        self.assertRegex(str(exc.exception), r'No such directory')
+        self.assertRegex(str(exc.exception), path)
 
     @TestArgs(
         (None, None),

headphones/soulseek.py

@@ -0,0 +1,252 @@
+from collections import defaultdict, namedtuple
+import os
+import time
+import slskd_api
+import headphones
+from headphones import logger
+from datetime import datetime, timedelta
+
+Result = namedtuple('Result', ['title', 'size', 'user', 'provider', 'type', 'matches', 'bandwidth', 'hasFreeUploadSlot', 'queueLength', 'files', 'kind', 'url', 'folder'])
+
+def initialize_soulseek_client():
+    host = headphones.CONFIG.SOULSEEK_API_URL
+    api_key = headphones.CONFIG.SOULSEEK_API_KEY
+    return slskd_api.SlskdClient(host=host, api_key=api_key)
+
+    # Search logic, calling search and processing fucntions
+def search(artist, album, year, num_tracks, losslessOnly, allow_lossless, user_search_term):
+    client = initialize_soulseek_client()
+
+    # override search string with user provided search term if entered
+    if user_search_term:
+        artist = user_search_term
+        album = ''
+        year = ''
+    
+    # Stage 1: Search with artist, album, year, and num_tracks
+    logger.info(f"Searching Soulseek using term: {artist} {album} {year}")
+    results = execute_search(client, artist, album, year, losslessOnly, allow_lossless)
+    processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks)
+    if processed_results or user_search_term or album.lower() == artist.lower():
+        return processed_results
+    
+    # Stage 2: If Stage 1 fails, search with artist, album, and num_tracks (excluding year)
+    logger.info("Soulseek search stage 1 did not meet criteria. Retrying without year...")
+    results = execute_search(client, artist, album, None, losslessOnly, allow_lossless)
+    processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks)
+    if processed_results or artist == "Various Artists":
+        return processed_results
+    
+    # Stage 3: Final attempt, search only with artist and album
+    logger.info("Soulseek search stage 2 did not meet criteria. Final attempt with only artist and album.")
+    results = execute_search(client, artist, album, None, losslessOnly, allow_lossless)
+    processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=True)
+
+    return processed_results
+
+def execute_search(client, artist, album, year, losslessOnly, allow_lossless):
+    search_text = f"{artist} {album}"
+    if year:
+        search_text += f" {year}"
+
+    if losslessOnly:
+        search_text += " flac"
+    elif not allow_lossless:
+            search_text += " mp3"
+
+    # Actual search
+    search_response = client.searches.search_text(searchText=search_text, filterResponses=True)
+    search_id = search_response.get('id')
+
+    # Wait for search completion and return response
+    while not client.searches.state(id=search_id).get('isComplete'):
+        time.sleep(2)
+    
+    return client.searches.search_responses(id=search_id)
+
+# Processing the search result passed
+def process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=False):
+
+    if losslessOnly:
+        valid_extensions = {'.flac'}
+    elif allow_lossless:
+        valid_extensions = {'.mp3', '.flac'}
+    else:
+        valid_extensions = {'.mp3'}
+
+    albums = defaultdict(lambda: {'files': [], 'user': None, 'hasFreeUploadSlot': None, 'queueLength': None, 'uploadSpeed': None})
+
+    # Extract info from the api response and combine files at album level
+    for result in results:
+        user = result.get('username')
+        hasFreeUploadSlot = result.get('hasFreeUploadSlot')
+        queueLength = result.get('queueLength')
+        uploadSpeed = result.get('uploadSpeed')
+
+        # Only handle .mp3 and .flac
+        for file in result.get('files', []):
+            filename = file.get('filename')
+            file_extension = os.path.splitext(filename)[1].lower()
+            if file_extension in valid_extensions:
+                #album_directory = os.path.dirname(filename)
+                album_directory = filename.rsplit('\\', 1)[0]
+                albums[album_directory]['files'].append(file)
+
+                # Update metadata only once per album_directory
+                if albums[album_directory]['user'] is None:
+                    albums[album_directory].update({
+                        'user': user,
+                        'hasFreeUploadSlot': hasFreeUploadSlot,
+                        'queueLength': queueLength,
+                        'uploadSpeed': uploadSpeed,
+                    })
+
+    # Filter albums based on num_tracks, add bunch of useful info to the compiled album
+    final_results = []
+    for directory, album_data in albums.items():
+        if ignore_track_count and len(album_data['files']) > 1 or len(album_data['files']) == num_tracks:
+            #album_title = os.path.basename(directory)
+            album_title = directory.rsplit('\\', 1)[1]
+            total_size = sum(file.get('size', 0) for file in album_data['files'])
+            final_results.append(Result(
+                title=album_title,
+                size=int(total_size),
+                user=album_data['user'],
+                provider="soulseek",
+                type="soulseek",
+                matches=True,
+                bandwidth=album_data['uploadSpeed'],
+                hasFreeUploadSlot=album_data['hasFreeUploadSlot'],
+                queueLength=album_data['queueLength'],
+                files=album_data['files'],
+                kind='soulseek',
+                url='http://' + album_data['user'] + album_title, # URL is needed in other parts of the program.
+                #folder=os.path.basename(directory)
+                folder = album_title
+            ))
+
+    return final_results
+
+
+def download(user, filelist):
+    client = initialize_soulseek_client()
+    client.transfers.enqueue(username=user, files=filelist)
+
+
+def download_completed():
+    client = initialize_soulseek_client()
+    all_downloads = client.transfers.get_all_downloads(includeRemoved=False)
+    album_completion_tracker = {}  # Tracks completion state of each album's songs
+    album_errored_tracker = {}  # Tracks albums with errored downloads
+
+    # Anything older than 24 hours will be canceled
+    cutoff_time = datetime.now() - timedelta(hours=24)
+
+    # Identify errored and completed albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                state = file_data.get('state', '')
+                requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00')
+                requested_at = parse_datetime(requested_at_str)
+
+                # Initialize or update album entry in trackers
+                if album_part not in album_completion_tracker:
+                    album_completion_tracker[album_part] = {'total': 0, 'completed': 0, 'errored': 0}
+                if album_part not in album_errored_tracker:
+                    album_errored_tracker[album_part] = False
+
+                album_completion_tracker[album_part]['total'] += 1
+
+                if 'Completed, Succeeded' in state:
+                    album_completion_tracker[album_part]['completed'] += 1
+                elif 'Completed, Errored' in state or requested_at < cutoff_time:
+                    album_completion_tracker[album_part]['errored'] += 1
+                    album_errored_tracker[album_part] = True  # Mark album as having errored downloads
+
+    # Identify errored albums
+    errored_albums = {album for album, errored in album_errored_tracker.items() if errored}
+
+    # Cancel downloads for errored albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                if album_part in errored_albums:
+                    # Extract 'id' and 'username' for each file to cancel the download
+                    file_id = file_data.get('id', '')
+                    username = file_data.get('username', '')
+                    success = client.transfers.cancel_download(username, file_id)
+                    if not success:
+                        logger.debug(f"Soulseek failed to cancel download for file ID: {file_id}")
+
+    # Clear completed/canceled/errored stuff from client downloads
+    try:
+        client.transfers.remove_completed_downloads()
+    except Exception as e:
+        logger.debug(f"Soulseek failed to remove completed downloads: {e}")
+
+    # Identify completed albums
+    completed_albums = {album for album, counts in album_completion_tracker.items() if counts['total'] == counts['completed']}
+
+    # Return both completed and errored albums
+    return completed_albums, errored_albums
+
+
+def download_completed_album(username, foldername):
+    client = initialize_soulseek_client()
+    downloads = client.transfers.get_downloads(username)
+
+    # Anything older than 24 hours will be canceled
+    cutoff_time = datetime.now() - timedelta(hours=24)
+
+    total_count = 0
+    completed_count = 0
+    errored_count = 0
+    file_ids = []
+
+    # Identify errored and completed album
+    directories = downloads.get('directories', [])
+    for directory in directories:
+        album_part = directory.get('directory', '').split('\\')[-1]
+        if album_part == foldername:
+            files = directory.get('files', [])
+            for file_data in files:
+                state = file_data.get('state', '')
+                requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00')
+                requested_at = parse_datetime(requested_at_str)
+
+                total_count += 1
+                file_id = file_data.get('id', '')
+                file_ids.append(file_id)
+
+                if 'Completed, Succeeded' in state:
+                    completed_count += 1
+                elif 'Completed, Errored' in state or requested_at < cutoff_time:
+                    errored_count += 1
+            break
+
+    completed = True if completed_count == total_count else False
+    errored = True if errored_count else False
+
+    # Cancel downloads for errored album
+    if errored:
+        for file_id in file_ids:
+            try:
+                success = client.transfers.cancel_download(username, file_id, remove=True)
+            except Exception as e:
+                logger.debug(f"Soulseek failed to cancel download for folder with file ID: {foldername} {file_id}")
+
+    return completed, errored
+
+
+def parse_datetime(datetime_string):
+    # Parse the datetime api response
+    if '.' in datetime_string:
+        datetime_string = datetime_string[:datetime_string.index('.')+7]
+    return datetime.strptime(datetime_string, '%Y-%m-%dT%H:%M:%S.%f')

headphones/transmission.py

@@ -15,8 +15,8 @@
 
 import time
 import json
-import base64
-import urlparse
+from base64 import b64encode
+import urllib.parse
 import os
 
 from headphones import logger, request
@@ -36,10 +36,10 @@ def addTorrent(link, data=None):
 
     if link.endswith('.torrent') and not link.startswith(('http', 'magnet')) or data:
         if data:
-            metainfo = str(base64.b64encode(data))
+            metainfo = b64encode(data).decode("utf-8")
         else:
             with open(link, 'rb') as f:
-                metainfo = str(base64.b64encode(f.read()))
+                metainfo = b64encode(f.read()).decode("utf-8")
         arguments = {'metainfo': metainfo, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
     else:
         arguments = {'filename': link, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
@@ -57,7 +57,7 @@ def addTorrent(link, data=None):
         else:
             retid = False
 
-        logger.info(u"Torrent sent to Transmission successfully")
+        logger.info("Torrent sent to Transmission successfully")
         return retid
 
     else:
@@ -167,7 +167,7 @@ def torrentAction(method, arguments):
 
     # Fix the URL. We assume that the user does not point to the RPC endpoint,
     # so add it if it is missing.
-    parts = list(urlparse.urlparse(host))
+    parts = list(urllib.parse.urlparse(host))
 
     if not parts[0] in ("http", "https"):
         parts[0] = "http"
@@ -175,7 +175,7 @@ def torrentAction(method, arguments):
     if not parts[2].endswith("/rpc"):
         parts[2] += "/transmission/rpc"
 
-    host = urlparse.urlunparse(parts)
+    host = urllib.parse.urlunparse(parts)
     data = {'method': method, 'arguments': arguments}
     data_json = json.dumps(data)
     auth = (username, password) if username and password else None
@@ -205,5 +205,4 @@ def torrentAction(method, arguments):
             continue
 
         resp_json = response.json()
-        print resp_json
         return resp_json

headphones/types.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass(frozen=True)
+class Result:
+    title: str
+    size: int
+    url: str
+    provider: str
+    kind: str
+    matches: bool

headphones/unittestcompat.py

@@ -44,7 +44,7 @@ class TestCase(TC):
 
     @_d
     def assertRegexpMatches(self, *args, **kw):
-        return super(TestCase, self).assertRegexpMatches(*args, **kw)
+        return super(TestCase, self).assertRegex(*args, **kw)
 
     # -----------------------------------------------------------
     # NOT DUMMY ASSERTIONS

headphones/utorrent.py

@@ -13,13 +13,13 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import json
 import time
 from collections import namedtuple
-import urllib2
-import urlparse
-import cookielib
+import urllib.request, urllib.error, urllib.parse
+import urllib.parse
+import http.cookiejar
 
 import re
 import os
@@ -52,23 +52,23 @@ class utorrentclient(object):
 
     def _make_opener(self, realm, base_url, username, password):
         """uTorrent API need HTTP Basic Auth and cookie support for token verify."""
-        auth = urllib2.HTTPBasicAuthHandler()
+        auth = urllib.request.HTTPBasicAuthHandler()
         auth.add_password(realm=realm, uri=base_url, user=username, passwd=password)
-        opener = urllib2.build_opener(auth)
-        urllib2.install_opener(opener)
+        opener = urllib.request.build_opener(auth)
+        urllib.request.install_opener(opener)
 
-        cookie_jar = cookielib.CookieJar()
-        cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
+        cookie_jar = http.cookiejar.CookieJar()
+        cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
 
         handlers = [auth, cookie_handler]
-        opener = urllib2.build_opener(*handlers)
+        opener = urllib.request.build_opener(*handlers)
         return opener
 
     def _get_token(self):
-        url = urlparse.urljoin(self.base_url, 'gui/token.html')
+        url = urllib.parse.urljoin(self.base_url, 'gui/token.html')
         try:
             response = self.opener.open(url)
-        except urllib2.HTTPError as err:
+        except urllib.error.HTTPError as err:
             logger.debug('URL: ' + str(url))
             logger.debug('Error getting Token. uTorrent responded with error: ' + str(err))
             return
@@ -77,7 +77,7 @@ class utorrentclient(object):
 
     def list(self, **kwargs):
         params = [('list', '1')]
-        params += kwargs.items()
+        params += list(kwargs.items())
         return self._action(params)
 
     def add_url(self, url):
@@ -150,8 +150,8 @@ class utorrentclient(object):
         if not self.token:
             return
 
-        url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urllib.urlencode(params)
-        request = urllib2.Request(url)
+        url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urllib.parse.urlencode(params)
+        request = urllib.request.Request(url)
 
         if body:
             request.add_data(body)
@@ -162,7 +162,7 @@ class utorrentclient(object):
         try:
             response = self.opener.open(request)
             return response.code, json.loads(response.read())
-        except urllib2.HTTPError as err:
+        except urllib.error.HTTPError as err:
             logger.debug('URL: ' + str(url))
             logger.debug('uTorrent webUI raised the following error: ' + str(err))
 

headphones/versioncheck.py

@@ -43,7 +43,7 @@ def runGit(args):
                                  shell=True,
                                  cwd=headphones.PROG_DIR)
             output, err = p.communicate()
-            output = output.strip()
+            output = output.decode('utf-8').strip()
 
             logger.debug('Git output: ' + output)
         except OSError as e:

headphones/webserve.py

@@ -15,34 +15,46 @@
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
 
-from operator import itemgetter
-import threading
-import hashlib
-import random
-import urllib
 import json
-import time
-import cgi
-import sys
-import urllib2
-
 import os
+import random
 import re
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers, crier
-from headphones.helpers import checked, radio, today, clean_name
-from mako.lookup import TemplateLookup
-from mako import exceptions
-import headphones
-import cherrypy
+import secrets
+import sys
+import threading
+import time
+from collections import OrderedDict
+from dataclasses import asdict
+from html import escape as html_escape
+from operator import itemgetter
+from urllib import parse
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
+import cherrypy
+from mako import exceptions
+from mako.lookup import TemplateLookup
+
+import headphones
+from headphones import (
+    crier,
+    db,
+    importer,
+    lastfm,
+    librarysync,
+    logger,
+    mb,
+    notifiers,
+    searcher,
+)
+from headphones.helpers import (
+    checked,
+    clean_name,
+    have_pct_have_total,
+    pattern_substitute,
+    radio,
+    replace_illegal_chars,
+    today,
+)
+from headphones.types import Result
 
 
 def serve_template(templatename, **kwargs):
@@ -97,7 +109,7 @@ class WebInterface(object):
         # Serve the extras up as a dict to make things easier for new templates (append new extras to the end)
         extras_list = headphones.POSSIBLE_EXTRAS
         if artist['Extras']:
-            artist_extras = map(int, artist['Extras'].split(','))
+            artist_extras = list(map(int, artist['Extras'].split(',')))
         else:
             artist_extras = []
 
@@ -158,8 +170,8 @@ class WebInterface(object):
         else:
             searchresults = mb.findSeries(name, limit=100)
         return serve_template(templatename="searchresults.html",
-                              title='Search Results for: "' + cgi.escape(name) + '"',
-                              searchresults=searchresults, name=cgi.escape(name), type=type)
+                              title='Search Results for: "' + html_escape(name) + '"',
+                              searchresults=searchresults, name=html_escape(name), type=type)
 
     @cherrypy.expose
     def addArtist(self, artistid):
@@ -230,7 +242,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def pauseArtist(self, ArtistID):
-        logger.info(u"Pausing artist: " + ArtistID)
+        logger.info("Pausing artist: " + ArtistID)
         myDB = db.DBConnection()
         controlValueDict = {'ArtistID': ArtistID}
         newValueDict = {'Status': 'Paused'}
@@ -239,7 +251,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def resumeArtist(self, ArtistID):
-        logger.info(u"Resuming artist: " + ArtistID)
+        logger.info("Resuming artist: " + ArtistID)
         myDB = db.DBConnection()
         controlValueDict = {'ArtistID': ArtistID}
         newValueDict = {'Status': 'Active'}
@@ -252,9 +264,9 @@ class WebInterface(object):
         for name in namecheck:
             artistname = name['ArtistName']
         try:
-            logger.info(u"Deleting all traces of artist: " + artistname)
+            logger.info("Deleting all traces of artist: " + artistname)
         except TypeError:
-            logger.info(u"Deleting all traces of artist: null")
+            logger.info("Deleting all traces of artist: null")
         myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
 
         from headphones import cache
@@ -291,7 +303,7 @@ class WebInterface(object):
         myDB = db.DBConnection()
         artist_name = myDB.select('SELECT DISTINCT ArtistName FROM artists WHERE ArtistID=?', [ArtistID])[0][0]
 
-        logger.info(u"Scanning artist: %s", artist_name)
+        logger.info("Scanning artist: %s", artist_name)
 
         full_folder_format = headphones.CONFIG.FOLDER_FORMAT
         folder_format = re.findall(r'(.*?[Aa]rtist?)\.*', full_folder_format)[0]
@@ -314,7 +326,7 @@ class WebInterface(object):
             sortname = artist
 
         if sortname[0].isdigit():
-            firstchar = u'0-9'
+            firstchar = '0-9'
         else:
             firstchar = sortname[0]
 
@@ -326,9 +338,9 @@ class WebInterface(object):
                   '$first': firstchar.lower(),
                   }
 
-        folder = helpers.pattern_substitute(folder_format.strip(), values, normalize=True)
+        folder = pattern_substitute(folder_format.strip(), values, normalize=True)
 
-        folder = helpers.replace_illegal_chars(folder, type="folder")
+        folder = replace_illegal_chars(folder, type="folder")
         folder = folder.replace('./', '_/').replace('/.', '/_')
 
         if folder.endswith('.'):
@@ -363,7 +375,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def deleteEmptyArtists(self):
-        logger.info(u"Deleting all empty artists")
+        logger.info("Deleting all empty artists")
         myDB = db.DBConnection()
         emptyArtistIDs = [row['ArtistID'] for row in
                           myDB.select("SELECT ArtistID FROM artists WHERE LatestAlbum IS NULL")]
@@ -415,15 +427,15 @@ class WebInterface(object):
             myDB = db.DBConnection()
             for artist in args:
                 myDB.action('DELETE FROM newartists WHERE ArtistName=?',
-                            [artist.decode(headphones.SYS_ENCODING, 'replace')])
+                            [artist])
                 myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?',
-                            [artist.decode(headphones.SYS_ENCODING, 'replace')])
+                            [artist])
                 logger.info("Artist %s removed from new artist list and set to ignored" % artist)
         raise cherrypy.HTTPRedirect("home")
 
     @cherrypy.expose
     def queueAlbum(self, AlbumID, ArtistID=None, new=False, redirect=None, lossless=False):
-        logger.info(u"Marking album: " + AlbumID + " as wanted...")
+        logger.info("Marking album: " + AlbumID + " as wanted...")
         myDB = db.DBConnection()
         controlValueDict = {'AlbumID': AlbumID}
         if lossless:
@@ -438,49 +450,36 @@ class WebInterface(object):
         raise cherrypy.HTTPRedirect(redirect)
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def choose_specific_download(self, AlbumID):
-        results = searcher.searchforalbum(AlbumID, choose_specific_download=True)
-
-        results_as_dicts = []
-
-        for result in results:
-            result_dict = {
-                'title': result[0],
-                'size': result[1],
-                'url': result[2],
-                'provider': result[3],
-                'kind': result[4],
-                'matches': result[5]
-            }
-            results_as_dicts.append(result_dict)
-        s = json.dumps(results_as_dicts)
-        cherrypy.response.headers['Content-type'] = 'application/json'
-        return s
+        results = searcher.searchforalbum(AlbumID, choose_specific_download=True) or []
+        return list(map(asdict, results))
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def download_specific_release(self, AlbumID, title, size, url, provider, kind, **kwargs):
         # Handle situations where the torrent url contains arguments that are parsed
         if kwargs:
-            url = urllib2.quote(url, safe=":?/=&") + '&' + urllib.urlencode(kwargs)
+            url = parse.quote(url, safe=":?/=&") + '&' + parse.urlencode(kwargs)
         try:
-            result = [(title, int(size), url, provider, kind)]
+            result = [Result(title, int(size), url, provider, kind, True)]
         except ValueError:
-            result = [(title, float(size), url, provider, kind)]
+            result = [Result(title, float(size), url, provider, kind, True)]
 
-        logger.info(u"Making sure we can download the chosen result")
-        (data, bestqual) = searcher.preprocess(result)
+        logger.info("Making sure we can download the chosen result")
+        data, result = searcher.preprocess(result)
 
-        if data and bestqual:
+        if data and result:
             myDB = db.DBConnection()
             album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-            searcher.send_to_downloader(data, bestqual, album)
-            return json.dumps({'result': 'success'})
+            searcher.send_to_downloader(data, result, album)
+            return {'result': 'success'}
         else:
-            return json.dumps({'result': 'failure'})
+            return {'result': 'failure'}
 
     @cherrypy.expose
     def unqueueAlbum(self, AlbumID, ArtistID):
-        logger.info(u"Marking album: " + AlbumID + "as skipped...")
+        logger.info("Marking album: " + AlbumID + "as skipped...")
         myDB = db.DBConnection()
         controlValueDict = {'AlbumID': AlbumID}
         newValueDict = {'Status': 'Skipped'}
@@ -489,7 +488,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def deleteAlbum(self, AlbumID, ArtistID=None):
-        logger.info(u"Deleting all traces of album: " + AlbumID)
+        logger.info("Deleting all traces of album: " + AlbumID)
         myDB = db.DBConnection()
 
         myDB.action('DELETE from have WHERE Matched=?', [AlbumID])
@@ -528,7 +527,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def editSearchTerm(self, AlbumID, SearchTerm):
-        logger.info(u"Updating search term for albumid: " + AlbumID)
+        logger.info("Updating search term for albumid: " + AlbumID)
         myDB = db.DBConnection()
         controlValueDict = {'AlbumID': AlbumID}
         newValueDict = {'SearchTerm': SearchTerm}
@@ -586,7 +585,7 @@ class WebInterface(object):
         for albums in have_albums:
             # Have to skip over manually matched tracks
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 # else:
                 #     original_clean = None
@@ -633,8 +632,8 @@ class WebInterface(object):
                 (artist, album))
 
         elif action == "matchArtist":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
             if new_artist_clean != existing_artist_clean:
                 have_tracks = myDB.action(
                     'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
@@ -678,10 +677,10 @@ class WebInterface(object):
                     "Artist %s already named appropriately; nothing to modify" % existing_artist)
 
         elif action == "matchAlbum":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
-            existing_album_clean = helpers.clean_name(existing_album).lower()
-            new_album_clean = helpers.clean_name(new_album).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
+            existing_album_clean = clean_name(existing_album).lower()
+            new_album_clean = clean_name(new_album).lower()
             existing_clean_string = existing_artist_clean + " " + existing_album_clean
             new_clean_string = new_artist_clean + " " + new_album_clean
             if existing_clean_string != new_clean_string:
@@ -737,7 +736,7 @@ class WebInterface(object):
             'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
         for albums in manualalbums:
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums[
                         'CleanName'] != original_clean:
@@ -778,7 +777,7 @@ class WebInterface(object):
                 [artist])
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 album = tracks['AlbumTitle']
@@ -810,7 +809,7 @@ class WebInterface(object):
                 (artist, album))
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 track_title = tracks['TrackTitle']
@@ -959,6 +958,7 @@ class WebInterface(object):
         raise cherrypy.HTTPRedirect("logs")
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getLog(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0="desc",
                sSearch="", **kwargs):
         iDisplayStart = int(iDisplayStart)
@@ -981,13 +981,14 @@ class WebInterface(object):
         rows = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)]
         rows = [[row[0], row[2], row[1]] for row in rows]
 
-        return json.dumps({
+        return {
             'iTotalDisplayRecords': len(filtered),
             'iTotalRecords': len(headphones.LOG_LIST),
             'aaData': rows,
-        })
+        }
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getArtists_json(self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSortCol_0='0',
                         sSortDir_0='asc', **kwargs):
         iDisplayStart = int(iDisplayStart)
@@ -1016,9 +1017,7 @@ class WebInterface(object):
             totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0]
 
         if sortbyhavepercent:
-            filtered.sort(key=lambda x: (
-                float(x['HaveTracks']) / x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,
-                x['HaveTracks'] if x['HaveTracks'] else 0.0), reverse=sSortDir_0 == "asc")
+            filtered.sort(key=have_pct_have_total, reverse=sSortDir_0 == "asc")
 
         # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill
         # just reverse it here and the first click on the "Latest Album" header will sort by descending release date
@@ -1055,61 +1054,58 @@ class WebInterface(object):
 
             rows.append(row)
 
-        dict = {'iTotalDisplayRecords': len(filtered),
+        data = {'iTotalDisplayRecords': len(filtered),
                 'iTotalRecords': totalcount,
                 'aaData': rows,
                 }
-        s = json.dumps(dict)
-        cherrypy.response.headers['Content-type'] = 'application/json'
-        return s
+        return data
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getAlbumsByArtist_json(self, artist=None):
         myDB = db.DBConnection()
-        album_json = {}
+        data = {}
         counter = 0
         album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist])
         for album in album_list:
-            album_json[counter] = album['AlbumTitle']
+            data[counter] = album['AlbumTitle']
             counter += 1
-        json_albums = json.dumps(album_json)
 
-        cherrypy.response.headers['Content-type'] = 'application/json'
-        return json_albums
+        return data 
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getArtistjson(self, ArtistID, **kwargs):
         myDB = db.DBConnection()
         artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [ArtistID]).fetchone()
-        artist_json = json.dumps({
+        return {
             'ArtistName': artist['ArtistName'],
             'Status': artist['Status']
-        })
-        return artist_json
+        }
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getAlbumjson(self, AlbumID, **kwargs):
         myDB = db.DBConnection()
         album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-        album_json = json.dumps({
+        return {
             'AlbumTitle': album['AlbumTitle'],
             'ArtistName': album['ArtistName'],
             'Status': album['Status']
-        })
-        return album_json
+        }
 
     @cherrypy.expose
     def clearhistory(self, type=None, date_added=None, title=None):
         myDB = db.DBConnection()
         if type:
             if type == 'all':
-                logger.info(u"Clearing all history")
+                logger.info("Clearing all history")
                 myDB.action('DELETE from snatched WHERE Status NOT LIKE "Seed%"')
             else:
-                logger.info(u"Clearing history where status is %s" % type)
+                logger.info("Clearing history where status is %s" % type)
                 myDB.action('DELETE from snatched WHERE Status=?', [type])
         else:
-            logger.info(u"Deleting '%s' from history" % title)
+            logger.info("Deleting '%s' from history" % title)
             myDB.action(
                 'DELETE from snatched WHERE Status NOT LIKE "Seed%" AND Title=? AND DateAdded=?',
                 [title, date_added])
@@ -1117,7 +1113,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     def generateAPI(self):
-        apikey = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[0:32]
+        apikey = secrets.token_hex(nbytes=16)
         logger.info("New API generated")
         return apikey
 
@@ -1187,6 +1183,7 @@ class WebInterface(object):
             "deluge_password": headphones.CONFIG.DELUGE_PASSWORD,
             "deluge_label": headphones.CONFIG.DELUGE_LABEL,
             "deluge_done_directory": headphones.CONFIG.DELUGE_DONE_DIRECTORY,
+            "deluge_download_directory": headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY,
             "deluge_paused": checked(headphones.CONFIG.DELUGE_PAUSED),
             "utorrent_host": headphones.CONFIG.UTORRENT_HOST,
             "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME,
@@ -1201,6 +1198,8 @@ class WebInterface(object):
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
             "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
+            "soulseek_download_dir": headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR,
+            "soulseek_incomplete_download_dir": headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
             "usenet_retention": headphones.CONFIG.USENET_RETENTION,
@@ -1232,13 +1231,6 @@ class WebInterface(object):
             "use_piratebay": checked(headphones.CONFIG.PIRATEBAY),
             "piratebay_proxy_url": headphones.CONFIG.PIRATEBAY_PROXY_URL,
             "piratebay_ratio": headphones.CONFIG.PIRATEBAY_RATIO,
-            "use_oldpiratebay": checked(headphones.CONFIG.OLDPIRATEBAY),
-            "oldpiratebay_url": headphones.CONFIG.OLDPIRATEBAY_URL,
-            "oldpiratebay_ratio": headphones.CONFIG.OLDPIRATEBAY_RATIO,
-            "use_waffles": checked(headphones.CONFIG.WAFFLES),
-            "waffles_uid": headphones.CONFIG.WAFFLES_UID,
-            "waffles_passkey": headphones.CONFIG.WAFFLES_PASSKEY,
-            "waffles_ratio": headphones.CONFIG.WAFFLES_RATIO,
             "use_rutracker": checked(headphones.CONFIG.RUTRACKER),
             "rutracker_user": headphones.CONFIG.RUTRACKER_USER,
             "rutracker_password": headphones.CONFIG.RUTRACKER_PASSWORD,
@@ -1250,6 +1242,7 @@ class WebInterface(object):
             "orpheus_ratio": headphones.CONFIG.ORPHEUS_RATIO,
             "orpheus_url": headphones.CONFIG.ORPHEUS_URL,
             "use_redacted": checked(headphones.CONFIG.REDACTED),
+            "redacted_apikey": headphones.CONFIG.REDACTED_APIKEY,
             "redacted_username": headphones.CONFIG.REDACTED_USERNAME,
             "redacted_password": headphones.CONFIG.REDACTED_PASSWORD,
             "redacted_ratio": headphones.CONFIG.REDACTED_RATIO,
@@ -1272,6 +1265,7 @@ class WebInterface(object):
             "cue_split_shntool_path": headphones.CONFIG.CUE_SPLIT_SHNTOOL_PATH,
             "move_files": checked(headphones.CONFIG.MOVE_FILES),
             "rename_files": checked(headphones.CONFIG.RENAME_FILES),
+            "rename_single_disc_ignore": checked(headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE),
             "correct_metadata": checked(headphones.CONFIG.CORRECT_METADATA),
             "cleanup_files": checked(headphones.CONFIG.CLEANUP_FILES),
             "keep_nfo": checked(headphones.CONFIG.KEEP_NFO),
@@ -1299,6 +1293,7 @@ class WebInterface(object):
             "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0),
             "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1),
             "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2),
+            "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3),
             "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0),
             "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1),
             "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2),
@@ -1388,6 +1383,7 @@ class WebInterface(object):
             "custompass": headphones.CONFIG.CUSTOMPASS,
             "hpuser": headphones.CONFIG.HPUSER,
             "hppass": headphones.CONFIG.HPPASS,
+            "lastfm_apikey": headphones.CONFIG.LASTFM_APIKEY,
             "songkick_enabled": checked(headphones.CONFIG.SONGKICK_ENABLED),
             "songkick_apikey": headphones.CONFIG.SONGKICK_APIKEY,
             "songkick_location": headphones.CONFIG.SONGKICK_LOCATION,
@@ -1415,10 +1411,15 @@ class WebInterface(object):
             "join_enabled": checked(headphones.CONFIG.JOIN_ENABLED),
             "join_onsnatch": checked(headphones.CONFIG.JOIN_ONSNATCH),
             "join_apikey": headphones.CONFIG.JOIN_APIKEY,
-            "join_deviceid": headphones.CONFIG.JOIN_DEVICEID
+            "join_deviceid": headphones.CONFIG.JOIN_DEVICEID,
+            "use_bandcamp": checked(headphones.CONFIG.BANDCAMP),
+            "bandcamp_dir": headphones.CONFIG.BANDCAMP_DIR,
+            'soulseek_api_url': headphones.CONFIG.SOULSEEK_API_URL,
+            'soulseek_api_key': headphones.CONFIG.SOULSEEK_API_KEY,
+            'use_soulseek': checked(headphones.CONFIG.SOULSEEK)
         }
 
-        for k, v in config.iteritems():
+        for k, v in config.items():
             if isinstance(v, headphones.config.path):
                 # need to apply SoftChroot to paths:
                 nv = headphones.SOFT_CHROOT.apply(v)
@@ -1435,7 +1436,7 @@ class WebInterface(object):
 
         extras_list = [extra_munges.get(x, x) for x in headphones.POSSIBLE_EXTRAS]
         if headphones.CONFIG.EXTRAS:
-            extras = map(int, headphones.CONFIG.EXTRAS.split(','))
+            extras = list(map(int, headphones.CONFIG.EXTRAS.split(',')))
         else:
             extras = []
 
@@ -1460,12 +1461,11 @@ class WebInterface(object):
         checked_configs = [
             "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer",
             "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled",
-            "use_nzbsorg", "use_omgwtfnzbs", "use_piratebay", "use_oldpiratebay",
-            "use_waffles", "use_rutracker",
+            "use_nzbsorg", "use_omgwtfnzbs", "use_piratebay", "use_rutracker",
             "use_orpheus", "use_redacted", "redacted_use_fltoken", "preferred_bitrate_allow_lossless",
             "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
-            "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
-            "embed_album_art", "embed_lyrics",
+            "rename_files", "rename_single_disc_ignore", "correct_metadata", "cleanup_files",
+            "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics",
             "replace_existing_folders", "keep_original_folder", "file_underscores",
             "include_extras", "official_releases_only",
             "wait_until_release_date", "autowant_upcoming", "autowant_all",
@@ -1484,7 +1484,7 @@ class WebInterface(object):
             "songkick_enabled", "songkick_filter_enabled",
             "mpc_enabled", "email_enabled", "email_ssl", "email_tls", "email_onsnatch",
             "customauth", "idtag", "deluge_paused",
-            "join_enabled", "join_onsnatch"
+            "join_enabled", "join_onsnatch", "use_bandcamp", "use_soulseek"
         ]
         for checked_config in checked_configs:
             if checked_config not in kwargs:
@@ -1496,7 +1496,7 @@ class WebInterface(object):
             kwargs[plain_config] = kwargs[use_config]
             del kwargs[use_config]
 
-        for k, v in kwargs.iteritems():
+        for k, v in kwargs.items():
             # TODO : HUGE crutch. It is all because there is no way to deal with options...
             try:
                 _conf = headphones.CONFIG._define(k)
@@ -1648,12 +1648,13 @@ class WebInterface(object):
         return a.fetchData()
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getInfo(self, ArtistID=None, AlbumID=None):
 
         from headphones import cache
         info_dict = cache.getInfo(ArtistID, AlbumID)
 
-        return json.dumps(info_dict)
+        return info_dict
 
     @cherrypy.expose
     def getArtwork(self, ArtistID=None, AlbumID=None):
@@ -1670,24 +1671,25 @@ class WebInterface(object):
     # If you just want to get the last.fm image links for an album, make sure
     # to pass a releaseid and not a releasegroupid
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     def getImageLinks(self, ArtistID=None, AlbumID=None):
         from headphones import cache
         image_dict = cache.getImageLinks(ArtistID, AlbumID)
 
         # Return the Cover Art Archive urls if not found on last.fm
         if AlbumID and not image_dict:
-            image_url = "http://coverartarchive.org/release/%s/front-500.jpg" % AlbumID
-            thumb_url = "http://coverartarchive.org/release/%s/front-250.jpg" % AlbumID
+            image_url = "https://coverartarchive.org/release/%s/front-500.jpg" % AlbumID
+            thumb_url = "https://coverartarchive.org/release/%s/front-250.jpg" % AlbumID
             image_dict = {'artwork': image_url, 'thumbnail': thumb_url}
         elif AlbumID and (not image_dict['artwork'] or not image_dict['thumbnail']):
             if not image_dict['artwork']:
                 image_dict[
-                    'artwork'] = "http://coverartarchive.org/release/%s/front-500.jpg" % AlbumID
+                    'artwork'] = "https://coverartarchive.org/release/%s/front-500.jpg" % AlbumID
             if not image_dict['thumbnail']:
                 image_dict[
-                    'thumbnail'] = "http://coverartarchive.org/release/%s/front-250.jpg" % AlbumID
+                    'thumbnail'] = "https://coverartarchive.org/release/%s/front-250.jpg" % AlbumID
 
-        return json.dumps(image_dict)
+        return image_dict
 
     @cherrypy.expose
     def twitterStep1(self):
@@ -1700,7 +1702,7 @@ class WebInterface(object):
         cherrypy.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store"
         tweet = notifiers.TwitterNotifier()
         result = tweet._get_credentials(key)
-        logger.info(u"result: " + str(result))
+        logger.info("result: " + str(result))
         if result:
             return "Key verification successful"
         else:
@@ -1732,14 +1734,14 @@ class WebInterface(object):
 
     @cherrypy.expose
     def testPushover(self):
-        logger.info(u"Sending Pushover notification")
+        logger.info("Sending Pushover notification")
         pushover = notifiers.PUSHOVER()
         result = pushover.notify("hooray!", "This is a test")
         return str(result)
 
     @cherrypy.expose
     def testPlex(self):
-        logger.info(u"Testing plex update")
+        logger.info("Testing plex update")
         plex = notifiers.Plex()
         plex.update()
 

headphones/webstart.py

@@ -111,12 +111,9 @@ def initialize(options):
         })
         conf['/api'] = {'tools.auth_basic.on': False}
 
-    # Prevent time-outs
-    cherrypy.engine.timeout_monitor.unsubscribe()
     cherrypy.tree.mount(WebInterface(), str(options['http_root']), config=conf)
 
     try:
-        cherrypy.process.servers.check_port(str(options['http_host']), options['http_port'])
         cherrypy.server.start()
     except IOError:
         sys.stderr.write(

lib/MultipartPostHandler.py

@@ -15,8 +15,8 @@
 # Lesser General Public License for more details.
 #
 
-import urllib
-import urllib2
+import urllib.request, urllib.parse, urllib.error
+import urllib.request, urllib.error, urllib.parse
 import mimetools, mimetypes
 import os, sys
 
@@ -24,8 +24,8 @@ import os, sys
 #  assigning a sequence.
 doseq = 1
 
-class MultipartPostHandler(urllib2.BaseHandler):
-    handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first
+class MultipartPostHandler(urllib.request.BaseHandler):
+    handler_order = urllib.request.HTTPHandler.handler_order - 10 # needs to run first
 
     def http_request(self, request):
         data = request.get_data()
@@ -33,23 +33,23 @@ class MultipartPostHandler(urllib2.BaseHandler):
             v_files = []
             v_vars = []
             try:
-                for(key, value) in data.items():
+                for(key, value) in list(data.items()):
                     if type(value) in (file, list, tuple):
                         v_files.append((key, value))
                     else:
                         v_vars.append((key, value))
             except TypeError:
                 systype, value, traceback = sys.exc_info()
-                raise TypeError, "not a valid non-string sequence or mapping object", traceback
+                raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback)
 
             if len(v_files) == 0:
-                data = urllib.urlencode(v_vars, doseq)
+                data = urllib.parse.urlencode(v_vars, doseq)
             else:
                 boundary, data = MultipartPostHandler.multipart_encode(v_vars, v_files)
                 contenttype = 'multipart/form-data; boundary=%s' % boundary
                 if(request.has_header('Content-Type')
                    and request.get_header('Content-Type').find('multipart/form-data') != 0):
-                    print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')
+                    print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
                 request.add_unredirected_header('Content-Type', contenttype)
 
             request.add_data(data)

lib/apscheduler/__init__.py

@@ -1,5 +1,10 @@
-version_info = (3, 0, 1)
-version = '3.0.1'
-release = '3.0.1'
+from pkg_resources import get_distribution, DistributionNotFound
 
-__version__ = release  # PEP 396
+try:
+    release = get_distribution('APScheduler').version.split('-')[0]
+except DistributionNotFound:
+    release = '3.5.0'
+
+version_info = tuple(int(x) if x.isdigit() else x for x in release.split('.'))
+version = __version__ = '.'.join(str(x) for x in version_info[:3])
+del get_distribution, DistributionNotFound

lib/apscheduler/events.py

@@ -1,25 +1,33 @@
-__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
-           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED', 'EVENT_JOB_ADDED',
-           'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+__all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDULER_PAUSED',
+           'EVENT_SCHEDULER_RESUMED', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
+           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
+           'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
+           'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
-EVENT_SCHEDULER_START = 1
-EVENT_SCHEDULER_SHUTDOWN = 2
-EVENT_EXECUTOR_ADDED = 4
-EVENT_EXECUTOR_REMOVED = 8
-EVENT_JOBSTORE_ADDED = 16
-EVENT_JOBSTORE_REMOVED = 32
-EVENT_ALL_JOBS_REMOVED = 64
-EVENT_JOB_ADDED = 128
-EVENT_JOB_REMOVED = 256
-EVENT_JOB_MODIFIED = 512
-EVENT_JOB_EXECUTED = 1024
-EVENT_JOB_ERROR = 2048
-EVENT_JOB_MISSED = 4096
-EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED |
+EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0
+EVENT_SCHEDULER_SHUTDOWN = 2 ** 1
+EVENT_SCHEDULER_PAUSED = 2 ** 2
+EVENT_SCHEDULER_RESUMED = 2 ** 3
+EVENT_EXECUTOR_ADDED = 2 ** 4
+EVENT_EXECUTOR_REMOVED = 2 ** 5
+EVENT_JOBSTORE_ADDED = 2 ** 6
+EVENT_JOBSTORE_REMOVED = 2 ** 7
+EVENT_ALL_JOBS_REMOVED = 2 ** 8
+EVENT_JOB_ADDED = 2 ** 9
+EVENT_JOB_REMOVED = 2 ** 10
+EVENT_JOB_MODIFIED = 2 ** 11
+EVENT_JOB_EXECUTED = 2 ** 12
+EVENT_JOB_ERROR = 2 ** 13
+EVENT_JOB_MISSED = 2 ** 14
+EVENT_JOB_SUBMITTED = 2 ** 15
+EVENT_JOB_MAX_INSTANCES = 2 ** 16
+EVENT_ALL = (EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED |
+             EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED |
+             EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | EVENT_ALL_JOBS_REMOVED |
              EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED | EVENT_JOB_EXECUTED |
-             EVENT_JOB_ERROR | EVENT_JOB_MISSED)
+             EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES)
 
 
 class SchedulerEvent(object):
@@ -55,9 +63,21 @@ class JobEvent(SchedulerEvent):
         self.jobstore = jobstore
 
 
+class JobSubmissionEvent(JobEvent):
+    """
+    An event that concerns the submission of a job to its executor.
+
+    :ivar scheduled_run_times: a list of datetimes when the job was intended to run
+    """
+
+    def __init__(self, code, job_id, jobstore, scheduled_run_times):
+        super(JobSubmissionEvent, self).__init__(code, job_id, jobstore)
+        self.scheduled_run_times = scheduled_run_times
+
+
 class JobExecutionEvent(JobEvent):
     """
-    An event that concerns the execution of individual jobs.
+    An event that concerns the running of a job within its executor.
 
     :ivar scheduled_run_time: the time when the job was scheduled to be run
     :ivar retval: the return value of the successfully executed job
@@ -65,7 +85,8 @@ class JobExecutionEvent(JobEvent):
     :ivar traceback: a formatted traceback for the exception
     """
 
-    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None, traceback=None):
+    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None,
+                 traceback=None):
         super(JobExecutionEvent, self).__init__(code, job_id, jobstore)
         self.scheduled_run_time = scheduled_run_time
         self.retval = retval

lib/apscheduler/executors/asyncio.py

@@ -1,28 +1,52 @@
 from __future__ import absolute_import
+
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.executors.base_py3 import run_coroutine_job
+from apscheduler.util import iscoroutinefunction_partial
 
 
 class AsyncIOExecutor(BaseExecutor):
     """
     Runs jobs in the default executor of the event loop.
 
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    event loop as soon as possible. All other functions are run in the event loop's default
+    executor which is usually a thread pool.
+
     Plugin alias: ``asyncio``
     """
 
     def start(self, scheduler, alias):
         super(AsyncIOExecutor, self).start(scheduler, alias)
         self._eventloop = scheduler._eventloop
+        self._pending_futures = set()
+
+    def shutdown(self, wait=True):
+        # There is no way to honor wait=True without converting this method into a coroutine method
+        for f in self._pending_futures:
+            if not f.done():
+                f.cancel()
+
+        self._pending_futures.clear()
 
     def _do_submit_job(self, job, run_times):
         def callback(f):
+            self._pending_futures.discard(f)
             try:
                 events = f.result()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        if iscoroutinefunction_partial(job.func):
+            coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+            f = self._eventloop.create_task(coro)
+        else:
+            f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times,
+                                                self._logger.name)
+
         f.add_done_callback(callback)
+        self._pending_futures.add(f)

lib/apscheduler/executors/base.py

@@ -8,13 +8,15 @@ import sys
 from pytz import utc
 import six
 
-from apscheduler.events import JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
 
 
 class MaxInstancesReachedError(Exception):
     def __init__(self, job):
         super(MaxInstancesReachedError, self).__init__(
-            'Job "%s" has already reached its maximum number of instances (%d)' % (job.id, job.max_instances))
+            'Job "%s" has already reached its maximum number of instances (%d)' %
+            (job.id, job.max_instances))
 
 
 class BaseExecutor(six.with_metaclass(ABCMeta, object)):
@@ -30,13 +32,14 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the executor is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the executor is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this executor
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this executor
         :param str|unicode alias: alias of this executor as it was assigned to the scheduler
-        """
 
+        """
         self._scheduler = scheduler
         self._lock = scheduler._create_lock()
         self._logger = logging.getLogger('apscheduler.executors.%s' % alias)
@@ -45,7 +48,8 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """
         Shuts down this executor.
 
-        :param bool wait: ``True`` to wait until all submitted jobs have been executed
+        :param bool wait: ``True`` to wait until all submitted jobs
+            have been executed
         """
 
     def submit_job(self, job, run_times):
@@ -53,10 +57,12 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         Submits job for execution.
 
         :param Job job: job to execute
-        :param list[datetime] run_times: list of datetimes specifying when the job should have been run
-        :raises MaxInstancesReachedError: if the maximum number of allowed instances for this job has been reached
-        """
+        :param list[datetime] run_times: list of datetimes specifying
+            when the job should have been run
+        :raises MaxInstancesReachedError: if the maximum number of
+            allowed instances for this job has been reached
 
+        """
         assert self._lock is not None, 'This executor has not been started yet'
         with self._lock:
             if self._instances[job.id] >= job.max_instances:
@@ -70,50 +76,71 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """Performs the actual task of scheduling `run_job` to be called."""
 
     def _run_job_success(self, job_id, events):
-        """Called by the executor with the list of generated events when `run_job` has been successfully called."""
+        """
+        Called by the executor with the list of generated events when :func:`run_job` has been
+        successfully called.
 
+        """
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         for event in events:
             self._scheduler._dispatch_event(event)
 
     def _run_job_error(self, job_id, exc, traceback=None):
-        """Called by the executor with the exception if there is an error calling `run_job`."""
-
+        """Called by the executor with the exception if there is an error  calling `run_job`."""
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         exc_info = (exc.__class__, exc, traceback)
         self._logger.error('Error running job %s', job_id, exc_info=exc_info)
 
 
 def run_job(job, jobstore_alias, run_times, logger_name):
-    """Called by executors to run the job. Returns a list of scheduler events to be dispatched by the scheduler."""
+    """
+    Called by executors to run the job. Returns a list of scheduler events to be dispatched by the
+    scheduler.
 
+    """
     events = []
     logger = logging.getLogger(logger_name)
     for run_time in run_times:
-        # See if the job missed its run time window, and handle possible misfires accordingly
+        # See if the job missed its run time window, and handle
+        # possible misfires accordingly
         if job.misfire_grace_time is not None:
             difference = datetime.now(utc) - run_time
             grace_time = timedelta(seconds=job.misfire_grace_time)
             if difference > grace_time:
-                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias, run_time))
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
                 logger.warning('Run time of job "%s" was missed by %s', job, difference)
                 continue
 
         logger.info('Running job "%s" (scheduled at %s)', job, run_time)
         try:
             retval = job.func(*job.args, **job.kwargs)
-        except:
+        except BaseException:
             exc, tb = sys.exc_info()[1:]
             formatted_tb = ''.join(format_tb(tb))
-            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time, exception=exc,
-                                            traceback=formatted_tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
             logger.exception('Job "%s" raised an exception', job)
+
+            # This is to prevent cyclic references that would lead to memory leaks
+            if six.PY2:
+                sys.exc_clear()
+                del tb
+            else:
+                import traceback
+                traceback.clear_frames(tb)
+                del tb
         else:
-            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time, retval=retval))
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
             logger.info('Job "%s" executed successfully', job)
 
     return events

lib/apscheduler/executors/base_py3.py

@@ -0,0 +1,43 @@
+import logging
+import sys
+import traceback
+from datetime import datetime, timedelta
+from traceback import format_tb
+
+from pytz import utc
+
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
+
+
+async def run_coroutine_job(job, jobstore_alias, run_times, logger_name):
+    """Coroutine version of run_job()."""
+    events = []
+    logger = logging.getLogger(logger_name)
+    for run_time in run_times:
+        # See if the job missed its run time window, and handle possible misfires accordingly
+        if job.misfire_grace_time is not None:
+            difference = datetime.now(utc) - run_time
+            grace_time = timedelta(seconds=job.misfire_grace_time)
+            if difference > grace_time:
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
+                logger.warning('Run time of job "%s" was missed by %s', job, difference)
+                continue
+
+        logger.info('Running job "%s" (scheduled at %s)', job, run_time)
+        try:
+            retval = await job.func(*job.args, **job.kwargs)
+        except BaseException:
+            exc, tb = sys.exc_info()[1:]
+            formatted_tb = ''.join(format_tb(tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
+            logger.exception('Job "%s" raised an exception', job)
+            traceback.clear_frames(tb)
+        else:
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
+            logger.info('Job "%s" executed successfully', job)
+
+    return events

lib/apscheduler/executors/debug.py

@@ -5,7 +5,8 @@ from apscheduler.executors.base import BaseExecutor, run_job
 
 class DebugExecutor(BaseExecutor):
     """
-    A special executor that executes the target callable directly instead of deferring it to a thread or process.
+    A special executor that executes the target callable directly instead of deferring it to a
+    thread or process.
 
     Plugin alias: ``debug``
     """
@@ -13,7 +14,7 @@ class DebugExecutor(BaseExecutor):
     def _do_submit_job(self, job, run_times):
         try:
             events = run_job(job, job._jobstore_alias, run_times, self._logger.name)
-        except:
+        except BaseException:
             self._run_job_error(job.id, *sys.exc_info()[1:])
         else:
             self._run_job_success(job.id, events)

lib/apscheduler/executors/gevent.py

@@ -21,9 +21,10 @@ class GeventExecutor(BaseExecutor):
         def callback(greenlet):
             try:
                 events = greenlet.get()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).link(callback)
+        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).\
+            link(callback)

lib/apscheduler/executors/pool.py

@@ -3,6 +3,11 @@ import concurrent.futures
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
+try:
+    from concurrent.futures.process import BrokenProcessPool
+except ImportError:
+    BrokenProcessPool = None
+
 
 class BasePoolExecutor(BaseExecutor):
     @abstractmethod
@@ -19,7 +24,13 @@ class BasePoolExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, f.result())
 
-        f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        try:
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        except BrokenProcessPool:
+            self._logger.warning('Process pool is broken; replacing pool with a fresh instance')
+            self._pool = self._pool.__class__(self._pool._max_workers)
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+
         f.add_done_callback(callback)
 
     def shutdown(self, wait=True):
@@ -33,10 +44,13 @@ class ThreadPoolExecutor(BasePoolExecutor):
     Plugin alias: ``threadpool``
 
     :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
-        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers))
+    def __init__(self, max_workers=10, pool_kwargs=None):
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers), **pool_kwargs)
         super(ThreadPoolExecutor, self).__init__(pool)
 
 
@@ -47,8 +61,11 @@ class ProcessPoolExecutor(BasePoolExecutor):
     Plugin alias: ``processpool``
 
     :param max_workers: the maximum number of spawned processes.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ProcessPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
-        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers))
+    def __init__(self, max_workers=10, pool_kwargs=None):
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers), **pool_kwargs)
         super(ProcessPoolExecutor, self).__init__(pool)

lib/apscheduler/executors/tornado.py

@@ -0,0 +1,54 @@
+from __future__ import absolute_import
+
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+from tornado.gen import convert_yielded
+
+from apscheduler.executors.base import BaseExecutor, run_job
+
+try:
+    from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
+except ImportError:
+    def iscoroutinefunction_partial(func):
+        return False
+
+
+class TornadoExecutor(BaseExecutor):
+    """
+    Runs jobs either in a thread pool or directly on the I/O loop.
+
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    I/O loop as soon as possible. All other functions are run in a thread pool.
+
+    Plugin alias: ``tornado``
+
+    :param int max_workers: maximum number of worker threads in the thread pool
+    """
+
+    def __init__(self, max_workers=10):
+        super(TornadoExecutor, self).__init__()
+        self.executor = ThreadPoolExecutor(max_workers)
+
+    def start(self, scheduler, alias):
+        super(TornadoExecutor, self).start(scheduler, alias)
+        self._ioloop = scheduler._ioloop
+
+    def _do_submit_job(self, job, run_times):
+        def callback(f):
+            try:
+                events = f.result()
+            except BaseException:
+                self._run_job_error(job.id, *sys.exc_info()[1:])
+            else:
+                self._run_job_success(job.id, events)
+
+        if iscoroutinefunction_partial(job.func):
+            f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+        else:
+            f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,
+                                     self._logger.name)
+
+        f = convert_yielded(f)
+        f.add_done_callback(callback)

lib/apscheduler/executors/twisted.py

@@ -21,5 +21,5 @@ class TwistedExecutor(BaseExecutor):
             else:
                 self._run_job_error(job.id, result.value, result.tb)
 
-        self._reactor.getThreadPool().callInThreadWithCallback(callback, run_job, job, job._jobstore_alias, run_times,
-                                                               self._logger.name)
+        self._reactor.getThreadPool().callInThreadWithCallback(
+            callback, run_job, job, job._jobstore_alias, run_times, self._logger.name)

lib/apscheduler/job.py

@@ -1,11 +1,17 @@
-from collections import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args, \
-    convert_to_datetime
+from apscheduler.util import (
+    ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
+    convert_to_datetime)
+
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
 
 
 class Job(object):
@@ -21,13 +27,20 @@ class Job(object):
     :var bool coalesce: whether to only run the job once when several run times are due
     :var trigger: the trigger object that controls the schedule of this job
     :var str executor: the name of the executor that will run this job
-    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to be late
-    :var int max_instances: the maximum number of concurrently executing instances allowed for this job
+    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to
+        be late (``None`` means "allow the job to run no matter how late it is")
+    :var int max_instances: the maximum number of concurrently executing instances allowed for this
+        job
     :var datetime.datetime next_run_time: the next scheduled run time of this job
+
+    .. note::
+        The ``misfire_grace_time`` has some non-obvious effects on job execution. See the
+        :ref:`missed-job-executions` section in the documentation for an in-depth explanation.
     """
 
-    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref', 'args', 'kwargs',
-                 'name', 'misfire_grace_time', 'coalesce', 'max_instances', 'next_run_time')
+    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref',
+                 'args', 'kwargs', 'name', 'misfire_grace_time', 'coalesce', 'max_instances',
+                 'next_run_time', '__weakref__')
 
     def __init__(self, scheduler, id=None, **kwargs):
         super(Job, self).__init__()
@@ -38,53 +51,69 @@ class Job(object):
     def modify(self, **changes):
         """
         Makes the given changes to this job and saves it in the associated job store.
+
         Accepted keyword arguments are the same as the variables on this class.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.modify_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.modify_job(self.id, self._jobstore_alias, **changes)
+        return self
 
     def reschedule(self, trigger, **trigger_args):
         """
         Shortcut for switching the trigger on this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.reschedule_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.reschedule_job(self.id, self._jobstore_alias, trigger, **trigger_args)
+        return self
 
     def pause(self):
         """
         Temporarily suspend the execution of this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.pause_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.pause_job(self.id, self._jobstore_alias)
+        return self
 
     def resume(self):
         """
         Resume the schedule of this job if previously paused.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.resume_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.resume_job(self.id, self._jobstore_alias)
+        return self
 
     def remove(self):
         """
         Unschedules this job and removes it from its associated job store.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.remove_job`
-        """
 
+        """
         self._scheduler.remove_job(self.id, self._jobstore_alias)
 
     @property
     def pending(self):
-        """Returns ``True`` if the referenced job is still waiting to be added to its designated job store."""
+        """
+        Returns ``True`` if the referenced job is still waiting to be added to its designated job
+        store.
 
+        """
         return self._jobstore_alias is None
 
     #
@@ -97,8 +126,8 @@ class Job(object):
 
         :type now: datetime.datetime
         :rtype: list[datetime.datetime]
-        """
 
+        """
         run_times = []
         next_run_time = self.next_run_time
         while next_run_time and next_run_time <= now:
@@ -108,8 +137,11 @@ class Job(object):
         return run_times
 
     def _modify(self, **changes):
-        """Validates the changes to the Job and makes the modifications if and only if all of them validate."""
+        """
+        Validates the changes to the Job and makes the modifications if and only if all of them
+        validate.
 
+        """
         approved = {}
 
         if 'id' in changes:
@@ -125,7 +157,7 @@ class Job(object):
             args = changes.pop('args') if 'args' in changes else self.args
             kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs
 
-            if isinstance(func, str):
+            if isinstance(func, six.string_types):
                 func_ref = func
                 func = ref_to_obj(func)
             elif callable(func):
@@ -177,7 +209,8 @@ class Job(object):
         if 'trigger' in changes:
             trigger = changes.pop('trigger')
             if not isinstance(trigger, BaseTrigger):
-                raise TypeError('Expected a trigger instance, got %s instead' % trigger.__class__.__name__)
+                raise TypeError('Expected a trigger instance, got %s instead' %
+                                trigger.__class__.__name__)
 
             approved['trigger'] = trigger
 
@@ -189,10 +222,12 @@ class Job(object):
 
         if 'next_run_time' in changes:
             value = changes.pop('next_run_time')
-            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone, 'next_run_time')
+            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone,
+                                                            'next_run_time')
 
         if changes:
-            raise AttributeError('The following are not modifiable attributes of Job: %s' % ', '.join(changes))
+            raise AttributeError('The following are not modifiable attributes of Job: %s' %
+                                 ', '.join(changes))
 
         for key, value in six.iteritems(approved):
             setattr(self, key, value)
@@ -200,9 +235,18 @@ class Job(object):
     def __getstate__(self):
         # Don't allow this Job to be serialized if the function reference could not be determined
         if not self.func_ref:
-            raise ValueError('This Job cannot be serialized since the reference to its callable (%r) could not be '
-                             'determined. Consider giving a textual reference (module:function name) instead.' %
-                             (self.func,))
+            raise ValueError(
+                'This Job cannot be serialized since the reference to its callable (%r) could not '
+                'be determined. Consider giving a textual reference (module:function name) '
+                'instead.' % (self.func,))
+
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        func = self.func
+        if ismethod(func) and not isclass(func.__self__) and obj_to_ref(func) == self.func_ref:
+            args = (func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
 
         return {
             'version': 1,
@@ -210,7 +254,7 @@ class Job(object):
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,
@@ -221,7 +265,8 @@ class Job(object):
 
     def __setstate__(self, state):
         if state.get('version', 1) > 1:
-            raise ValueError('Job has version %s, but only version 1 can be handled' % state['version'])
+            raise ValueError('Job has version %s, but only version 1 can be handled' %
+                             state['version'])
 
         self.id = state['id']
         self.func_ref = state['func']
@@ -245,8 +290,13 @@ class Job(object):
         return '<Job (id=%s name=%s)>' % (repr_escape(self.id), repr_escape(self.name))
 
     def __str__(self):
-        return '%s (trigger: %s, next run at: %s)' % (repr_escape(self.name), repr_escape(str(self.trigger)),
-                                                      datetime_repr(self.next_run_time))
+        return repr_escape(self.__unicode__())
 
     def __unicode__(self):
-        return six.u('%s (trigger: %s, next run at: %s)') % (self.name, self.trigger, datetime_repr(self.next_run_time))
+        if hasattr(self, 'next_run_time'):
+            status = ('next run at: ' + datetime_repr(self.next_run_time) if
+                      self.next_run_time else 'paused')
+        else:
+            status = 'pending'
+
+        return u'%s (trigger: %s, %s)' % (self.name, self.trigger, status)

lib/apscheduler/jobstores/base.py

@@ -8,23 +8,27 @@ class JobLookupError(KeyError):
     """Raised when the job store cannot find a job for update or removal."""
 
     def __init__(self, job_id):
-        super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id)
+        super(JobLookupError, self).__init__(u'No job by the id of %s was found' % job_id)
 
 
 class ConflictingIdError(KeyError):
     """Raised when the uniqueness of job IDs is being violated."""
 
     def __init__(self, job_id):
-        super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id)
+        super(ConflictingIdError, self).__init__(
+            u'Job identifier (%s) conflicts with an existing job' % job_id)
 
 
 class TransientJobError(ValueError):
-    """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected."""
+    """
+    Raised when an attempt to add transient (with no func_ref) job to a persistent job store is
+    detected.
+    """
 
     def __init__(self, job_id):
         super(TransientJobError, self).__init__(
-            six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be '
-                  'determined.') % job_id)
+            u'Job (%s) cannot be added to this job store because a reference to the callable '
+            u'could not be determined.' % job_id)
 
 
 class BaseJobStore(six.with_metaclass(ABCMeta)):
@@ -36,10 +40,11 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the job store is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the job store is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this job store
         :param str|unicode alias: alias of this job store as it was assigned to the scheduler
         """
 
@@ -50,13 +55,22 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     def shutdown(self):
         """Frees any resources still bound to this job store."""
 
+    def _fix_paused_jobs_sorting(self, jobs):
+        for i, job in enumerate(jobs):
+            if job.next_run_time is not None:
+                if i > 0:
+                    paused_jobs = jobs[:i]
+                    del jobs[:i]
+                    jobs.extend(paused_jobs)
+                break
+
     @abstractmethod
     def lookup_job(self, job_id):
         """
         Returns a specific job, or ``None`` if it isn't found..
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned job to point to the scheduler and itself, respectively.
 
         :param str|unicode job_id: identifier of the job
         :rtype: Job
@@ -75,7 +89,8 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_next_run_time(self):
         """
-        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs.
+        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if
+        there are no active jobs.
 
         :rtype: datetime.datetime
         """
@@ -83,11 +98,12 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_all_jobs(self):
         """
-        Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending).
-        Paused jobs (next_run_time is None) should be sorted last.
+        Returns a list of all jobs in this job store.
+        The returned jobs should be sorted by next run time (ascending).
+        Paused jobs (next_run_time == None) should be sorted last.
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned jobs to point to the scheduler and itself, respectively.
 
         :rtype: list[Job]
         """

lib/apscheduler/jobstores/memory.py

@@ -13,7 +13,8 @@ class MemoryJobStore(BaseJobStore):
 
     def __init__(self):
         super(MemoryJobStore, self).__init__()
-        self._jobs = []  # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        self._jobs = []
         self._jobs_index = {}  # id -> (job, timestamp) lookup table
 
     def lookup_job(self, job_id):
@@ -80,13 +81,13 @@ class MemoryJobStore(BaseJobStore):
 
     def _get_job_index(self, timestamp, job_id):
         """
-        Returns the index of the given job, or if it's not found, the index where the job should be inserted based on
-        the given timestamp.
+        Returns the index of the given job, or if it's not found, the index where the job should be
+        inserted based on the given timestamp.
 
         :type timestamp: int
         :type job_id: str
-        """
 
+        """
         lo, hi = 0, len(self._jobs)
         timestamp = float('inf') if timestamp is None else timestamp
         while lo < hi:

lib/apscheduler/jobstores/mongodb.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import warnings
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
@@ -19,16 +20,18 @@ except ImportError:  # pragma: nocover
 
 class MongoDBJobStore(BaseJobStore):
     """
-    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient
+    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to
+    pymongo's `MongoClient
     <http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_.
 
     Plugin alias: ``mongodb``
 
     :param str database: database to store jobs in
     :param str collection: collection to store jobs in
-    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection
-                   arguments
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, database='apscheduler', collection='jobs', client=None,
@@ -42,13 +45,22 @@ class MongoDBJobStore(BaseJobStore):
             raise ValueError('The "collection" parameter must not be empty')
 
         if client:
-            self.connection = maybe_ref(client)
+            self.client = maybe_ref(client)
         else:
             connect_args.setdefault('w', 1)
-            self.connection = MongoClient(**connect_args)
+            self.client = MongoClient(**connect_args)
 
-        self.collection = self.connection[database][collection]
-        self.collection.ensure_index('next_run_time', sparse=True)
+        self.collection = self.client[database][collection]
+
+    def start(self, scheduler, alias):
+        super(MongoDBJobStore, self).start(scheduler, alias)
+        self.collection.create_index('next_run_time', sparse=True)
+
+    @property
+    def connection(self):
+        warnings.warn('The "connection" member is deprecated -- use "client" instead',
+                      DeprecationWarning)
+        return self.client
 
     def lookup_job(self, job_id):
         document = self.collection.find_one(job_id, ['job_state'])
@@ -59,16 +71,19 @@ class MongoDBJobStore(BaseJobStore):
         return self._get_jobs({'next_run_time': {'$lte': timestamp}})
 
     def get_next_run_time(self):
-        document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'],
+        document = self.collection.find_one({'next_run_time': {'$ne': None}},
+                                            projection=['next_run_time'],
                                             sort=[('next_run_time', ASCENDING)])
         return utc_timestamp_to_datetime(document['next_run_time']) if document else None
 
     def get_all_jobs(self):
-        return self._get_jobs({})
+        jobs = self._get_jobs({})
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         try:
-            self.collection.insert({
+            self.collection.insert_one({
                 '_id': job.id,
                 'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
                 'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
@@ -81,20 +96,20 @@ class MongoDBJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
-        result = self.collection.update({'_id': job.id}, {'$set': changes})
-        if result and result['n'] == 0:
-            raise JobLookupError(id)
+        result = self.collection.update_one({'_id': job.id}, {'$set': changes})
+        if result and result.matched_count == 0:
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
-        result = self.collection.remove(job_id)
-        if result and result['n'] == 0:
+        result = self.collection.delete_one({'_id': job_id})
+        if result and result.deleted_count == 0:
             raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
-        self.collection.remove()
+        self.collection.delete_many({})
 
     def shutdown(self):
-        self.connection.disconnect()
+        self.client.close()
 
     def _reconstitute_job(self, job_state):
         job_state = pickle.loads(job_state)
@@ -107,18 +122,20 @@ class MongoDBJobStore(BaseJobStore):
     def _get_jobs(self, conditions):
         jobs = []
         failed_job_ids = []
-        for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]):
+        for document in self.collection.find(conditions, ['_id', 'job_state'],
+                                             sort=[('next_run_time', ASCENDING)]):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
-                self._logger.exception('Unable to restore job "%s" -- removing it', document['_id'])
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it',
+                                       document['_id'])
                 failed_job_ids.append(document['_id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            self.collection.remove({'_id': {'$in': failed_job_ids}})
+            self.collection.delete_many({'_id': {'$in': failed_job_ids}})
 
         return jobs
 
     def __repr__(self):
-        return '<%s (client=%s)>' % (self.__class__.__name__, self.connection)
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/jobstores/redis.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+from datetime import datetime
 
+from pytz import utc
 import six
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
@@ -12,21 +14,23 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
 
 class RedisJobStore(BaseJobStore):
     """
-    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis.
+    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's
+    :class:`~redis.StrictRedis`.
 
     Plugin alias: ``redis``
 
     :param int db: the database number to store jobs in
     :param str jobs_key: key to store jobs in
     :param str run_times_key: key to store the jobs' run times in
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',
@@ -43,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -65,7 +69,8 @@ class RedisJobStore(BaseJobStore):
     def get_all_jobs(self):
         job_states = self.redis.hgetall(self.jobs_key)
         jobs = self._reconstitute_jobs(six.iteritems(job_states))
-        return sorted(jobs, key=lambda job: job.next_run_time)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job: job.next_run_time or paused_sort_key)
 
     def add_job(self, job):
         if self.redis.hexists(self.jobs_key, job.id):
@@ -73,8 +78,12 @@ class RedisJobStore(BaseJobStore):
 
         with self.redis.pipeline() as pipe:
             pipe.multi()
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
-            pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
+            if job.next_run_time:
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -82,11 +91,14 @@ class RedisJobStore(BaseJobStore):
             raise JobLookupError(job.id)
 
         with self.redis.pipeline() as pipe:
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):
@@ -121,7 +133,7 @@ class RedisJobStore(BaseJobStore):
         for job_id, job_state in job_states:
             try:
                 jobs.append(self._reconstitute_job(job_state))
-            except:
+            except BaseException:
                 self._logger.exception('Unable to restore job "%s" -- removing it', job_id)
                 failed_job_ids.append(job_id)
 

lib/apscheduler/jobstores/rethinkdb.py

@@ -0,0 +1,155 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from rethinkdb import RethinkDB
+except ImportError:  # pragma: nocover
+    raise ImportError('RethinkDBJobStore requires rethinkdb installed')
+
+
+class RethinkDBJobStore(BaseJobStore):
+    """
+    Stores jobs in a RethinkDB database. Any leftover keyword arguments are directly passed to
+    rethinkdb's `RethinkdbClient <http://www.rethinkdb.com/api/#connect>`_.
+
+    Plugin alias: ``rethinkdb``
+
+    :param str database: database to store jobs in
+    :param str collection: collection to store jobs in
+    :param client: a :class:`rethinkdb.net.Connection` instance to use instead of providing
+        connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, database='apscheduler', table='jobs', client=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(RethinkDBJobStore, self).__init__()
+
+        if not database:
+            raise ValueError('The "database" parameter must not be empty')
+        if not table:
+            raise ValueError('The "table" parameter must not be empty')
+
+        self.database = database
+        self.table_name = table
+        self.table = None
+        self.client = client
+        self.pickle_protocol = pickle_protocol
+        self.connect_args = connect_args
+        self.r = RethinkDB()
+        self.conn = None
+
+    def start(self, scheduler, alias):
+        super(RethinkDBJobStore, self).start(scheduler, alias)
+
+        if self.client:
+            self.conn = maybe_ref(self.client)
+        else:
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
+
+        if self.database not in self.r.db_list().run(self.conn):
+            self.r.db_create(self.database).run(self.conn)
+
+        if self.table_name not in self.r.table_list().run(self.conn):
+            self.r.table_create(self.table_name).run(self.conn)
+
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
+
+        self.table = self.r.db(self.database).table(self.table_name)
+
+    def lookup_job(self, job_id):
+        results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
+        return self._reconstitute_job(results[0]['job_state']) if results else None
+
+    def get_due_jobs(self, now):
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+
+    def get_next_run_time(self):
+        results = list(
+            self.table
+            .filter(self.r.row['next_run_time'] != None)  # noqa
+            .order_by(self.r.asc('next_run_time'))
+            .map(lambda x: x['next_run_time'])
+            .limit(1)
+            .run(self.conn)
+        )
+        return utc_timestamp_to_datetime(results[0]) if results else None
+
+    def get_all_jobs(self):
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        job_dict = {
+            'id': job.id,
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.insert(job_dict).run(self.conn)
+        if results['errors'] > 0:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.get_all(job.id).update(changes).run(self.conn)
+        skipped = False in map(lambda x: results[x] == 0, results.keys())
+        if results['skipped'] > 0 or results['errors'] > 0 or not skipped:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        results = self.table.get_all(job_id).delete().run(self.conn)
+        if results['deleted'] + results['skipped'] != 1:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        self.table.delete().run(self.conn)
+
+    def shutdown(self):
+        self.conn.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, predicate=None):
+        jobs = []
+        failed_job_ids = []
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
+                 if predicate else self.table)
+        query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
+
+        for document in query.run(self.conn):
+            try:
+                jobs.append(self._reconstitute_job(document['job_state']))
+            except Exception:
+                self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
+                failed_job_ids.append(document['id'])
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            self.r.expr(failed_job_ids).for_each(
+                lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
+
+        return jobs
+
+    def __repr__(self):
+        connection = self.conn
+        return '<%s (connection=%s)>' % (self.__class__.__name__, connection)

lib/apscheduler/jobstores/sqlalchemy.py

@@ -10,29 +10,38 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select
+    from sqlalchemy import (
+        create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select, and_)
     from sqlalchemy.exc import IntegrityError
+    from sqlalchemy.sql.expression import null
 except ImportError:  # pragma: nocover
     raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed')
 
 
 class SQLAlchemyJobStore(BaseJobStore):
     """
-    Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database.
+    Stores jobs in a database table using SQLAlchemy.
+    The table will be created if it doesn't exist in the database.
 
     Plugin alias: ``sqlalchemy``
 
-    :param str url: connection string (see `SQLAlchemy documentation
-                    <http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#database-urls>`_
-                    on this)
-    :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url``
+    :param str url: connection string (see
+        :ref:`SQLAlchemy documentation <sqlalchemy:database_urls>` on this)
+    :param engine: an SQLAlchemy :class:`~sqlalchemy.engine.Engine` to use instead of creating a
+        new one based on ``url``
     :param str tablename: name of the table to store jobs in
-    :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param metadata: a :class:`~sqlalchemy.schema.MetaData` instance to use instead of creating a
+        new one
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    :param str tableschema: name of the (existing) schema in the target database where the table
+        should be
+    :param dict engine_options: keyword arguments to :func:`~sqlalchemy.create_engine`
+        (ignored if ``engine`` is given)
     """
 
     def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None,
-                 pickle_protocol=pickle.HIGHEST_PROTOCOL):
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, tableschema=None, engine_options=None):
         super(SQLAlchemyJobStore, self).__init__()
         self.pickle_protocol = pickle_protocol
         metadata = maybe_ref(metadata) or MetaData()
@@ -40,37 +49,46 @@ class SQLAlchemyJobStore(BaseJobStore):
         if engine:
             self.engine = maybe_ref(engine)
         elif url:
-            self.engine = create_engine(url)
+            self.engine = create_engine(url, **(engine_options or {}))
         else:
             raise ValueError('Need either "engine" or "url" defined')
 
-        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float
+        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables,
+        # 25 = precision that translates to an 8-byte float
         self.jobs_t = Table(
             tablename, metadata,
-            Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True),
+            Column('id', Unicode(191), primary_key=True),
             Column('next_run_time', Float(25), index=True),
-            Column('job_state', LargeBinary, nullable=False)
+            Column('job_state', LargeBinary, nullable=False),
+            schema=tableschema
         )
 
+    def start(self, scheduler, alias):
+        super(SQLAlchemyJobStore, self).start(scheduler, alias)
         self.jobs_t.create(self.engine, True)
 
     def lookup_job(self, job_id):
-        selectable = select([self.jobs_t.c.job_state]).where(self.jobs_t.c.id == job_id)
-        job_state = self.engine.execute(selectable).scalar()
-        return self._reconstitute_job(job_state) if job_state else None
+        selectable = select(self.jobs_t.c.job_state).where(self.jobs_t.c.id == job_id)
+        with self.engine.begin() as connection:
+            job_state = connection.execute(selectable).scalar()
+            return self._reconstitute_job(job_state) if job_state else None
 
     def get_due_jobs(self, now):
         timestamp = datetime_to_utc_timestamp(now)
         return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp)
 
     def get_next_run_time(self):
-        selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\
+        selectable = select(self.jobs_t.c.next_run_time).\
+            where(self.jobs_t.c.next_run_time != null()).\
             order_by(self.jobs_t.c.next_run_time).limit(1)
-        next_run_time = self.engine.execute(selectable).scalar()
-        return utc_timestamp_to_datetime(next_run_time)
+        with self.engine.begin() as connection:
+            next_run_time = connection.execute(selectable).scalar()
+            return utc_timestamp_to_datetime(next_run_time)
 
     def get_all_jobs(self):
-        return self._get_jobs()
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         insert = self.jobs_t.insert().values(**{
@@ -78,29 +96,33 @@ class SQLAlchemyJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         })
-        try:
-            self.engine.execute(insert)
-        except IntegrityError:
-            raise ConflictingIdError(job.id)
+        with self.engine.begin() as connection:
+            try:
+                connection.execute(insert)
+            except IntegrityError:
+                raise ConflictingIdError(job.id)
 
     def update_job(self, job):
         update = self.jobs_t.update().values(**{
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         }).where(self.jobs_t.c.id == job.id)
-        result = self.engine.execute(update)
-        if result.rowcount == 0:
-            raise JobLookupError(id)
+        with self.engine.begin() as connection:
+            result = connection.execute(update)
+            if result.rowcount == 0:
+                raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)
-        result = self.engine.execute(delete)
-        if result.rowcount == 0:
-            raise JobLookupError(job_id)
+        with self.engine.begin() as connection:
+            result = connection.execute(delete)
+            if result.rowcount == 0:
+                raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
         delete = self.jobs_t.delete()
-        self.engine.execute(delete)
+        with self.engine.begin() as connection:
+            connection.execute(delete)
 
     def shutdown(self):
         self.engine.dispose()
@@ -116,20 +138,22 @@ class SQLAlchemyJobStore(BaseJobStore):
 
     def _get_jobs(self, *conditions):
         jobs = []
-        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time)
-        selectable = selectable.where(*conditions) if conditions else selectable
+        selectable = select(self.jobs_t.c.id, self.jobs_t.c.job_state).\
+            order_by(self.jobs_t.c.next_run_time)
+        selectable = selectable.where(and_(*conditions)) if conditions else selectable
         failed_job_ids = set()
-        for row in self.engine.execute(selectable):
-            try:
-                jobs.append(self._reconstitute_job(row.job_state))
-            except:
-                self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
-                failed_job_ids.add(row.id)
+        with self.engine.begin() as connection:
+            for row in connection.execute(selectable):
+                try:
+                    jobs.append(self._reconstitute_job(row.job_state))
+                except BaseException:
+                    self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
+                    failed_job_ids.add(row.id)
 
-        # Remove all the jobs we failed to restore
-        if failed_job_ids:
-            delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
-            self.engine.execute(delete)
+            # Remove all the jobs we failed to restore
+            if failed_job_ids:
+                delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
+                connection.execute(delete)
 
         return jobs
 

lib/apscheduler/jobstores/zookeeper.py

@@ -0,0 +1,178 @@
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from pytz import utc
+from kazoo.exceptions import NoNodeError, NodeExistsError
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from kazoo.client import KazooClient
+except ImportError:  # pragma: nocover
+    raise ImportError('ZooKeeperJobStore requires Kazoo installed')
+
+
+class ZooKeeperJobStore(BaseJobStore):
+    """
+    Stores jobs in a ZooKeeper tree. Any leftover keyword arguments are directly passed to
+    kazoo's `KazooClient
+    <http://kazoo.readthedocs.io/en/latest/api/client.html>`_.
+
+    Plugin alias: ``zookeeper``
+
+    :param str path: path to store jobs in
+    :param client: a :class:`~kazoo.client.KazooClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, path='/apscheduler', client=None, close_connection_on_exit=False,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(ZooKeeperJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+        self.close_connection_on_exit = close_connection_on_exit
+
+        if not path:
+            raise ValueError('The "path" parameter must not be empty')
+
+        self.path = path
+
+        if client:
+            self.client = maybe_ref(client)
+        else:
+            self.client = KazooClient(**connect_args)
+        self._ensured_path = False
+
+    def _ensure_paths(self):
+        if not self._ensured_path:
+            self.client.ensure_path(self.path)
+        self._ensured_path = True
+
+    def start(self, scheduler, alias):
+        super(ZooKeeperJobStore, self).start(scheduler, alias)
+        if not self.client.connected:
+            self.client.start()
+
+    def lookup_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            content, _ = self.client.get(node_path)
+            doc = pickle.loads(content)
+            job = self._reconstitute_job(doc['job_state'])
+            return job
+        except BaseException:
+            return None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        jobs = [job_def['job'] for job_def in self._get_jobs()
+                if job_def['next_run_time'] is not None and job_def['next_run_time'] <= timestamp]
+        return jobs
+
+    def get_next_run_time(self):
+        next_runs = [job_def['next_run_time'] for job_def in self._get_jobs()
+                     if job_def['next_run_time'] is not None]
+        return utc_timestamp_to_datetime(min(next_runs)) if len(next_runs) > 0 else None
+
+    def get_all_jobs(self):
+        jobs = [job_def['job'] for job_def in self._get_jobs()]
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        value = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(value, self.pickle_protocol)
+        try:
+            self.client.create(node_path, value=data)
+        except NodeExistsError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(changes, self.pickle_protocol)
+        try:
+            self.client.set(node_path, value=data)
+        except NoNodeError:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            self.client.delete(node_path)
+        except NoNodeError:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        try:
+            self.client.delete(self.path, recursive=True)
+        except NoNodeError:
+            pass
+        self._ensured_path = False
+
+    def shutdown(self):
+        if self.close_connection_on_exit:
+            self.client.stop()
+            self.client.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = job_state
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self):
+        self._ensure_paths()
+        jobs = []
+        failed_job_ids = []
+        all_ids = self.client.get_children(self.path)
+        for node_name in all_ids:
+            try:
+                node_path = self.path + "/" + node_name
+                content, _ = self.client.get(node_path)
+                doc = pickle.loads(content)
+                job_def = {
+                    'job_id': node_name,
+                    'next_run_time': doc['next_run_time'] if doc['next_run_time'] else None,
+                    'job_state': doc['job_state'],
+                    'job': self._reconstitute_job(doc['job_state']),
+                    'creation_time': _.ctime
+                }
+                jobs.append(job_def)
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it' % node_name)
+                failed_job_ids.append(node_name)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            for failed_id in failed_job_ids:
+                self.remove_job(failed_id)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job_def: (job_def['job'].next_run_time or paused_sort_key,
+                                                 job_def['creation_time']))
+
+    def __repr__(self):
+        self._logger.exception('<%s (client=%s)>' % (self.__class__.__name__, self.client))
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/schedulers/asyncio.py

@@ -1,22 +1,16 @@
 from __future__ import absolute_import
-from functools import wraps
+import asyncio
+from functools import wraps, partial
 
 from apscheduler.schedulers.base import BaseScheduler
 from apscheduler.util import maybe_ref
 
-try:
-    import asyncio
-except ImportError:  # pragma: nocover
-    try:
-        import trollius as asyncio
-    except ImportError:
-        raise ImportError('AsyncIOScheduler requires either Python 3.4 or the asyncio package installed')
-
 
 def run_in_event_loop(func):
     @wraps(func)
     def wrapper(self, *args, **kwargs):
-        self._eventloop.call_soon_threadsafe(func, self, *args, **kwargs)
+        wrapped = partial(func, self, *args, **kwargs)
+        self._eventloop.call_soon_threadsafe(wrapped)
     return wrapper
 
 
@@ -24,6 +18,8 @@ class AsyncIOScheduler(BaseScheduler):
     """
     A scheduler that runs on an asyncio (:pep:`3156`) event loop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     Extra options:
 
     ============== =============================================================
@@ -34,9 +30,11 @@ class AsyncIOScheduler(BaseScheduler):
     _eventloop = None
     _timeout = None
 
-    def start(self):
-        super(AsyncIOScheduler, self).start()
-        self.wakeup()
+    def start(self, paused=False):
+        if not self._eventloop:
+            self._eventloop = asyncio.get_event_loop()
+
+        super(AsyncIOScheduler, self).start(paused)
 
     @run_in_event_loop
     def shutdown(self, wait=True):
@@ -44,7 +42,7 @@ class AsyncIOScheduler(BaseScheduler):
         self._stop_timer()
 
     def _configure(self, config):
-        self._eventloop = maybe_ref(config.pop('event_loop', None)) or asyncio.get_event_loop()
+        self._eventloop = maybe_ref(config.pop('event_loop', None))
         super(AsyncIOScheduler, self)._configure(config)
 
     def _start_timer(self, wait_seconds):

lib/apscheduler/schedulers/background.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from threading import Thread, Event
 
 from apscheduler.schedulers.base import BaseScheduler
@@ -13,11 +14,12 @@ class BackgroundScheduler(BlockingScheduler):
 
     Extra options:
 
-    ========== ============================================================================================
-    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``,
-               see `the documentation <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
+    ========== =============================================================================
+    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``, see
+               `the documentation
+               <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
                for further details)
-    ========== ============================================================================================
+    ========== =============================================================================
     """
 
     _thread = None
@@ -26,14 +28,16 @@ class BackgroundScheduler(BlockingScheduler):
         self._daemon = asbool(config.pop('daemon', True))
         super(BackgroundScheduler, self)._configure(config)
 
-    def start(self):
-        BaseScheduler.start(self)
-        self._event = Event()
+    def start(self, *args, **kwargs):
+        if self._event is None or self._event.is_set():
+            self._event = Event()
+
+        BaseScheduler.start(self, *args, **kwargs)
         self._thread = Thread(target=self._main_loop, name='APScheduler')
         self._thread.daemon = self._daemon
         self._thread.start()
 
-    def shutdown(self, wait=True):
-        super(BackgroundScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(BackgroundScheduler, self).shutdown(*args, **kwargs)
         self._thread.join()
         del self._thread

lib/apscheduler/schedulers/blocking.py

@@ -1,21 +1,23 @@
 from __future__ import absolute_import
+
 from threading import Event
 
-from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.schedulers.base import BaseScheduler, STATE_STOPPED
+from apscheduler.util import TIMEOUT_MAX
 
 
 class BlockingScheduler(BaseScheduler):
     """
-    A scheduler that runs in the foreground (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
+    A scheduler that runs in the foreground
+    (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
     """
-
-    MAX_WAIT_TIME = 4294967  # Maximum value accepted by Event.wait() on Windows
-
     _event = None
 
-    def start(self):
-        super(BlockingScheduler, self).start()
-        self._event = Event()
+    def start(self, *args, **kwargs):
+        if self._event is None or self._event.is_set():
+            self._event = Event()
+
+        super(BlockingScheduler, self).start(*args, **kwargs)
         self._main_loop()
 
     def shutdown(self, wait=True):
@@ -23,10 +25,11 @@ class BlockingScheduler(BaseScheduler):
         self._event.set()
 
     def _main_loop(self):
-        while self.running:
-            wait_seconds = self._process_jobs()
-            self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME)
+        wait_seconds = TIMEOUT_MAX
+        while self.state != STATE_STOPPED:
+            self._event.wait(wait_seconds)
             self._event.clear()
+            wait_seconds = self._process_jobs()
 
     def wakeup(self):
         self._event.set()

lib/apscheduler/schedulers/gevent.py

@@ -16,14 +16,14 @@ class GeventScheduler(BlockingScheduler):
 
     _greenlet = None
 
-    def start(self):
-        BaseScheduler.start(self)
+    def start(self, *args, **kwargs):
         self._event = Event()
+        BaseScheduler.start(self, *args, **kwargs)
         self._greenlet = gevent.spawn(self._main_loop)
         return self._greenlet
 
-    def shutdown(self, wait=True):
-        super(GeventScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(GeventScheduler, self).shutdown(*args, **kwargs)
         self._greenlet.join()
         del self._greenlet
 

lib/apscheduler/schedulers/qt.py

@@ -4,14 +4,21 @@ from apscheduler.schedulers.base import BaseScheduler
 
 try:
     from PyQt5.QtCore import QObject, QTimer
-except ImportError:  # pragma: nocover
+except (ImportError, RuntimeError):  # pragma: nocover
     try:
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide6.QtCore import QObject, QTimer  # noqa
         except ImportError:
-            raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
+            try:
+                from PySide2.QtCore import QObject, QTimer  # noqa
+            except ImportError:
+                try:
+                    from PySide.QtCore import QObject, QTimer  # noqa
+                except ImportError:
+                    raise ImportError('QtScheduler requires either PyQt5, PyQt4, PySide6, PySide2 '
+                                      'or PySide installed')
 
 
 class QtScheduler(BaseScheduler):
@@ -19,18 +26,15 @@ class QtScheduler(BaseScheduler):
 
     _timer = None
 
-    def start(self):
-        super(QtScheduler, self).start()
-        self.wakeup()
-
-    def shutdown(self, wait=True):
-        super(QtScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(QtScheduler, self).shutdown(*args, **kwargs)
         self._stop_timer()
 
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(int(wait_seconds * 1000), 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/schedulers/tornado.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from datetime import timedelta
 from functools import wraps
 
@@ -22,6 +23,8 @@ class TornadoScheduler(BaseScheduler):
     """
     A scheduler that runs on a Tornado IOLoop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     =========== ===============================================================
     ``io_loop`` Tornado IOLoop instance to use (defaults to the global IO loop)
     =========== ===============================================================
@@ -30,10 +33,6 @@ class TornadoScheduler(BaseScheduler):
     _ioloop = None
     _timeout = None
 
-    def start(self):
-        super(TornadoScheduler, self).start()
-        self.wakeup()
-
     @run_in_ioloop
     def shutdown(self, wait=True):
         super(TornadoScheduler, self).shutdown(wait)
@@ -53,6 +52,10 @@ class TornadoScheduler(BaseScheduler):
             self._ioloop.remove_timeout(self._timeout)
             del self._timeout
 
+    def _create_default_executor(self):
+        from apscheduler.executors.tornado import TornadoExecutor
+        return TornadoExecutor()
+
     @run_in_ioloop
     def wakeup(self):
         self._stop_timer()

lib/apscheduler/schedulers/twisted.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from functools import wraps
 
 from apscheduler.schedulers.base import BaseScheduler
@@ -35,10 +36,6 @@ class TwistedScheduler(BaseScheduler):
         self._reactor = maybe_ref(config.pop('reactor', default_reactor))
         super(TwistedScheduler, self)._configure(config)
 
-    def start(self):
-        super(TwistedScheduler, self).start()
-        self.wakeup()
-
     @run_in_reactor
     def shutdown(self, wait=True):
         super(TwistedScheduler, self).shutdown(wait)

lib/apscheduler/triggers/base.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from datetime import timedelta
+import random
 
 import six
 
@@ -6,11 +8,30 @@ import six
 class BaseTrigger(six.with_metaclass(ABCMeta)):
     """Abstract base class that defines the interface that every trigger must implement."""
 
+    __slots__ = ()
+
     @abstractmethod
     def get_next_fire_time(self, previous_fire_time, now):
         """
-        Returns the next datetime to fire on, If no such datetime can be calculated, returns ``None``.
+        Returns the next datetime to fire on, If no such datetime can be calculated, returns
+        ``None``.
 
         :param datetime.datetime previous_fire_time: the previous time the trigger was fired
         :param datetime.datetime now: current datetime
         """
+
+    def _apply_jitter(self, next_fire_time, jitter, now):
+        """
+        Randomize ``next_fire_time`` by adding a random value (the jitter).
+
+        :param datetime.datetime|None next_fire_time: next fire time without jitter applied. If
+            ``None``, returns ``None``.
+        :param int|None jitter: maximum number of seconds to add to ``next_fire_time``
+            (if ``None`` or ``0``, returns ``next_fire_time``)
+        :param datetime.datetime now: current datetime
+        :return datetime.datetime|None: next fire time with a jitter.
+        """
+        if next_fire_time is None or not jitter:
+            return next_fire_time
+
+        return next_fire_time + timedelta(seconds=random.uniform(0, jitter))

lib/apscheduler/triggers/combining.py

@@ -0,0 +1,95 @@
+from apscheduler.triggers.base import BaseTrigger
+from apscheduler.util import obj_to_ref, ref_to_obj
+
+
+class BaseCombiningTrigger(BaseTrigger):
+    __slots__ = ('triggers', 'jitter')
+
+    def __init__(self, triggers, jitter=None):
+        self.triggers = triggers
+        self.jitter = jitter
+
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'triggers': [(obj_to_ref(trigger.__class__), trigger.__getstate__())
+                         for trigger in self.triggers],
+            'jitter': self.jitter
+        }
+
+    def __setstate__(self, state):
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 1 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.jitter = state['jitter']
+        self.triggers = []
+        for clsref, state in state['triggers']:
+            cls = ref_to_obj(clsref)
+            trigger = cls.__new__(cls)
+            trigger.__setstate__(state)
+            self.triggers.append(trigger)
+
+    def __repr__(self):
+        return '<{}({}{})>'.format(self.__class__.__name__, self.triggers,
+                                   ', jitter={}'.format(self.jitter) if self.jitter else '')
+
+
+class AndTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time that all the given triggers can agree on.
+    The trigger is considered to be finished when any of the given triggers has finished its
+    schedule.
+
+    Trigger alias: ``and``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        while True:
+            fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                          for trigger in self.triggers]
+            if None in fire_times:
+                return None
+            elif min(fire_times) == max(fire_times):
+                return self._apply_jitter(fire_times[0], self.jitter, now)
+            else:
+                now = max(fire_times)
+
+    def __str__(self):
+        return 'and[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))
+
+
+class OrTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time produced by any of the given triggers.
+    The trigger is considered finished when all the given triggers have finished their schedules.
+
+    Trigger alias: ``or``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+
+    .. note:: Triggers that depends on the previous fire time, such as the interval trigger, may
+        seem to behave strangely since they are always passed the previous fire time produced by
+        any of the given triggers.
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                      for trigger in self.triggers]
+        fire_times = [fire_time for fire_time in fire_times if fire_time is not None]
+        if fire_times:
+            return self._apply_jitter(min(fire_times), self.jitter, now)
+        else:
+            return None
+
+    def __str__(self):
+        return 'or[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))

lib/apscheduler/triggers/cron/__init__.py

@@ -4,17 +4,20 @@ from tzlocal import get_localzone
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.triggers.cron.fields import BaseField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES
-from apscheduler.util import datetime_ceil, convert_to_datetime, datetime_repr, astimezone
+from apscheduler.triggers.cron.fields import (
+    BaseField, MonthField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES)
+from apscheduler.util import (
+    datetime_ceil, convert_to_datetime, datetime_repr, astimezone, localize, normalize)
 
 
 class CronTrigger(BaseTrigger):
     """
-    Triggers when current time matches all specified time constraints, similarly to how the UNIX cron scheduler works.
+    Triggers when current time matches all specified time constraints,
+    similarly to how the UNIX cron scheduler works.
 
     :param int|str year: 4-digit year
     :param int|str month: month (1-12)
-    :param int|str day: day of the (1-31)
+    :param int|str day: day of month (1-31)
     :param int|str week: ISO week (1-53)
     :param int|str day_of_week: number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun)
     :param int|str hour: hour (0-23)
@@ -22,8 +25,9 @@ class CronTrigger(BaseTrigger):
     :param int|str second: second (0-59)
     :param datetime|str start_date: earliest possible date/time to trigger on (inclusive)
     :param datetime|str end_date: latest possible date/time to trigger on (inclusive)
-    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
-                                         (defaults to scheduler timezone)
+    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (defaults
+        to scheduler timezone)
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
 
     .. note:: The first weekday is always **monday**.
     """
@@ -31,7 +35,7 @@ class CronTrigger(BaseTrigger):
     FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second')
     FIELDS_MAP = {
         'year': BaseField,
-        'month': BaseField,
+        'month': MonthField,
         'week': WeekField,
         'day': DayOfMonthField,
         'day_of_week': DayOfWeekField,
@@ -40,15 +44,16 @@ class CronTrigger(BaseTrigger):
         'second': BaseField
     }
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'fields'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'fields', 'jitter'
 
-    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None,
-                 second=None, start_date=None, end_date=None, timezone=None):
+    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None,
+                 minute=None, second=None, start_date=None, end_date=None, timezone=None,
+                 jitter=None):
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -56,6 +61,8 @@ class CronTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
         values = dict((key, value) for (key, value) in six.iteritems(locals())
                       if key in self.FIELD_NAMES and value is not None)
         self.fields = []
@@ -76,13 +83,35 @@ class CronTrigger(BaseTrigger):
             field = field_class(field_name, exprs, is_default)
             self.fields.append(field)
 
+    @classmethod
+    def from_crontab(cls, expr, timezone=None):
+        """
+        Create a :class:`~CronTrigger` from a standard crontab expression.
+
+        See https://en.wikipedia.org/wiki/Cron for more information on the format accepted here.
+
+        :param expr: minute, hour, day of month, month, day of week
+        :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (
+            defaults to scheduler timezone)
+        :return: a :class:`~CronTrigger` instance
+
+        """
+        values = expr.split()
+        if len(values) != 5:
+            raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))
+
+        return cls(minute=values[0], hour=values[1], day=values[2], month=values[3],
+                   day_of_week=values[4], timezone=timezone)
+
     def _increment_field_value(self, dateval, fieldnum):
         """
-        Increments the designated field and resets all less significant fields to their minimum values.
+        Increments the designated field and resets all less significant fields to their minimum
+        values.
 
         :type dateval: datetime
         :type fieldnum: int
-        :return: a tuple containing the new date, and the number of the field that was actually incremented
+        :return: a tuple containing the new date, and the number of the field that was actually
+            incremented
         :rtype: tuple
         """
 
@@ -115,7 +144,7 @@ class CronTrigger(BaseTrigger):
                     i += 1
 
         difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference), fieldnum
+        return normalize(dateval + difference), fieldnum
 
     def _set_field_value(self, dateval, fieldnum, new_value):
         values = {}
@@ -128,12 +157,13 @@ class CronTrigger(BaseTrigger):
                 else:
                     values[field.name] = new_value
 
-        difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference)
+        return localize(datetime(**values), self.timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
-            start_date = max(now, previous_fire_time + timedelta(microseconds=1))
+            start_date = min(now, previous_fire_time + timedelta(microseconds=1))
+            if start_date == previous_fire_time:
+                start_date += timedelta(microseconds=1)
         else:
             start_date = max(now, self.start_date) if self.start_date else now
 
@@ -163,7 +193,34 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            return next_date
+            next_date = self._apply_jitter(next_date, self.jitter, now)
+            return min(next_date, self.end_date) if self.end_date else next_date
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'fields': self.fields,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.fields = state['fields']
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
@@ -172,5 +229,11 @@ class CronTrigger(BaseTrigger):
     def __repr__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
         if self.start_date:
-            options.append("start_date='%s'" % datetime_repr(self.start_date))
-        return '<%s (%s)>' % (self.__class__.__name__, ', '.join(options))
+            options.append("start_date=%r" % datetime_repr(self.start_date))
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/triggers/cron/expressions.py

@@ -1,17 +1,16 @@
-"""
-This module contains the expressions applicable for CronTrigger's fields.
-"""
+"""This module contains the expressions applicable for CronTrigger's fields."""
 
 from calendar import monthrange
 import re
 
 from apscheduler.util import asint
 
-__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', 'WeekdayPositionExpression',
-           'LastDayOfMonthExpression')
+__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
+           'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
 WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
+MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 
 class AllExpression(object):
@@ -22,6 +21,14 @@ class AllExpression(object):
         if self.step == 0:
             raise ValueError('Increment must be higher than 0')
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        value_range = MAX_VALUES[field_name] - MIN_VALUES[field_name]
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
         start = field.get_value(date)
         minval = field.get_min(date)
@@ -37,6 +44,9 @@ class AllExpression(object):
         if next <= maxval:
             return next
 
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.step == other.step
+
     def __str__(self):
         if self.step:
             return '*/%d' % self.step
@@ -51,7 +61,7 @@ class RangeExpression(AllExpression):
         r'(?P<first>\d+)(?:-(?P<last>\d+))?(?:/(?P<step>\d+))?$')
 
     def __init__(self, first, last=None, step=None):
-        AllExpression.__init__(self, step)
+        super(RangeExpression, self).__init__(step)
         first = asint(first)
         last = asint(last)
         if last is None and step is None:
@@ -61,25 +71,41 @@ class RangeExpression(AllExpression):
         self.first = first
         self.last = last
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        super(RangeExpression, self).validate_range(field_name)
+        if self.first < MIN_VALUES[field_name]:
+            raise ValueError('the first value ({}) is lower than the minimum value ({})'
+                             .format(self.first, MIN_VALUES[field_name]))
+        if self.last is not None and self.last > MAX_VALUES[field_name]:
+            raise ValueError('the last value ({}) is higher than the maximum value ({})'
+                             .format(self.last, MAX_VALUES[field_name]))
+        value_range = (self.last or MAX_VALUES[field_name]) - self.first
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
-        start = field.get_value(date)
+        startval = field.get_value(date)
         minval = field.get_min(date)
         maxval = field.get_max(date)
 
         # Apply range limits
         minval = max(minval, self.first)
-        if self.last is not None:
-            maxval = min(maxval, self.last)
-        start = max(start, minval)
+        maxval = min(maxval, self.last) if self.last is not None else maxval
+        nextval = max(minval, startval)
 
-        if not self.step:
-            next = start
-        else:
-            distance_to_next = (self.step - (start - minval)) % self.step
-            next = start + distance_to_next
+        # Apply the step if defined
+        if self.step:
+            distance_to_next = (self.step - (nextval - minval)) % self.step
+            nextval += distance_to_next
 
-        if next <= maxval:
-            return next
+        return nextval if nextval <= maxval else None
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and self.first == other.first and
+                self.last == other.last)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -100,6 +126,37 @@ class RangeExpression(AllExpression):
         return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 
+class MonthRangeExpression(RangeExpression):
+    value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
+
+    def __init__(self, first, last=None):
+        try:
+            first_num = MONTHS.index(first.lower()) + 1
+        except ValueError:
+            raise ValueError('Invalid month name "%s"' % first)
+
+        if last:
+            try:
+                last_num = MONTHS.index(last.lower()) + 1
+            except ValueError:
+                raise ValueError('Invalid month name "%s"' % last)
+        else:
+            last_num = None
+
+        super(MonthRangeExpression, self).__init__(first_num, last_num)
+
+    def __str__(self):
+        if self.last != self.first and self.last is not None:
+            return '%s-%s' % (MONTHS[self.first - 1], MONTHS[self.last - 1])
+        return MONTHS[self.first - 1]
+
+    def __repr__(self):
+        args = ["'%s'" % MONTHS[self.first]]
+        if self.last != self.first and self.last is not None:
+            args.append("'%s'" % MONTHS[self.last - 1])
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+
 class WeekdayRangeExpression(RangeExpression):
     value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
 
@@ -117,7 +174,7 @@ class WeekdayRangeExpression(RangeExpression):
         else:
             last_num = None
 
-        RangeExpression.__init__(self, first_num, last_num)
+        super(WeekdayRangeExpression, self).__init__(first_num, last_num)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -133,9 +190,11 @@ class WeekdayRangeExpression(RangeExpression):
 
 class WeekdayPositionExpression(AllExpression):
     options = ['1st', '2nd', '3rd', '4th', '5th', 'last']
-    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' % '|'.join(options), re.IGNORECASE)
+    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' %
+                          '|'.join(options), re.IGNORECASE)
 
     def __init__(self, option_name, weekday_name):
+        super(WeekdayPositionExpression, self).__init__(None)
         try:
             self.option_num = self.options.index(option_name.lower())
         except ValueError:
@@ -147,8 +206,7 @@ class WeekdayPositionExpression(AllExpression):
             raise ValueError('Invalid weekday name "%s"' % weekday_name)
 
     def get_next_value(self, date, field):
-        # Figure out the weekday of the month's first day and the number
-        # of days in that month
+        # Figure out the weekday of the month's first day and the number of days in that month
         first_day_wday, last_day = monthrange(date.year, date.month)
 
         # Calculate which day of the month is the first of the target weekdays
@@ -160,23 +218,28 @@ class WeekdayPositionExpression(AllExpression):
         if self.option_num < 5:
             target_day = first_hit_day + self.option_num * 7
         else:
-            target_day = first_hit_day + ((last_day - first_hit_day) / 7) * 7
+            target_day = first_hit_day + ((last_day - first_hit_day) // 7) * 7
 
         if target_day <= last_day and target_day >= date.day:
             return target_day
 
+    def __eq__(self, other):
+        return (super(WeekdayPositionExpression, self).__eq__(other) and
+                self.option_num == other.option_num and self.weekday == other.weekday)
+
     def __str__(self):
         return '%s %s' % (self.options[self.option_num], WEEKDAYS[self.weekday])
 
     def __repr__(self):
-        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num], WEEKDAYS[self.weekday])
+        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num],
+                                   WEEKDAYS[self.weekday])
 
 
 class LastDayOfMonthExpression(AllExpression):
     value_re = re.compile(r'last', re.IGNORECASE)
 
     def __init__(self):
-        pass
+        super(LastDayOfMonthExpression, self).__init__(None)
 
     def get_next_value(self, date, field):
         return monthrange(date.year, date.month)[1]

lib/apscheduler/triggers/cron/fields.py

@@ -1,22 +1,26 @@
-"""
-Fields represent CronTrigger options which map to :class:`~datetime.datetime`
-fields.
-"""
+"""Fields represent CronTrigger options which map to :class:`~datetime.datetime` fields."""
 
 from calendar import monthrange
+import re
+
+import six
 
 from apscheduler.triggers.cron.expressions import (
-    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression, WeekdayRangeExpression)
+    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression,
+    WeekdayRangeExpression, MonthRangeExpression)
 
 
-__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField', 'DayOfMonthField', 'DayOfWeekField')
+__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField',
+           'DayOfMonthField', 'DayOfWeekField')
 
 
-MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0}
-MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, 'day_of_week': 6, 'hour': 23, 'minute': 59,
-              'second': 59}
-DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0, 'minute': 0,
-                  'second': 0}
+MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0,
+              'minute': 0, 'second': 0}
+MAX_VALUES = {'year': 9999, 'month': 12, 'day': 31, 'week': 53, 'day_of_week': 6, 'hour': 23,
+              'minute': 59, 'second': 59}
+DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0,
+                  'minute': 0, 'second': 0}
+SEPARATOR = re.compile(' *, *')
 
 
 class BaseField(object):
@@ -50,23 +54,29 @@ class BaseField(object):
         self.expressions = []
 
         # Split a comma-separated expression list, if any
-        exprs = str(exprs).strip()
-        if ',' in exprs:
-            for expr in exprs.split(','):
-                self.compile_expression(expr)
-        else:
-            self.compile_expression(exprs)
+        for expr in SEPARATOR.split(str(exprs).strip()):
+            self.compile_expression(expr)
 
     def compile_expression(self, expr):
         for compiler in self.COMPILERS:
             match = compiler.value_re.match(expr)
             if match:
                 compiled_expr = compiler(**match.groupdict())
+
+                try:
+                    compiled_expr.validate_range(self.name)
+                except ValueError as e:
+                    exc = ValueError('Error validating expression {!r}: {}'.format(expr, e))
+                    six.raise_from(exc, None)
+
                 self.expressions.append(compiled_expr)
                 return
 
         raise ValueError('Unrecognized expression "%s" for field "%s"' % (expr, self.name))
 
+    def __eq__(self, other):
+        return isinstance(self, self.__class__) and self.expressions == other.expressions
+
     def __str__(self):
         expr_strings = (str(e) for e in self.expressions)
         return ','.join(expr_strings)
@@ -95,3 +105,7 @@ class DayOfWeekField(BaseField):
 
     def get_value(self, dateval):
         return dateval.weekday()
+
+
+class MonthField(BaseField):
+    COMPILERS = BaseField.COMPILERS + [MonthRangeExpression]

lib/apscheduler/triggers/date.py

@@ -14,15 +14,36 @@ class DateTrigger(BaseTrigger):
     :param datetime.tzinfo|str timezone: time zone for ``run_date`` if it doesn't have one already
     """
 
-    __slots__ = 'timezone', 'run_date'
+    __slots__ = 'run_date'
 
     def __init__(self, run_date=None, timezone=None):
         timezone = astimezone(timezone) or get_localzone()
-        self.run_date = convert_to_datetime(run_date or datetime.now(), timezone, 'run_date')
+        if run_date is not None:
+            self.run_date = convert_to_datetime(run_date, timezone, 'run_date')
+        else:
+            self.run_date = datetime.now(timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         return self.run_date if previous_fire_time is None else None
 
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'run_date': self.run_date
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only version 1 can be handled' %
+                (state['version'], self.__class__.__name__))
+
+        self.run_date = state['run_date']
+
     def __str__(self):
         return 'date[%s]' % datetime_repr(self.run_date)
 

lib/apscheduler/triggers/interval.py

@@ -4,13 +4,15 @@ from math import ceil
 from tzlocal import get_localzone
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import convert_to_datetime, timedelta_seconds, datetime_repr, astimezone
+from apscheduler.util import (
+    convert_to_datetime, normalize, timedelta_seconds, datetime_repr,
+    astimezone)
 
 
 class IntervalTrigger(BaseTrigger):
     """
-    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` + interval
-    otherwise.
+    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` +
+    interval otherwise.
 
     :param int weeks: number of weeks to wait
     :param int days: number of days to wait
@@ -20,12 +22,15 @@ class IntervalTrigger(BaseTrigger):
     :param datetime|str start_date: starting point for the interval calculation
     :param datetime|str end_date: latest possible date/time to trigger on
     :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
     """
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'interval'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'interval', 'interval_length', 'jitter'
 
-    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None):
-        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)
+    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None,
+                 end_date=None, timezone=None, jitter=None):
+        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes,
+                                  seconds=seconds)
         self.interval_length = timedelta_seconds(self.interval)
         if self.interval_length == 0:
             self.interval = timedelta(seconds=1)
@@ -33,9 +38,9 @@ class IntervalTrigger(BaseTrigger):
 
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -44,6 +49,8 @@ class IntervalTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
             next_fire_time = previous_fire_time + self.interval
@@ -54,12 +61,48 @@ class IntervalTrigger(BaseTrigger):
             next_interval_num = int(ceil(timediff_seconds / self.interval_length))
             next_fire_time = self.start_date + self.interval * next_interval_num
 
+        if self.jitter is not None:
+            next_fire_time = self._apply_jitter(next_fire_time, self.jitter, now)
+
         if not self.end_date or next_fire_time <= self.end_date:
-            return self.timezone.normalize(next_fire_time)
+            return normalize(next_fire_time)
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'interval': self.interval,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.interval = state['interval']
+        self.interval_length = timedelta_seconds(self.interval)
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         return 'interval[%s]' % str(self.interval)
 
     def __repr__(self):
-        return "<%s (interval=%r, start_date='%s')>" % (self.__class__.__name__, self.interval,
-                                                        datetime_repr(self.start_date))
+        options = ['interval=%r' % self.interval, 'start_date=%r' % datetime_repr(self.start_date)]
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/util.py

@@ -1,25 +1,32 @@
 """This module contains several handy functions primarily meant for internal use."""
 
 from __future__ import division
-from datetime import date, datetime, time, timedelta, tzinfo
-from inspect import isfunction, ismethod, getargspec
-from calendar import timegm
-import re
 
-from pytz import timezone, utc
+from asyncio import iscoroutinefunction
+from datetime import date, datetime, time, timedelta, tzinfo
+from calendar import timegm
+from functools import partial
+from inspect import isclass, ismethod
+import re
+import sys
+
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
     from inspect import signature
 except ImportError:  # pragma: nocover
-    try:
-        from funcsigs import signature
-    except ImportError:
-        signature = None
+    from funcsigs import signature
+
+try:
+    from threading import TIMEOUT_MAX
+except ImportError:
+    TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
-           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name', 'obj_to_ref',
-           'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'normalize', 'localize', 'TIMEOUT_MAX')
 
 
 class _Undefined(object):
@@ -32,17 +39,18 @@ class _Undefined(object):
     def __repr__(self):
         return '<undefined>'
 
+
 undefined = _Undefined()  #: a unique object that only signifies that no value is defined
 
 
 def asint(text):
     """
-    Safely converts a string to an integer, returning None if the string is None.
+    Safely converts a string to an integer, returning ``None`` if the string is ``None``.
 
     :type text: str
     :rtype: int
-    """
 
+    """
     if text is not None:
         return int(text)
 
@@ -52,8 +60,8 @@ def asbool(obj):
     Interprets an object as a boolean value.
 
     :rtype: bool
-    """
 
+    """
     if isinstance(obj, str):
         obj = obj.strip().lower()
         if obj in ('true', 'yes', 'on', 'y', 't', '1'):
@@ -69,15 +77,17 @@ def astimezone(obj):
     Interprets an object as a timezone.
 
     :rtype: tzinfo
-    """
 
+    """
     if isinstance(obj, six.string_types):
         return timezone(obj)
     if isinstance(obj, tzinfo):
-        if not hasattr(obj, 'localize') or not hasattr(obj, 'normalize'):
-            raise TypeError('Only timezones from the pytz library are supported')
-        if obj.zone == 'local':
-            raise ValueError('Unable to determine the name of the local timezone -- use an explicit timezone instead')
+        if obj.tzname(None) == 'local':
+            raise ValueError(
+                'Unable to determine the name of the local timezone -- you must explicitly '
+                'specify the name of the local timezone. Please refrain from using timezones like '
+                'EST to prevent problems with daylight saving time. Instead, use a locale based '
+                'timezone name (such as Europe/Helsinki).')
         return obj
     if obj is not None:
         raise TypeError('Expected tzinfo, got %s instead' % obj.__class__.__name__)
@@ -85,27 +95,30 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
     """
     Converts the given object to a timezone aware datetime object.
+
     If a timezone aware datetime object is passed, it is returned unmodified.
     If a native datetime object is passed, it is given the specified timezone.
     If the input is a string, it is parsed as a datetime with the given timezone.
 
-    Date strings are accepted in three different forms: date only (Y-m-d),
-    date with time (Y-m-d H:M:S) or with date+time with microseconds
-    (Y-m-d H:M:S.micro).
+    Date strings are accepted in three different forms: date only (Y-m-d), date with time
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
     :param str arg_name: the name of the argument (used in an error message)
     :rtype: datetime
-    """
 
+    """
     if input is None:
         return
     elif isinstance(input, datetime):
@@ -116,8 +129,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
-        values = dict(values)
+
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -125,14 +147,12 @@ def convert_to_datetime(input, tz, arg_name):
     if datetime_.tzinfo is not None:
         return datetime_
     if tz is None:
-        raise ValueError('The "tz" argument must be specified if %s has no timezone information' % arg_name)
+        raise ValueError(
+            'The "tz" argument must be specified if %s has no timezone information' % arg_name)
     if isinstance(tz, six.string_types):
         tz = timezone(tz)
 
-    try:
-        return tz.localize(datetime_, is_dst=None)
-    except AttributeError:
-        raise TypeError('Only pytz timezones are supported (need the localize() and normalize() methods)')
+    return localize(datetime_, tz)
 
 
 def datetime_to_utc_timestamp(timeval):
@@ -141,8 +161,8 @@ def datetime_to_utc_timestamp(timeval):
 
     :type timeval: datetime
     :rtype: float
-    """
 
+    """
     if timeval is not None:
         return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000
 
@@ -153,8 +173,8 @@ def utc_timestamp_to_datetime(timestamp):
 
     :type timestamp: float
     :rtype: datetime
-    """
 
+    """
     if timestamp is not None:
         return datetime.fromtimestamp(timestamp, utc)
 
@@ -165,8 +185,8 @@ def timedelta_seconds(delta):
 
     :type delta: timedelta
     :rtype: float
-    """
 
+    """
     return delta.days * 24 * 60 * 60 + delta.seconds + \
         delta.microseconds / 1000000.0
 
@@ -176,8 +196,8 @@ def datetime_ceil(dateval):
     Rounds the given datetime object upwards.
 
     :type dateval: datetime
-    """
 
+    """
     if dateval.microsecond > 0:
         return dateval + timedelta(seconds=1, microseconds=-dateval.microsecond)
     return dateval
@@ -192,8 +212,8 @@ def get_callable_name(func):
     Returns the best available display name for the given function/callable.
 
     :rtype: str
-    """
 
+    """
     # the easy case (on Python 3.3+)
     if hasattr(func, '__qualname__'):
         return func.__qualname__
@@ -201,7 +221,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -222,20 +242,35 @@ def get_callable_name(func):
 
 def obj_to_ref(obj):
     """
-    Returns the path to the given object.
+    Returns the path to the given callable.
 
     :rtype: str
+    :raises TypeError: if the given object is not callable
+    :raises ValueError: if the given object is a :class:`~functools.partial`, lambda or a nested
+        function
+
     """
+    if isinstance(obj, partial):
+        raise ValueError('Cannot create a reference to a partial()')
 
-    try:
-        ref = '%s:%s' % (obj.__module__, get_callable_name(obj))
-        obj2 = ref_to_obj(ref)
-        if obj != obj2:
-            raise ValueError
-    except Exception:
-        raise ValueError('Cannot determine the reference to %r' % obj)
+    name = get_callable_name(obj)
+    if '<lambda>' in name:
+        raise ValueError('Cannot create a reference to a lambda')
+    if '<locals>' in name:
+        raise ValueError('Cannot create a reference to a nested function')
 
-    return ref
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -243,8 +278,8 @@ def ref_to_obj(ref):
     Returns the object pointed to by ``ref``.
 
     :type ref: str
-    """
 
+    """
     if not isinstance(ref, six.string_types):
         raise TypeError('References must be strings')
     if ':' not in ref:
@@ -252,12 +287,12 @@ def ref_to_obj(ref):
 
     modulename, rest = ref.split(':', 1)
     try:
-        obj = __import__(modulename)
+        obj = __import__(modulename, fromlist=[rest])
     except ImportError:
         raise LookupError('Error resolving reference %s: could not import module' % ref)
 
     try:
-        for name in modulename.split('.')[1:] + rest.split('.'):
+        for name in rest.split('.'):
             obj = getattr(obj, name)
         return obj
     except Exception:
@@ -268,8 +303,8 @@ def maybe_ref(ref):
     """
     Returns the object that the given reference points to, if it is indeed a reference.
     If it is not a reference, the object is returned as-is.
-    """
 
+    """
     if not isinstance(ref, str):
         return ref
     return ref_to_obj(ref)
@@ -281,7 +316,8 @@ if six.PY2:
             return string.encode('ascii', 'backslashreplace')
         return string
 else:
-    repr_escape = lambda string: string
+    def repr_escape(string):
+        return string
 
 
 def check_callable_args(func, args, kwargs):
@@ -290,70 +326,54 @@ def check_callable_args(func, args, kwargs):
 
     :type args: tuple
     :type kwargs: dict
-    """
 
+    """
     pos_kwargs_conflicts = []  # parameters that have a match in both args and kwargs
     positional_only_kwargs = []  # positional-only parameters that have a match in kwargs
     unsatisfied_args = []  # parameters in signature that don't have a match in args or kwargs
     unsatisfied_kwargs = []  # keyword-only arguments that don't have a match in kwargs
     unmatched_args = list(args)  # args that didn't match any of the parameters in the signature
-    unmatched_kwargs = list(kwargs)  # kwargs that didn't match any of the parameters in the signature
-    has_varargs = has_var_kwargs = False  # indicates if the signature defines *args and **kwargs respectively
+    # kwargs that didn't match any of the parameters in the signature
+    unmatched_kwargs = list(kwargs)
+    # indicates if the signature defines *args and **kwargs respectively
+    has_varargs = has_var_kwargs = False
 
-    if signature:
-        try:
+    try:
+        if sys.version_info >= (3, 5):
+            sig = signature(func, follow_wrapped=False)
+        else:
             sig = signature(func)
-        except ValueError:
-            return  # signature() doesn't work against every kind of callable
+    except ValueError:
+        # signature() doesn't work against every kind of callable
+        return
 
-        for param in six.itervalues(sig.parameters):
-            if param.kind == param.POSITIONAL_OR_KEYWORD:
-                if param.name in unmatched_kwargs and unmatched_args:
-                    pos_kwargs_conflicts.append(param.name)
-                elif unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.POSITIONAL_ONLY:
-                if unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                    positional_only_kwargs.append(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.KEYWORD_ONLY:
-                if param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_kwargs.append(param.name)
-            elif param.kind == param.VAR_POSITIONAL:
-                has_varargs = True
-            elif param.kind == param.VAR_KEYWORD:
-                has_var_kwargs = True
-    else:
-        if not isfunction(func) and not ismethod(func) and hasattr(func, '__call__'):
-            func = func.__call__
-
-        try:
-            argspec = getargspec(func)
-        except TypeError:
-            return  # getargspec() doesn't work certain callables
-
-        argspec_args = argspec.args if not ismethod(func) else argspec.args[1:]
-        has_varargs = bool(argspec.varargs)
-        has_var_kwargs = bool(argspec.keywords)
-        for arg, default in six.moves.zip_longest(argspec_args, argspec.defaults or (), fillvalue=undefined):
-            if arg in unmatched_kwargs and unmatched_args:
-                pos_kwargs_conflicts.append(arg)
+    for param in six.itervalues(sig.parameters):
+        if param.kind == param.POSITIONAL_OR_KEYWORD:
+            if param.name in unmatched_kwargs and unmatched_args:
+                pos_kwargs_conflicts.append(param.name)
             elif unmatched_args:
                 del unmatched_args[0]
-            elif arg in unmatched_kwargs:
-                unmatched_kwargs.remove(arg)
-            elif default is undefined:
-                unsatisfied_args.append(arg)
+            elif param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+            elif param.default is param.empty:
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.POSITIONAL_ONLY:
+            if unmatched_args:
+                del unmatched_args[0]
+            elif param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+                positional_only_kwargs.append(param.name)
+            elif param.default is param.empty:
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.KEYWORD_ONLY:
+            if param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+            elif param.default is param.empty:
+                unsatisfied_kwargs.append(param.name)
+        elif param.kind == param.VAR_POSITIONAL:
+            has_varargs = True
+        elif param.kind == param.VAR_KEYWORD:
+            has_var_kwargs = True
 
     # Make sure there are no conflicts between args and kwargs
     if pos_kwargs_conflicts:
@@ -365,21 +385,46 @@ def check_callable_args(func, args, kwargs):
         raise ValueError('The following arguments cannot be given as keyword arguments: %s' %
                          ', '.join(positional_only_kwargs))
 
-    # Check that the number of positional arguments minus the number of matched kwargs matches the argspec
+    # Check that the number of positional arguments minus the number of matched kwargs matches the
+    # argspec
     if unsatisfied_args:
-        raise ValueError('The following arguments have not been supplied: %s' % ', '.join(unsatisfied_args))
+        raise ValueError('The following arguments have not been supplied: %s' %
+                         ', '.join(unsatisfied_args))
 
     # Check that all keyword-only arguments have been supplied
     if unsatisfied_kwargs:
-        raise ValueError('The following keyword-only arguments have not been supplied in kwargs: %s' %
-                         ', '.join(unsatisfied_kwargs))
+        raise ValueError(
+            'The following keyword-only arguments have not been supplied in kwargs: %s' %
+            ', '.join(unsatisfied_kwargs))
 
     # Check that the callable can accept the given number of positional arguments
     if not has_varargs and unmatched_args:
-        raise ValueError('The list of positional arguments is longer than the target callable can handle '
-                         '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
+        raise ValueError(
+            'The list of positional arguments is longer than the target callable can handle '
+            '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
 
     # Check that the callable can accept the given keyword arguments
     if not has_var_kwargs and unmatched_kwargs:
-        raise ValueError('The target callable does not accept the following keyword arguments: %s' %
-                         ', '.join(unmatched_kwargs))
+        raise ValueError(
+            'The target callable does not accept the following keyword arguments: %s' %
+            ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)
+
+
+def normalize(dt):
+    return datetime.fromtimestamp(dt.timestamp(), dt.tzinfo)
+
+
+def localize(dt, tzinfo):
+    if hasattr(tzinfo, 'localize'):
+        return tzinfo.localize(dt)
+
+    return normalize(dt.replace(tzinfo=tzinfo))

lib/beets/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -13,33 +12,30 @@
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 
-from __future__ import division, absolute_import, print_function
 
-import os
+from sys import stderr
 
-from beets.util import confit
+import confuse
 
-# This particular version has been slightly modified to work with Headphones
-# https://github.com/rembo10/headphones
-__version__ = u'1.4.4-headphones'
-__author__ = u'Adrian Sampson <adrian@radbox.org>'
+__version__ = "2.0.0"
+__author__ = "Adrian Sampson <adrian@radbox.org>"
 
 
-class IncludeLazyConfig(confit.LazyConfig):
-    """A version of Confit's LazyConfig that also merges in data from
+class IncludeLazyConfig(confuse.LazyConfig):
+    """A version of Confuse's LazyConfig that also merges in data from
     YAML files specified in an `include` setting.
     """
+
     def read(self, user=True, defaults=True):
-        super(IncludeLazyConfig, self).read(user, defaults)
+        super().read(user, defaults)
 
         try:
-            for view in self['include']:
-                filename = view.as_filename()
-                if os.path.isfile(filename):
-                    self.set_file(filename)
-        except confit.NotFoundError:
+            for view in self["include"]:
+                self.set_file(view.as_filename())
+        except confuse.NotFoundError:
             pass
+        except confuse.ConfigReadError as err:
+            stderr.write("configuration `import` failed: {}".format(err.reason))
 
-# headphones
-#config = IncludeLazyConfig('beets', __name__)
-config = IncludeLazyConfig(os.path.dirname(__file__), __name__)
+
+config = IncludeLazyConfig("beets", __name__)

lib/beets/__main__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2017, Adrian Sampson.
 #
@@ -17,9 +16,9 @@
 `python -m beets`.
 """
 
-from __future__ import division, absolute_import, print_function
 
 import sys
+
 from .ui import main
 
 if __name__ == "__main__":

lib/beets/art.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -17,23 +16,20 @@
 music and items' embedded album art.
 """
 
-from __future__ import division, absolute_import, print_function
 
-import subprocess
-import platform
-from tempfile import NamedTemporaryFile
 import os
+from tempfile import NamedTemporaryFile
 
-from beets.util import displayable_path, syspath, bytestring_path
+import mediafile
+
+from beets.util import bytestring_path, displayable_path, syspath
 from beets.util.artresizer import ArtResizer
-from beets import mediafile
 
 
 def mediafile_image(image_path, maxwidth=None):
-    """Return a `mediafile.Image` object for the path.
-    """
+    """Return a `mediafile.Image` object for the path."""
 
-    with open(syspath(image_path), 'rb') as f:
+    with open(syspath(image_path), "rb") as f:
         data = f.read()
     return mediafile.Image(data, type=mediafile.ImageType.front)
 
@@ -43,166 +39,168 @@ def get_art(log, item):
     try:
         mf = mediafile.MediaFile(syspath(item.path))
     except mediafile.UnreadableFileError as exc:
-        log.warning(u'Could not extract art from {0}: {1}',
-                    displayable_path(item.path), exc)
+        log.warning(
+            "Could not extract art from {0}: {1}",
+            displayable_path(item.path),
+            exc,
+        )
         return
 
     return mf.art
 
 
-def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
-               compare_threshold=0, ifempty=False, as_album=False):
-    """Embed an image into the item's media file.
-    """
-    # Conditions and filters.
+def embed_item(
+    log,
+    item,
+    imagepath,
+    maxwidth=None,
+    itempath=None,
+    compare_threshold=0,
+    ifempty=False,
+    as_album=False,
+    id3v23=None,
+    quality=0,
+):
+    """Embed an image into the item's media file."""
+    # Conditions.
     if compare_threshold:
-        if not check_art_similarity(log, item, imagepath, compare_threshold):
-            log.info(u'Image not similar; skipping.')
+        is_similar = check_art_similarity(
+            log, item, imagepath, compare_threshold
+        )
+        if is_similar is None:
+            log.warning("Error while checking art similarity; skipping.")
             return
+        elif not is_similar:
+            log.info("Image not similar; skipping.")
+            return
+
     if ifempty and get_art(log, item):
-            log.info(u'media file already contained art')
-            return
+        log.info("media file already contained art")
+        return
+
+    # Filters.
     if maxwidth and not as_album:
-        imagepath = resize_image(log, imagepath, maxwidth)
+        imagepath = resize_image(log, imagepath, maxwidth, quality)
 
     # Get the `Image` object from the file.
     try:
-        log.debug(u'embedding {0}', displayable_path(imagepath))
+        log.debug("embedding {0}", displayable_path(imagepath))
         image = mediafile_image(imagepath, maxwidth)
-    except IOError as exc:
-        log.warning(u'could not read image file: {0}', exc)
+    except OSError as exc:
+        log.warning("could not read image file: {0}", exc)
         return
 
     # Make sure the image kind is safe (some formats only support PNG
     # and JPEG).
-    if image.mime_type not in ('image/jpeg', 'image/png'):
-        log.info('not embedding image of unsupported type: {}',
-                 image.mime_type)
+    if image.mime_type not in ("image/jpeg", "image/png"):
+        log.info("not embedding image of unsupported type: {}", image.mime_type)
         return
 
-    item.try_write(path=itempath, tags={'images': [image]})
+    item.try_write(path=itempath, tags={"images": [image]}, id3v23=id3v23)
 
 
-def embed_album(log, album, maxwidth=None, quiet=False,
-                compare_threshold=0, ifempty=False):
-    """Embed album art into all of the album's items.
-    """
+def embed_album(
+    log,
+    album,
+    maxwidth=None,
+    quiet=False,
+    compare_threshold=0,
+    ifempty=False,
+    quality=0,
+):
+    """Embed album art into all of the album's items."""
     imagepath = album.artpath
     if not imagepath:
-        log.info(u'No album art present for {0}', album)
+        log.info("No album art present for {0}", album)
         return
     if not os.path.isfile(syspath(imagepath)):
-        log.info(u'Album art not found at {0} for {1}',
-                 displayable_path(imagepath), album)
+        log.info(
+            "Album art not found at {0} for {1}",
+            displayable_path(imagepath),
+            album,
+        )
         return
     if maxwidth:
-        imagepath = resize_image(log, imagepath, maxwidth)
+        imagepath = resize_image(log, imagepath, maxwidth, quality)
 
-    log.info(u'Embedding album art into {0}', album)
+    log.info("Embedding album art into {0}", album)
 
     for item in album.items():
-        embed_item(log, item, imagepath, maxwidth, None,
-                   compare_threshold, ifempty, as_album=True)
+        embed_item(
+            log,
+            item,
+            imagepath,
+            maxwidth,
+            None,
+            compare_threshold,
+            ifempty,
+            as_album=True,
+            quality=quality,
+        )
 
 
-def resize_image(log, imagepath, maxwidth):
-    """Returns path to an image resized to maxwidth.
+def resize_image(log, imagepath, maxwidth, quality):
+    """Returns path to an image resized to maxwidth and encoded with the
+    specified quality level.
     """
-    log.debug(u'Resizing album art to {0} pixels wide', maxwidth)
-    imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath))
+    log.debug(
+        "Resizing album art to {0} pixels wide and encoding at quality \
+              level {1}",
+        maxwidth,
+        quality,
+    )
+    imagepath = ArtResizer.shared.resize(
+        maxwidth, syspath(imagepath), quality=quality
+    )
     return imagepath
 
 
-def check_art_similarity(log, item, imagepath, compare_threshold):
+def check_art_similarity(
+    log,
+    item,
+    imagepath,
+    compare_threshold,
+    artresizer=None,
+):
     """A boolean indicating if an image is similar to embedded item art.
+
+    If no embedded art exists, always return `True`. If the comparison fails
+    for some reason, the return value is `None`.
+
+    This must only be called if `ArtResizer.shared.can_compare` is `True`.
     """
     with NamedTemporaryFile(delete=True) as f:
         art = extract(log, f.name, item)
 
-        if art:
-            is_windows = platform.system() == "Windows"
+        if not art:
+            return True
 
-            # Converting images to grayscale tends to minimize the weight
-            # of colors in the diff score. So we first convert both images
-            # to grayscale and then pipe them into the `compare` command.
-            # On Windows, ImageMagick doesn't support the magic \\?\ prefix
-            # on paths, so we pass `prefix=False` to `syspath`.
-            convert_cmd = ['convert', syspath(imagepath, prefix=False),
-                           syspath(art, prefix=False),
-                           '-colorspace', 'gray', 'MIFF:-']
-            compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:']
-            log.debug(u'comparing images with pipeline {} | {}',
-                      convert_cmd, compare_cmd)
-            convert_proc = subprocess.Popen(
-                convert_cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                close_fds=not is_windows,
-            )
-            compare_proc = subprocess.Popen(
-                compare_cmd,
-                stdin=convert_proc.stdout,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                close_fds=not is_windows,
-            )
+        if artresizer is None:
+            artresizer = ArtResizer.shared
 
-            # Check the convert output. We're not interested in the
-            # standard output; that gets piped to the next stage.
-            convert_proc.stdout.close()
-            convert_stderr = convert_proc.stderr.read()
-            convert_proc.stderr.close()
-            convert_proc.wait()
-            if convert_proc.returncode:
-                log.debug(
-                    u'ImageMagick convert failed with status {}: {!r}',
-                    convert_proc.returncode,
-                    convert_stderr,
-                )
-                return
-
-            # Check the compare output.
-            stdout, stderr = compare_proc.communicate()
-            if compare_proc.returncode:
-                if compare_proc.returncode != 1:
-                    log.debug(u'ImageMagick compare failed: {0}, {1}',
-                              displayable_path(imagepath),
-                              displayable_path(art))
-                    return
-                out_str = stderr
-            else:
-                out_str = stdout
-
-            try:
-                phash_diff = float(out_str)
-            except ValueError:
-                log.debug(u'IM output is not a number: {0!r}', out_str)
-                return
-
-            log.debug(u'ImageMagick compare score: {0}', phash_diff)
-            return phash_diff <= compare_threshold
-
-    return True
+        return artresizer.compare(art, imagepath, compare_threshold)
 
 
 def extract(log, outpath, item):
     art = get_art(log, item)
     outpath = bytestring_path(outpath)
     if not art:
-        log.info(u'No album art present in {0}, skipping.', item)
+        log.info("No album art present in {0}, skipping.", item)
         return
 
     # Add an extension to the filename.
     ext = mediafile.image_extension(art)
     if not ext:
-        log.warning(u'Unknown image type in {0}.',
-                    displayable_path(item.path))
+        log.warning("Unknown image type in {0}.", displayable_path(item.path))
         return
-    outpath += bytestring_path('.' + ext)
+    outpath += bytestring_path("." + ext)
 
-    log.info(u'Extracting album art from: {0} to: {1}',
-             item, displayable_path(outpath))
-    with open(syspath(outpath), 'wb') as f:
+    log.info(
+        "Extracting album art from: {0} to: {1}",
+        item,
+        displayable_path(outpath),
+    )
+    with open(syspath(outpath), "wb") as f:
         f.write(art)
     return outpath
 
@@ -216,7 +214,7 @@ def extract_first(log, outpath, items):
 
 def clear(log, lib, query):
     items = lib.items(query)
-    log.info(u'Clearing album art from {0} items', len(items))
+    log.info("Clearing album art from {0} items", len(items))
     for item in items:
-        log.debug(u'Clearing art for {0}', item)
-        item.try_write(tags={'images': None})
+        log.debug("Clearing art for {0}", item)
+        item.try_write(tags={"images": None})

lib/beets/autotag/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -15,79 +14,156 @@
 
 """Facilities for automatically determining files' correct metadata.
 """
+from typing import Mapping
 
-from __future__ import division, absolute_import, print_function
-
-from beets import logging
-from beets import config
+from beets import config, logging
+from beets.library import Item
 
 # Parts of external interface.
-from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch  # noqa
-from .match import tag_item, tag_album, Proposal  # noqa
+from .hooks import (  # noqa
+    AlbumInfo,
+    AlbumMatch,
+    Distance,
+    TrackInfo,
+    TrackMatch,
+)
 from .match import Recommendation  # noqa
+from .match import Proposal, current_metadata, tag_album, tag_item  # noqa
 
 # Global logger.
-log = logging.getLogger('beets')
+log = logging.getLogger("beets")
+
+# Metadata fields that are already hardcoded, or where the tag name changes.
+SPECIAL_FIELDS = {
+    "album": (
+        "va",
+        "releasegroup_id",
+        "artist_id",
+        "artists_ids",
+        "album_id",
+        "mediums",
+        "tracks",
+        "year",
+        "month",
+        "day",
+        "artist",
+        "artists",
+        "artist_credit",
+        "artists_credit",
+        "artist_sort",
+        "artists_sort",
+        "data_url",
+    ),
+    "track": (
+        "track_alt",
+        "artist_id",
+        "artists_ids",
+        "release_track_id",
+        "medium",
+        "index",
+        "medium_index",
+        "title",
+        "artist_credit",
+        "artists_credit",
+        "artist_sort",
+        "artists_sort",
+        "artist",
+        "artists",
+        "track_id",
+        "medium_total",
+        "data_url",
+        "length",
+    ),
+}
 
 
 # Additional utilities for the main interface.
 
-def apply_item_metadata(item, track_info):
-    """Set an item's metadata from its matched TrackInfo object.
-    """
+
+def apply_item_metadata(item: Item, track_info: TrackInfo):
+    """Set an item's metadata from its matched TrackInfo object."""
     item.artist = track_info.artist
+    item.artists = track_info.artists
     item.artist_sort = track_info.artist_sort
+    item.artists_sort = track_info.artists_sort
     item.artist_credit = track_info.artist_credit
+    item.artists_credit = track_info.artists_credit
     item.title = track_info.title
     item.mb_trackid = track_info.track_id
+    item.mb_releasetrackid = track_info.release_track_id
     if track_info.artist_id:
         item.mb_artistid = track_info.artist_id
-    if track_info.data_source:
-        item.data_source = track_info.data_source
+    if track_info.artists_ids:
+        item.mb_artistids = track_info.artists_ids
 
-    if track_info.lyricist is not None:
-        item.lyricist = track_info.lyricist
-    if track_info.composer is not None:
-        item.composer = track_info.composer
-    if track_info.arranger is not None:
-        item.arranger = track_info.arranger
+    for field, value in track_info.items():
+        # We only overwrite fields that are not already hardcoded.
+        if field in SPECIAL_FIELDS["track"]:
+            continue
+        if value is None:
+            continue
+        item[field] = value
 
     # At the moment, the other metadata is left intact (including album
     # and track number). Perhaps these should be emptied?
 
 
-def apply_metadata(album_info, mapping):
+def apply_metadata(album_info: AlbumInfo, mapping: Mapping[Item, TrackInfo]):
     """Set the items' metadata to match an AlbumInfo object using a
     mapping from Items to TrackInfo objects.
     """
     for item, track_info in mapping.items():
-        # Album, artist, track count.
-        if track_info.artist:
-            item.artist = track_info.artist
+        # Artist or artist credit.
+        if config["artist_credit"]:
+            item.artist = (
+                track_info.artist_credit
+                or track_info.artist
+                or album_info.artist_credit
+                or album_info.artist
+            )
+            item.artists = (
+                track_info.artists_credit
+                or track_info.artists
+                or album_info.artists_credit
+                or album_info.artists
+            )
+            item.albumartist = album_info.artist_credit or album_info.artist
+            item.albumartists = album_info.artists_credit or album_info.artists
         else:
-            item.artist = album_info.artist
-        item.albumartist = album_info.artist
+            item.artist = track_info.artist or album_info.artist
+            item.artists = track_info.artists or album_info.artists
+            item.albumartist = album_info.artist
+            item.albumartists = album_info.artists
+
+        # Album.
         item.album = album_info.album
 
         # Artist sort and credit names.
         item.artist_sort = track_info.artist_sort or album_info.artist_sort
-        item.artist_credit = (track_info.artist_credit or
-                              album_info.artist_credit)
+        item.artists_sort = track_info.artists_sort or album_info.artists_sort
+        item.artist_credit = (
+            track_info.artist_credit or album_info.artist_credit
+        )
+        item.artists_credit = (
+            track_info.artists_credit or album_info.artists_credit
+        )
         item.albumartist_sort = album_info.artist_sort
+        item.albumartists_sort = album_info.artists_sort
         item.albumartist_credit = album_info.artist_credit
+        item.albumartists_credit = album_info.artists_credit
 
         # Release date.
-        for prefix in '', 'original_':
-            if config['original_date'] and not prefix:
+        for prefix in "", "original_":
+            if config["original_date"] and not prefix:
                 # Ignore specific release date.
                 continue
 
-            for suffix in 'year', 'month', 'day':
+            for suffix in "year", "month", "day":
                 key = prefix + suffix
                 value = getattr(album_info, key) or 0
 
                 # If we don't even have a year, apply nothing.
-                if suffix == 'year' and not value:
+                if suffix == "year" and not value:
                     break
 
                 # Otherwise, set the fetched value (or 0 for the month
@@ -96,13 +172,13 @@ def apply_metadata(album_info, mapping):
 
                 # If we're using original release date for both fields,
                 # also set item.year = info.original_year, etc.
-                if config['original_date']:
+                if config["original_date"]:
                     item[suffix] = value
 
         # Title.
         item.title = track_info.title
 
-        if config['per_disc_numbering']:
+        if config["per_disc_numbering"]:
             # We want to let the track number be zero, but if the medium index
             # is not provided we need to fall back to the overall index.
             if track_info.medium_index is not None:
@@ -120,45 +196,43 @@ def apply_metadata(album_info, mapping):
 
         # MusicBrainz IDs.
         item.mb_trackid = track_info.track_id
+        item.mb_releasetrackid = track_info.release_track_id
         item.mb_albumid = album_info.album_id
         if track_info.artist_id:
             item.mb_artistid = track_info.artist_id
         else:
             item.mb_artistid = album_info.artist_id
+
+        if track_info.artists_ids:
+            item.mb_artistids = track_info.artists_ids
+        else:
+            item.mb_artistids = album_info.artists_ids
+
         item.mb_albumartistid = album_info.artist_id
+        item.mb_albumartistids = album_info.artists_ids
         item.mb_releasegroupid = album_info.releasegroup_id
 
         # Compilation flag.
         item.comp = album_info.va
 
-        # Miscellaneous metadata.
-        for field in ('albumtype',
-                      'label',
-                      'asin',
-                      'catalognum',
-                      'script',
-                      'language',
-                      'country',
-                      'albumstatus',
-                      'albumdisambig',
-                      'data_source',):
-            value = getattr(album_info, field)
-            if value is not None:
-                item[field] = value
-        if track_info.disctitle is not None:
-            item.disctitle = track_info.disctitle
-
-        if track_info.media is not None:
-            item.media = track_info.media
-
-        if track_info.lyricist is not None:
-            item.lyricist = track_info.lyricist
-        if track_info.composer is not None:
-            item.composer = track_info.composer
-        if track_info.arranger is not None:
-            item.arranger = track_info.arranger
-
+        # Track alt.
         item.track_alt = track_info.track_alt
 
-        # Headphones seal of approval
-        item.comments = 'tagged by headphones/beets'
+        # Don't overwrite fields with empty values unless the
+        # field is explicitly allowed to be overwritten
+        for field, value in album_info.items():
+            if field in SPECIAL_FIELDS["album"]:
+                continue
+            clobber = field in config["overwrite_null"]["album"].as_str_seq()
+            if value is None and not clobber:
+                continue
+            item[field] = value
+
+        for field, value in track_info.items():
+            if field in SPECIAL_FIELDS["track"]:
+                continue
+            clobber = field in config["overwrite_null"]["track"].as_str_seq()
+            value = getattr(track_info, field)
+            if value is None and not clobber:
+                continue
+            item[field] = value

lib/beets/autotag/hooks.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -14,27 +13,59 @@
 # included in all copies or substantial portions of the Software.
 
 """Glue between metadata sources and the matching logic."""
-from __future__ import division, absolute_import, print_function
 
+from __future__ import annotations
+
+import re
 from collections import namedtuple
 from functools import total_ordering
-import re
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
 
-from beets import logging
-from beets import plugins
-from beets import config
-from beets.util import as_string
-from beets.autotag import mb
 from jellyfish import levenshtein_distance
 from unidecode import unidecode
-import six
 
-log = logging.getLogger('beets')
+from beets import config, logging, plugins
+from beets.autotag import mb
+from beets.library import Item
+from beets.util import as_string, cached_classproperty
+
+log = logging.getLogger("beets")
+
+V = TypeVar("V")
 
 
 # Classes used to represent candidate options.
+class AttrDict(Dict[str, V]):
+    """A dictionary that supports attribute ("dot") access, so `d.field`
+    is equivalent to `d['field']`.
+    """
 
-class AlbumInfo(object):
+    def __getattr__(self, attr: str) -> V:
+        if attr in self:
+            return self[attr]
+        else:
+            raise AttributeError
+
+    def __setattr__(self, key: str, value: V):
+        self.__setitem__(key, value)
+
+    def __hash__(self):
+        return id(self)
+
+
+class AlbumInfo(AttrDict):
     """Describes a canonical release that may be used to match a release
     in the library. Consists of these data members:
 
@@ -43,124 +74,163 @@ class AlbumInfo(object):
     - ``artist``: name of the release's primary artist
     - ``artist_id``
     - ``tracks``: list of TrackInfo objects making up the release
-    - ``asin``: Amazon ASIN
-    - ``albumtype``: string describing the kind of release
-    - ``va``: boolean: whether the release has "various artists"
-    - ``year``: release year
-    - ``month``: release month
-    - ``day``: release day
-    - ``label``: music label responsible for the release
-    - ``mediums``: the number of discs in this release
-    - ``artist_sort``: name of the release's artist for sorting
-    - ``releasegroup_id``: MBID for the album's release group
-    - ``catalognum``: the label's catalog number for the release
-    - ``script``: character set used for metadata
-    - ``language``: human language of the metadata
-    - ``country``: the release country
-    - ``albumstatus``: MusicBrainz release status (Official, etc.)
-    - ``media``: delivery mechanism (Vinyl, etc.)
-    - ``albumdisambig``: MusicBrainz release disambiguation comment
-    - ``artist_credit``: Release-specific artist name
-    - ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
-    - ``data_url``: The data source release URL.
 
-    The fields up through ``tracks`` are required. The others are
-    optional and may be None.
+    ``mediums`` along with the fields up through ``tracks`` are required.
+    The others are optional and may be None.
     """
-    def __init__(self, album, album_id, artist, artist_id, tracks, asin=None,
-                 albumtype=None, va=False, year=None, month=None, day=None,
-                 label=None, mediums=None, artist_sort=None,
-                 releasegroup_id=None, catalognum=None, script=None,
-                 language=None, country=None, albumstatus=None, media=None,
-                 albumdisambig=None, artist_credit=None, original_year=None,
-                 original_month=None, original_day=None, data_source=None,
-                 data_url=None):
+
+    # TYPING: are all of these correct? I've assumed optional strings
+    def __init__(
+        self,
+        tracks: List[TrackInfo],
+        album: Optional[str] = None,
+        album_id: Optional[str] = None,
+        artist: Optional[str] = None,
+        artist_id: Optional[str] = None,
+        artists: Optional[List[str]] = None,
+        artists_ids: Optional[List[str]] = None,
+        asin: Optional[str] = None,
+        albumtype: Optional[str] = None,
+        albumtypes: Optional[List[str]] = None,
+        va: bool = False,
+        year: Optional[int] = None,
+        month: Optional[int] = None,
+        day: Optional[int] = None,
+        label: Optional[str] = None,
+        barcode: Optional[str] = None,
+        mediums: Optional[int] = None,
+        artist_sort: Optional[str] = None,
+        artists_sort: Optional[List[str]] = None,
+        releasegroup_id: Optional[str] = None,
+        release_group_title: Optional[str] = None,
+        catalognum: Optional[str] = None,
+        script: Optional[str] = None,
+        language: Optional[str] = None,
+        country: Optional[str] = None,
+        style: Optional[str] = None,
+        genre: Optional[str] = None,
+        albumstatus: Optional[str] = None,
+        media: Optional[str] = None,
+        albumdisambig: Optional[str] = None,
+        releasegroupdisambig: Optional[str] = None,
+        artist_credit: Optional[str] = None,
+        artists_credit: Optional[List[str]] = None,
+        original_year: Optional[int] = None,
+        original_month: Optional[int] = None,
+        original_day: Optional[int] = None,
+        data_source: Optional[str] = None,
+        data_url: Optional[str] = None,
+        discogs_albumid: Optional[str] = None,
+        discogs_labelid: Optional[str] = None,
+        discogs_artistid: Optional[str] = None,
+        **kwargs,
+    ):
         self.album = album
         self.album_id = album_id
         self.artist = artist
         self.artist_id = artist_id
+        self.artists = artists or []
+        self.artists_ids = artists_ids or []
         self.tracks = tracks
         self.asin = asin
         self.albumtype = albumtype
+        self.albumtypes = albumtypes or []
         self.va = va
         self.year = year
         self.month = month
         self.day = day
         self.label = label
+        self.barcode = barcode
         self.mediums = mediums
         self.artist_sort = artist_sort
+        self.artists_sort = artists_sort or []
         self.releasegroup_id = releasegroup_id
+        self.release_group_title = release_group_title
         self.catalognum = catalognum
         self.script = script
         self.language = language
         self.country = country
+        self.style = style
+        self.genre = genre
         self.albumstatus = albumstatus
         self.media = media
         self.albumdisambig = albumdisambig
+        self.releasegroupdisambig = releasegroupdisambig
         self.artist_credit = artist_credit
+        self.artists_credit = artists_credit or []
         self.original_year = original_year
         self.original_month = original_month
         self.original_day = original_day
         self.data_source = data_source
         self.data_url = data_url
+        self.discogs_albumid = discogs_albumid
+        self.discogs_labelid = discogs_labelid
+        self.discogs_artistid = discogs_artistid
+        self.update(kwargs)
 
-    # Work around a bug in python-musicbrainz-ngs that causes some
-    # strings to be bytes rather than Unicode.
-    # https://github.com/alastair/python-musicbrainz-ngs/issues/85
-    def decode(self, codec='utf-8'):
-        """Ensure that all string attributes on this object, and the
-        constituent `TrackInfo` objects, are decoded to Unicode.
-        """
-        for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort',
-                    'catalognum', 'script', 'language', 'country',
-                    'albumstatus', 'albumdisambig', 'artist_credit', 'media']:
-            value = getattr(self, fld)
-            if isinstance(value, bytes):
-                setattr(self, fld, value.decode(codec, 'ignore'))
-
-        if self.tracks:
-            for track in self.tracks:
-                track.decode(codec)
+    def copy(self) -> AlbumInfo:
+        dupe = AlbumInfo([])
+        dupe.update(self)
+        dupe.tracks = [track.copy() for track in self.tracks]
+        return dupe
 
 
-class TrackInfo(object):
+class TrackInfo(AttrDict):
     """Describes a canonical track present on a release. Appears as part
     of an AlbumInfo's ``tracks`` list. Consists of these data members:
 
     - ``title``: name of the track
     - ``track_id``: MusicBrainz ID; UUID fragment only
-    - ``artist``: individual track artist name
-    - ``artist_id``
-    - ``length``: float: duration of the track in seconds
-    - ``index``: position on the entire release
-    - ``media``: delivery mechanism (Vinyl, etc.)
-    - ``medium``: the disc number this track appears on in the album
-    - ``medium_index``: the track's position on the disc
-    - ``medium_total``: the number of tracks on the item's disc
-    - ``artist_sort``: name of the track artist for sorting
-    - ``disctitle``: name of the individual medium (subtitle)
-    - ``artist_credit``: Recording-specific artist name
-    - ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
-    - ``data_url``: The data source release URL.
-    - ``lyricist``: individual track lyricist name
-    - ``composer``: individual track composer name
-    - ``arranger`: individual track arranger name
-    - ``track_alt``: alternative track number (tape, vinyl, etc.)
 
     Only ``title`` and ``track_id`` are required. The rest of the fields
     may be None. The indices ``index``, ``medium``, and ``medium_index``
     are all 1-based.
     """
-    def __init__(self, title, track_id, artist=None, artist_id=None,
-                 length=None, index=None, medium=None, medium_index=None,
-                 medium_total=None, artist_sort=None, disctitle=None,
-                 artist_credit=None, data_source=None, data_url=None,
-                 media=None, lyricist=None, composer=None, arranger=None,
-                 track_alt=None):
+
+    # TYPING: are all of these correct? I've assumed optional strings
+    def __init__(
+        self,
+        title: Optional[str] = None,
+        track_id: Optional[str] = None,
+        release_track_id: Optional[str] = None,
+        artist: Optional[str] = None,
+        artist_id: Optional[str] = None,
+        artists: Optional[List[str]] = None,
+        artists_ids: Optional[List[str]] = None,
+        length: Optional[float] = None,
+        index: Optional[int] = None,
+        medium: Optional[int] = None,
+        medium_index: Optional[int] = None,
+        medium_total: Optional[int] = None,
+        artist_sort: Optional[str] = None,
+        artists_sort: Optional[List[str]] = None,
+        disctitle: Optional[str] = None,
+        artist_credit: Optional[str] = None,
+        artists_credit: Optional[List[str]] = None,
+        data_source: Optional[str] = None,
+        data_url: Optional[str] = None,
+        media: Optional[str] = None,
+        lyricist: Optional[str] = None,
+        composer: Optional[str] = None,
+        composer_sort: Optional[str] = None,
+        arranger: Optional[str] = None,
+        track_alt: Optional[str] = None,
+        work: Optional[str] = None,
+        mb_workid: Optional[str] = None,
+        work_disambig: Optional[str] = None,
+        bpm: Optional[str] = None,
+        initial_key: Optional[str] = None,
+        genre: Optional[str] = None,
+        album: Optional[str] = None,
+        **kwargs,
+    ):
         self.title = title
         self.track_id = track_id
+        self.release_track_id = release_track_id
         self.artist = artist
         self.artist_id = artist_id
+        self.artists = artists or []
+        self.artists_ids = artists_ids or []
         self.length = length
         self.index = index
         self.media = media
@@ -168,65 +238,70 @@ class TrackInfo(object):
         self.medium_index = medium_index
         self.medium_total = medium_total
         self.artist_sort = artist_sort
+        self.artists_sort = artists_sort or []
         self.disctitle = disctitle
         self.artist_credit = artist_credit
+        self.artists_credit = artists_credit or []
         self.data_source = data_source
         self.data_url = data_url
         self.lyricist = lyricist
         self.composer = composer
+        self.composer_sort = composer_sort
         self.arranger = arranger
         self.track_alt = track_alt
+        self.work = work
+        self.mb_workid = mb_workid
+        self.work_disambig = work_disambig
+        self.bpm = bpm
+        self.initial_key = initial_key
+        self.genre = genre
+        self.album = album
+        self.update(kwargs)
 
-    # As above, work around a bug in python-musicbrainz-ngs.
-    def decode(self, codec='utf-8'):
-        """Ensure that all string attributes on this object are decoded
-        to Unicode.
-        """
-        for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle',
-                    'artist_credit', 'media']:
-            value = getattr(self, fld)
-            if isinstance(value, bytes):
-                setattr(self, fld, value.decode(codec, 'ignore'))
+    def copy(self) -> TrackInfo:
+        dupe = TrackInfo()
+        dupe.update(self)
+        return dupe
 
 
 # Candidate distance scoring.
 
 # Parameters for string distance function.
 # Words that can be moved to the end of a string using a comma.
-SD_END_WORDS = ['the', 'a', 'an']
+SD_END_WORDS = ["the", "a", "an"]
 # Reduced weights for certain portions of the string.
 SD_PATTERNS = [
-    (r'^the ', 0.1),
-    (r'[\[\(]?(ep|single)[\]\)]?', 0.0),
-    (r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1),
-    (r'\(.*?\)', 0.3),
-    (r'\[.*?\]', 0.3),
-    (r'(, )?(pt\.|part) .+', 0.2),
+    (r"^the ", 0.1),
+    (r"[\[\(]?(ep|single)[\]\)]?", 0.0),
+    (r"[\[\(]?(featuring|feat|ft)[\. :].+", 0.1),
+    (r"\(.*?\)", 0.3),
+    (r"\[.*?\]", 0.3),
+    (r"(, )?(pt\.|part) .+", 0.2),
 ]
 # Replacements to use before testing distance.
 SD_REPLACE = [
-    (r'&', 'and'),
+    (r"&", "and"),
 ]
 
 
-def _string_dist_basic(str1, str2):
+def _string_dist_basic(str1: str, str2: str) -> float:
     """Basic edit distance between two strings, ignoring
     non-alphanumeric characters and case. Comparisons are based on a
     transliteration/lowering to ASCII characters. Normalized by string
     length.
     """
-    assert isinstance(str1, six.text_type)
-    assert isinstance(str2, six.text_type)
+    assert isinstance(str1, str)
+    assert isinstance(str2, str)
     str1 = as_string(unidecode(str1))
     str2 = as_string(unidecode(str2))
-    str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
-    str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
+    str1 = re.sub(r"[^a-z0-9]", "", str1.lower())
+    str2 = re.sub(r"[^a-z0-9]", "", str2.lower())
     if not str1 and not str2:
         return 0.0
     return levenshtein_distance(str1, str2) / float(max(len(str1), len(str2)))
 
 
-def string_dist(str1, str2):
+def string_dist(str1: Optional[str], str2: Optional[str]) -> float:
     """Gives an "intuitive" edit distance between two strings. This is
     an edit distance, normalized by the string length, with a number of
     tweaks that reflect intuition about text.
@@ -243,10 +318,10 @@ def string_dist(str1, str2):
     # example, "the something" should be considered equal to
     # "something, the".
     for word in SD_END_WORDS:
-        if str1.endswith(', %s' % word):
-            str1 = '%s %s' % (word, str1[:-len(word) - 2])
-        if str2.endswith(', %s' % word):
-            str2 = '%s %s' % (word, str2[:-len(word) - 2])
+        if str1.endswith(", %s" % word):
+            str1 = "{} {}".format(word, str1[: -len(word) - 2])
+        if str2.endswith(", %s" % word):
+            str2 = "{} {}".format(word, str2[: -len(word) - 2])
 
     # Perform a couple of basic normalizing substitutions.
     for pat, repl in SD_REPLACE:
@@ -261,8 +336,8 @@ def string_dist(str1, str2):
     penalty = 0.0
     for pat, weight in SD_PATTERNS:
         # Get strings that drop the pattern.
-        case_str1 = re.sub(pat, '', str1)
-        case_str2 = re.sub(pat, '', str2)
+        case_str1 = re.sub(pat, "", str1)
+        case_str2 = re.sub(pat, "", str2)
 
         if case_str1 != str1 or case_str2 != str2:
             # If the pattern was present (i.e., it is deleted in the
@@ -284,37 +359,21 @@ def string_dist(str1, str2):
     return base_dist + penalty
 
 
-class LazyClassProperty(object):
-    """A decorator implementing a read-only property that is *lazy* in
-    the sense that the getter is only invoked once. Subsequent accesses
-    through *any* instance use the cached result.
-    """
-    def __init__(self, getter):
-        self.getter = getter
-        self.computed = False
-
-    def __get__(self, obj, owner):
-        if not self.computed:
-            self.value = self.getter(owner)
-            self.computed = True
-        return self.value
-
-
 @total_ordering
-@six.python_2_unicode_compatible
-class Distance(object):
+class Distance:
     """Keeps track of multiple distance penalties. Provides a single
     weighted distance for all penalties as well as a weighted distance
     for each individual penalty.
     """
+
     def __init__(self):
         self._penalties = {}
+        self.tracks: Dict[TrackInfo, Distance] = {}
 
-    @LazyClassProperty
-    def _weights(cls):  # noqa
-        """A dictionary from keys to floating-point weights.
-        """
-        weights_view = config['match']['distance_weights']
+    @cached_classproperty
+    def _weights(cls) -> Dict[str, float]:  # noqa: N805
+        """A dictionary from keys to floating-point weights."""
+        weights_view = config["match"]["distance_weights"]
         weights = {}
         for key in weights_view.keys():
             weights[key] = weights_view[key].as_number()
@@ -323,7 +382,7 @@ class Distance(object):
     # Access the components and their aggregates.
 
     @property
-    def distance(self):
+    def distance(self) -> float:
         """Return a weighted and normalized distance across all
         penalties.
         """
@@ -333,24 +392,22 @@ class Distance(object):
         return 0.0
 
     @property
-    def max_distance(self):
-        """Return the maximum distance penalty (normalization factor).
-        """
+    def max_distance(self) -> float:
+        """Return the maximum distance penalty (normalization factor)."""
         dist_max = 0.0
         for key, penalty in self._penalties.items():
             dist_max += len(penalty) * self._weights[key]
         return dist_max
 
     @property
-    def raw_distance(self):
-        """Return the raw (denormalized) distance.
-        """
+    def raw_distance(self) -> float:
+        """Return the raw (denormalized) distance."""
         dist_raw = 0.0
         for key, penalty in self._penalties.items():
             dist_raw += sum(penalty) * self._weights[key]
         return dist_raw
 
-    def items(self):
+    def items(self) -> List[Tuple[str, float]]:
         """Return a list of (key, dist) pairs, with `dist` being the
         weighted distance, sorted from highest to lowest. Does not
         include penalties with a zero value.
@@ -364,87 +421,88 @@ class Distance(object):
         # ascending order (for keys, when the penalty is equal) and
         # still get the items with the biggest distance first.
         return sorted(
-            list_,
-            key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0])
+            list_, key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0])
         )
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return id(self)
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return self.distance == other
 
     # Behave like a float.
 
-    def __lt__(self, other):
+    def __lt__(self, other) -> bool:
         return self.distance < other
 
-    def __float__(self):
+    def __float__(self) -> float:
         return self.distance
 
-    def __sub__(self, other):
+    def __sub__(self, other) -> float:
         return self.distance - other
 
-    def __rsub__(self, other):
+    def __rsub__(self, other) -> float:
         return other - self.distance
 
-    def __str__(self):
-        return "{0:.2f}".format(self.distance)
+    def __str__(self) -> str:
+        return f"{self.distance:.2f}"
 
     # Behave like a dict.
 
-    def __getitem__(self, key):
-        """Returns the weighted distance for a named penalty.
-        """
+    def __getitem__(self, key) -> float:
+        """Returns the weighted distance for a named penalty."""
         dist = sum(self._penalties[key]) * self._weights[key]
         dist_max = self.max_distance
         if dist_max:
             return dist / dist_max
         return 0.0
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Tuple[str, float]]:
         return iter(self.items())
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.items())
 
-    def keys(self):
+    def keys(self) -> List[str]:
         return [key for key, _ in self.items()]
 
-    def update(self, dist):
-        """Adds all the distance penalties from `dist`.
-        """
+    def update(self, dist: "Distance"):
+        """Adds all the distance penalties from `dist`."""
         if not isinstance(dist, Distance):
             raise ValueError(
-                u'`dist` must be a Distance object, not {0}'.format(type(dist))
+                "`dist` must be a Distance object, not {}".format(type(dist))
             )
         for key, penalties in dist._penalties.items():
             self._penalties.setdefault(key, []).extend(penalties)
 
     # Adding components.
 
-    def _eq(self, value1, value2):
+    def _eq(self, value1: Union[re.Pattern[str], Any], value2: Any) -> bool:
         """Returns True if `value1` is equal to `value2`. `value1` may
         be a compiled regular expression, in which case it will be
         matched against `value2`.
         """
-        if isinstance(value1, re._pattern_type):
+        if isinstance(value1, re.Pattern):
+            value2 = cast(str, value2)
             return bool(value1.match(value2))
         return value1 == value2
 
-    def add(self, key, dist):
+    def add(self, key: str, dist: float):
         """Adds a distance penalty. `key` must correspond with a
         configured weight setting. `dist` must be a float between 0.0
         and 1.0, and will be added to any existing distance penalties
         for the same key.
         """
         if not 0.0 <= dist <= 1.0:
-            raise ValueError(
-                u'`dist` must be between 0.0 and 1.0, not {0}'.format(dist)
-            )
+            raise ValueError(f"`dist` must be between 0.0 and 1.0, not {dist}")
         self._penalties.setdefault(key, []).append(dist)
 
-    def add_equality(self, key, value, options):
+    def add_equality(
+        self,
+        key: str,
+        value: Any,
+        options: Union[List[Any], Tuple[Any, ...], Any],
+    ):
         """Adds a distance penalty of 1.0 if `value` doesn't match any
         of the values in `options`. If an option is a compiled regular
         expression, it will be considered equal if it matches against
@@ -460,7 +518,7 @@ class Distance(object):
             dist = 1.0
         self.add(key, dist)
 
-    def add_expr(self, key, expr):
+    def add_expr(self, key: str, expr: bool):
         """Adds a distance penalty of 1.0 if `expr` evaluates to True,
         or 0.0.
         """
@@ -469,7 +527,7 @@ class Distance(object):
         else:
             self.add(key, 0.0)
 
-    def add_number(self, key, number1, number2):
+    def add_number(self, key: str, number1: int, number2: int):
         """Adds a distance penalty of 1.0 for each number of difference
         between `number1` and `number2`, or 0.0 when there is no
         difference. Use this when there is no upper limit on the
@@ -482,7 +540,12 @@ class Distance(object):
         else:
             self.add(key, 0.0)
 
-    def add_priority(self, key, value, options):
+    def add_priority(
+        self,
+        key: str,
+        value: Any,
+        options: Union[List[Any], Tuple[Any, ...], Any],
+    ):
         """Adds a distance penalty that corresponds to the position at
         which `value` appears in `options`. A distance penalty of 0.0
         for the first option, or 1.0 if there is no matching option. If
@@ -500,7 +563,12 @@ class Distance(object):
             dist = 1.0
         self.add(key, dist)
 
-    def add_ratio(self, key, number1, number2):
+    def add_ratio(
+        self,
+        key: str,
+        number1: Union[int, float],
+        number2: Union[int, float],
+    ):
         """Adds a distance penalty for `number1` as a ratio of `number2`.
         `number1` is bound at 0 and `number2`.
         """
@@ -511,7 +579,7 @@ class Distance(object):
             dist = 0.0
         self.add(key, dist)
 
-    def add_string(self, key, str1, str2):
+    def add_string(self, key: str, str1: Optional[str], str2: Optional[str]):
         """Adds a distance penalty based on the edit distance between
         `str1` and `str2`.
         """
@@ -521,100 +589,118 @@ class Distance(object):
 
 # Structures that compose all the information for a candidate match.
 
-AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping',
-                                       'extra_items', 'extra_tracks'])
+AlbumMatch = namedtuple(
+    "AlbumMatch", ["distance", "info", "mapping", "extra_items", "extra_tracks"]
+)
 
-TrackMatch = namedtuple('TrackMatch', ['distance', 'info'])
+TrackMatch = namedtuple("TrackMatch", ["distance", "info"])
 
 
 # Aggregation of sources.
 
-def album_for_mbid(release_id):
+
+def album_for_mbid(release_id: str) -> Optional[AlbumInfo]:
     """Get an AlbumInfo object for a MusicBrainz release ID. Return None
     if the ID is not found.
     """
     try:
-        return mb.album_for_id(release_id)
+        album = mb.album_for_id(release_id)
+        if album:
+            plugins.send("albuminfo_received", info=album)
+        return album
     except mb.MusicBrainzAPIError as exc:
         exc.log(log)
+        return None
 
 
-def track_for_mbid(recording_id):
+def track_for_mbid(recording_id: str) -> Optional[TrackInfo]:
     """Get a TrackInfo object for a MusicBrainz recording ID. Return None
     if the ID is not found.
     """
     try:
-        return mb.track_for_id(recording_id)
+        track = mb.track_for_id(recording_id)
+        if track:
+            plugins.send("trackinfo_received", info=track)
+        return track
     except mb.MusicBrainzAPIError as exc:
         exc.log(log)
+        return None
 
 
-@plugins.notify_info_yielded(u'albuminfo_received')
-def albums_for_id(album_id):
+def albums_for_id(album_id: str) -> Iterable[AlbumInfo]:
     """Get a list of albums for an ID."""
     a = album_for_mbid(album_id)
     if a:
         yield a
     for a in plugins.album_for_id(album_id):
         if a:
+            plugins.send("albuminfo_received", info=a)
             yield a
 
 
-@plugins.notify_info_yielded(u'trackinfo_received')
-def tracks_for_id(track_id):
+def tracks_for_id(track_id: str) -> Iterable[TrackInfo]:
     """Get a list of tracks for an ID."""
     t = track_for_mbid(track_id)
     if t:
         yield t
     for t in plugins.track_for_id(track_id):
         if t:
+            plugins.send("trackinfo_received", info=t)
             yield t
 
 
-@plugins.notify_info_yielded(u'albuminfo_received')
-def album_candidates(items, artist, album, va_likely):
+def invoke_mb(call_func: Callable, *args):
+    try:
+        return call_func(*args)
+    except mb.MusicBrainzAPIError as exc:
+        exc.log(log)
+        return ()
+
+
+@plugins.notify_info_yielded("albuminfo_received")
+def album_candidates(
+    items: List[Item],
+    artist: str,
+    album: str,
+    va_likely: bool,
+    extra_tags: Dict,
+) -> Iterable[Tuple]:
     """Search for album matches. ``items`` is a list of Item objects
     that make up the album. ``artist`` and ``album`` are the respective
     names (strings), which may be derived from the item list or may be
     entered by the user. ``va_likely`` is a boolean indicating whether
-    the album is likely to be a "various artists" release.
+    the album is likely to be a "various artists" release. ``extra_tags``
+    is an optional dictionary of additional tags used to further
+    constrain the search.
     """
-    # Base candidates if we have album and artist to match.
-    if artist and album:
-        try:
-            for candidate in mb.match_album(artist, album, len(items)):
-                yield candidate
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
 
-    # Also add VA matches from MusicBrainz where appropriate.
-    if va_likely and album:
-        try:
-            for candidate in mb.match_album(None, album, len(items)):
-                yield candidate
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
+    if config["musicbrainz"]["enabled"]:
+        # Base candidates if we have album and artist to match.
+        if artist and album:
+            yield from invoke_mb(
+                mb.match_album, artist, album, len(items), extra_tags
+            )
+
+        # Also add VA matches from MusicBrainz where appropriate.
+        if va_likely and album:
+            yield from invoke_mb(
+                mb.match_album, None, album, len(items), extra_tags
+            )
 
     # Candidates from plugins.
-    for candidate in plugins.candidates(items, artist, album, va_likely):
-        yield candidate
+    yield from plugins.candidates(items, artist, album, va_likely, extra_tags)
 
 
-@plugins.notify_info_yielded(u'trackinfo_received')
-def item_candidates(item, artist, title):
+@plugins.notify_info_yielded("trackinfo_received")
+def item_candidates(item: Item, artist: str, title: str) -> Iterable[Tuple]:
     """Search for item matches. ``item`` is the Item to be matched.
     ``artist`` and ``title`` are strings and either reflect the item or
     are specified by the user.
     """
 
     # MusicBrainz candidates.
-    if artist and title:
-        try:
-            for candidate in mb.match_track(artist, title):
-                yield candidate
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
+    if config["musicbrainz"]["enabled"] and artist and title:
+        yield from invoke_mb(mb.match_track, artist, title)
 
     # Plugin candidates.
-    for candidate in plugins.item_candidates(item, artist, title):
-        yield candidate
+    yield from plugins.item_candidates(item, artist, title)

lib/beets/autotag/match.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -17,36 +16,56 @@
 releases and tracks.
 """
 
-from __future__ import division, absolute_import, print_function
 
 import datetime
 import re
-from munkres import Munkres
 from collections import namedtuple
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
 
-from beets import logging
-from beets import plugins
-from beets import config
+from munkres import Munkres
+
+from beets import config, logging, plugins
+from beets.autotag import (
+    AlbumInfo,
+    AlbumMatch,
+    Distance,
+    TrackInfo,
+    TrackMatch,
+    hooks,
+)
+from beets.library import Item
 from beets.util import plurality
-from beets.autotag import hooks
 from beets.util.enumeration import OrderedEnum
 
 # Artist signals that indicate "various artists". These are used at the
 # album level to determine whether a given release is likely a VA
 # release and also on the track level to to remove the penalty for
 # differing artists.
-VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown')
+VA_ARTISTS = ("", "various artists", "various", "va", "unknown")
 
 # Global logger.
-log = logging.getLogger('beets')
+log = logging.getLogger("beets")
 
 
 # Recommendation enumeration.
 
+
 class Recommendation(OrderedEnum):
     """Indicates a qualitative suggestion to the user about what should
     be done with a given match.
     """
+
     none = 0
     low = 1
     medium = 2
@@ -57,12 +76,15 @@ class Recommendation(OrderedEnum):
 # consists of a list of possible candidates (i.e., AlbumInfo or TrackInfo
 # objects) and a recommendation value.
 
-Proposal = namedtuple('Proposal', ('candidates', 'recommendation'))
+Proposal = namedtuple("Proposal", ("candidates", "recommendation"))
 
 
 # Primary matching functionality.
 
-def current_metadata(items):
+
+def current_metadata(
+    items: Iterable[Item],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """Extract the likely current metadata for an album given a list of its
     items. Return two dictionaries:
      - The most common value for each field.
@@ -72,22 +94,36 @@ def current_metadata(items):
 
     likelies = {}
     consensus = {}
-    fields = ['artist', 'album', 'albumartist', 'year', 'disctotal',
-              'mb_albumid', 'label', 'catalognum', 'country', 'media',
-              'albumdisambig']
+    fields = [
+        "artist",
+        "album",
+        "albumartist",
+        "year",
+        "disctotal",
+        "mb_albumid",
+        "label",
+        "barcode",
+        "catalognum",
+        "country",
+        "media",
+        "albumdisambig",
+    ]
     for field in fields:
         values = [item[field] for item in items if item]
         likelies[field], freq = plurality(values)
-        consensus[field] = (freq == len(values))
+        consensus[field] = freq == len(values)
 
     # If there's an album artist consensus, use this for the artist.
-    if consensus['albumartist'] and likelies['albumartist']:
-        likelies['artist'] = likelies['albumartist']
+    if consensus["albumartist"] and likelies["albumartist"]:
+        likelies["artist"] = likelies["albumartist"]
 
     return likelies, consensus
 
 
-def assign_items(items, tracks):
+def assign_items(
+    items: Sequence[Item],
+    tracks: Sequence[TrackInfo],
+) -> Tuple[Dict[Item, TrackInfo], List[Item], List[TrackInfo]]:
     """Given a list of Items and a list of TrackInfo objects, find the
     best mapping between them. Returns a mapping from Items to TrackInfo
     objects, a set of extra Items, and a set of extra TrackInfo
@@ -95,20 +131,20 @@ def assign_items(items, tracks):
     of objects of the two types.
     """
     # Construct the cost matrix.
-    costs = []
+    costs: List[List[Distance]] = []
     for item in items:
         row = []
-        for i, track in enumerate(tracks):
+        for track in tracks:
             row.append(track_distance(item, track))
         costs.append(row)
 
     # Find a minimum-cost bipartite matching.
-    log.debug('Computing track assignment...')
+    log.debug("Computing track assignment...")
     matching = Munkres().compute(costs)
-    log.debug('...done.')
+    log.debug("...done.")
 
     # Produce the output matching.
-    mapping = dict((items[i], tracks[j]) for (i, j) in matching)
+    mapping = {items[i]: tracks[j] for (i, j) in matching}
     extra_items = list(set(items) - set(mapping.keys()))
     extra_items.sort(key=lambda i: (i.disc, i.track, i.title))
     extra_tracks = list(set(tracks) - set(mapping.values()))
@@ -116,14 +152,18 @@ def assign_items(items, tracks):
     return mapping, extra_items, extra_tracks
 
 
-def track_index_changed(item, track_info):
+def track_index_changed(item: Item, track_info: TrackInfo) -> bool:
     """Returns True if the item and track info index is different. Tolerates
     per disc and per release numbering.
     """
     return item.track not in (track_info.medium_index, track_info.index)
 
 
-def track_distance(item, track_info, incl_artist=False):
+def track_distance(
+    item: Item,
+    track_info: TrackInfo,
+    incl_artist: bool = False,
+) -> Distance:
     """Determines the significance of a track metadata change. Returns a
     Distance object. `incl_artist` indicates that a distance component should
     be included for the track artist (i.e., for various-artist releases).
@@ -132,26 +172,37 @@ def track_distance(item, track_info, incl_artist=False):
 
     # Length.
     if track_info.length:
-        diff = abs(item.length - track_info.length) - \
-            config['match']['track_length_grace'].as_number()
-        dist.add_ratio('track_length', diff,
-                       config['match']['track_length_max'].as_number())
+        item_length = cast(float, item.length)
+        track_length_grace = cast(
+            Union[float, int],
+            config["match"]["track_length_grace"].as_number(),
+        )
+        track_length_max = cast(
+            Union[float, int],
+            config["match"]["track_length_max"].as_number(),
+        )
+
+        diff = abs(item_length - track_info.length) - track_length_grace
+        dist.add_ratio("track_length", diff, track_length_max)
 
     # Title.
-    dist.add_string('track_title', item.title, track_info.title)
+    dist.add_string("track_title", item.title, track_info.title)
 
     # Artist. Only check if there is actually an artist in the track data.
-    if incl_artist and track_info.artist and \
-            item.artist.lower() not in VA_ARTISTS:
-        dist.add_string('track_artist', item.artist, track_info.artist)
+    if (
+        incl_artist
+        and track_info.artist
+        and item.artist.lower() not in VA_ARTISTS
+    ):
+        dist.add_string("track_artist", item.artist, track_info.artist)
 
     # Track index.
     if track_info.index and item.track:
-        dist.add_expr('track_index', track_index_changed(item, track_info))
+        dist.add_expr("track_index", track_index_changed(item, track_info))
 
     # Track ID.
     if item.mb_trackid:
-        dist.add_expr('track_id', item.mb_trackid != track_info.track_id)
+        dist.add_expr("track_id", item.mb_trackid != track_info.track_id)
 
     # Plugins.
     dist.update(plugins.track_distance(item, track_info))
@@ -159,7 +210,11 @@ def track_distance(item, track_info, incl_artist=False):
     return dist
 
 
-def distance(items, album_info, mapping):
+def distance(
+    items: Sequence[Item],
+    album_info: AlbumInfo,
+    mapping: Dict[Item, TrackInfo],
+) -> Distance:
     """Determines how "significant" an album metadata change would be.
     Returns a Distance object. `album_info` is an AlbumInfo object
     reflecting the album to be compared. `items` is a sequence of all
@@ -174,90 +229,96 @@ def distance(items, album_info, mapping):
 
     # Artist, if not various.
     if not album_info.va:
-        dist.add_string('artist', likelies['artist'], album_info.artist)
+        dist.add_string("artist", likelies["artist"], album_info.artist)
 
     # Album.
-    dist.add_string('album', likelies['album'], album_info.album)
+    dist.add_string("album", likelies["album"], album_info.album)
 
     # Current or preferred media.
     if album_info.media:
         # Preferred media options.
-        patterns = config['match']['preferred']['media'].as_str_seq()
-        options = [re.compile(r'(\d+x)?(%s)' % pat, re.I) for pat in patterns]
+        patterns = config["match"]["preferred"]["media"].as_str_seq()
+        patterns = cast(Sequence[str], patterns)
+        options = [re.compile(r"(\d+x)?(%s)" % pat, re.I) for pat in patterns]
         if options:
-            dist.add_priority('media', album_info.media, options)
+            dist.add_priority("media", album_info.media, options)
         # Current media.
-        elif likelies['media']:
-            dist.add_equality('media', album_info.media, likelies['media'])
+        elif likelies["media"]:
+            dist.add_equality("media", album_info.media, likelies["media"])
 
     # Mediums.
-    if likelies['disctotal'] and album_info.mediums:
-        dist.add_number('mediums', likelies['disctotal'], album_info.mediums)
+    if likelies["disctotal"] and album_info.mediums:
+        dist.add_number("mediums", likelies["disctotal"], album_info.mediums)
 
     # Prefer earliest release.
-    if album_info.year and config['match']['preferred']['original_year']:
+    if album_info.year and config["match"]["preferred"]["original_year"]:
         # Assume 1889 (earliest first gramophone discs) if we don't know the
         # original year.
         original = album_info.original_year or 1889
         diff = abs(album_info.year - original)
         diff_max = abs(datetime.date.today().year - original)
-        dist.add_ratio('year', diff, diff_max)
+        dist.add_ratio("year", diff, diff_max)
     # Year.
-    elif likelies['year'] and album_info.year:
-        if likelies['year'] in (album_info.year, album_info.original_year):
+    elif likelies["year"] and album_info.year:
+        if likelies["year"] in (album_info.year, album_info.original_year):
             # No penalty for matching release or original year.
-            dist.add('year', 0.0)
+            dist.add("year", 0.0)
         elif album_info.original_year:
             # Prefer matchest closest to the release year.
-            diff = abs(likelies['year'] - album_info.year)
-            diff_max = abs(datetime.date.today().year -
-                           album_info.original_year)
-            dist.add_ratio('year', diff, diff_max)
+            diff = abs(likelies["year"] - album_info.year)
+            diff_max = abs(
+                datetime.date.today().year - album_info.original_year
+            )
+            dist.add_ratio("year", diff, diff_max)
         else:
             # Full penalty when there is no original year.
-            dist.add('year', 1.0)
+            dist.add("year", 1.0)
 
     # Preferred countries.
-    patterns = config['match']['preferred']['countries'].as_str_seq()
+    patterns = config["match"]["preferred"]["countries"].as_str_seq()
+    patterns = cast(Sequence[str], patterns)
     options = [re.compile(pat, re.I) for pat in patterns]
     if album_info.country and options:
-        dist.add_priority('country', album_info.country, options)
+        dist.add_priority("country", album_info.country, options)
     # Country.
-    elif likelies['country'] and album_info.country:
-        dist.add_string('country', likelies['country'], album_info.country)
+    elif likelies["country"] and album_info.country:
+        dist.add_string("country", likelies["country"], album_info.country)
 
     # Label.
-    if likelies['label'] and album_info.label:
-        dist.add_string('label', likelies['label'], album_info.label)
+    if likelies["label"] and album_info.label:
+        dist.add_string("label", likelies["label"], album_info.label)
 
     # Catalog number.
-    if likelies['catalognum'] and album_info.catalognum:
-        dist.add_string('catalognum', likelies['catalognum'],
-                        album_info.catalognum)
+    if likelies["catalognum"] and album_info.catalognum:
+        dist.add_string(
+            "catalognum", likelies["catalognum"], album_info.catalognum
+        )
 
     # Disambiguation.
-    if likelies['albumdisambig'] and album_info.albumdisambig:
-        dist.add_string('albumdisambig', likelies['albumdisambig'],
-                        album_info.albumdisambig)
+    if likelies["albumdisambig"] and album_info.albumdisambig:
+        dist.add_string(
+            "albumdisambig", likelies["albumdisambig"], album_info.albumdisambig
+        )
 
     # Album ID.
-    if likelies['mb_albumid']:
-        dist.add_equality('album_id', likelies['mb_albumid'],
-                          album_info.album_id)
+    if likelies["mb_albumid"]:
+        dist.add_equality(
+            "album_id", likelies["mb_albumid"], album_info.album_id
+        )
 
     # Tracks.
     dist.tracks = {}
     for item, track in mapping.items():
         dist.tracks[track] = track_distance(item, track, album_info.va)
-        dist.add('tracks', dist.tracks[track].distance)
+        dist.add("tracks", dist.tracks[track].distance)
 
     # Missing tracks.
-    for i in range(len(album_info.tracks) - len(mapping)):
-        dist.add('missing_tracks', 1.0)
+    for _ in range(len(album_info.tracks) - len(mapping)):
+        dist.add("missing_tracks", 1.0)
 
     # Unmatched tracks.
-    for i in range(len(items) - len(mapping)):
-        dist.add('unmatched_tracks', 1.0)
+    for _ in range(len(items) - len(mapping)):
+        dist.add("unmatched_tracks", 1.0)
 
     # Plugins.
     dist.update(plugins.album_distance(items, album_info, mapping))
@@ -265,7 +326,7 @@ def distance(items, album_info, mapping):
     return dist
 
 
-def match_by_id(items):
+def match_by_id(items: Iterable[Item]):
     """If the items are tagged with a MusicBrainz album ID, returns an
     AlbumInfo object for the corresponding album. Otherwise, returns
     None.
@@ -276,20 +337,22 @@ def match_by_id(items):
     try:
         first = next(albumids)
     except StopIteration:
-        log.debug(u'No album ID found.')
+        log.debug("No album ID found.")
         return None
 
     # Is there a consensus on the MB album ID?
     for other in albumids:
         if other != first:
-            log.debug(u'No album ID consensus.')
+            log.debug("No album ID consensus.")
             return None
     # If all album IDs are equal, look up the album.
-    log.debug(u'Searching for discovered album ID: {0}', first)
+    log.debug("Searching for discovered album ID: {0}", first)
     return hooks.album_for_mbid(first)
 
 
-def _recommendation(results):
+def _recommendation(
+    results: Sequence[Union[AlbumMatch, TrackMatch]],
+) -> Recommendation:
     """Given a sorted list of AlbumMatch or TrackMatch objects, return a
     recommendation based on the results' distances.
 
@@ -303,17 +366,19 @@ def _recommendation(results):
 
     # Basic distance thresholding.
     min_dist = results[0].distance
-    if min_dist < config['match']['strong_rec_thresh'].as_number():
+    if min_dist < config["match"]["strong_rec_thresh"].as_number():
         # Strong recommendation level.
         rec = Recommendation.strong
-    elif min_dist <= config['match']['medium_rec_thresh'].as_number():
+    elif min_dist <= config["match"]["medium_rec_thresh"].as_number():
         # Medium recommendation level.
         rec = Recommendation.medium
     elif len(results) == 1:
         # Only a single candidate.
         rec = Recommendation.low
-    elif results[1].distance - min_dist >= \
-            config['match']['rec_gap_thresh'].as_number():
+    elif (
+        results[1].distance - min_dist
+        >= config["match"]["rec_gap_thresh"].as_number()
+    ):
         # Gap between first two candidates is large.
         rec = Recommendation.low
     else:
@@ -326,48 +391,60 @@ def _recommendation(results):
     if isinstance(results[0], hooks.AlbumMatch):
         for track_dist in min_dist.tracks.values():
             keys.update(list(track_dist.keys()))
-    max_rec_view = config['match']['max_rec']
+    max_rec_view = config["match"]["max_rec"]
     for key in keys:
         if key in list(max_rec_view.keys()):
-            max_rec = max_rec_view[key].as_choice({
-                'strong': Recommendation.strong,
-                'medium': Recommendation.medium,
-                'low': Recommendation.low,
-                'none': Recommendation.none,
-            })
+            max_rec = max_rec_view[key].as_choice(
+                {
+                    "strong": Recommendation.strong,
+                    "medium": Recommendation.medium,
+                    "low": Recommendation.low,
+                    "none": Recommendation.none,
+                }
+            )
             rec = min(rec, max_rec)
 
     return rec
 
 
-def _sort_candidates(candidates):
+AnyMatch = TypeVar("AnyMatch", TrackMatch, AlbumMatch)
+
+
+def _sort_candidates(candidates: Iterable[AnyMatch]) -> Sequence[AnyMatch]:
     """Sort candidates by distance."""
     return sorted(candidates, key=lambda match: match.distance)
 
 
-def _add_candidate(items, results, info):
+def _add_candidate(
+    items: Sequence[Item],
+    results: Dict[Any, AlbumMatch],
+    info: AlbumInfo,
+):
     """Given a candidate AlbumInfo object, attempt to add the candidate
     to the output dictionary of AlbumMatch objects. This involves
     checking the track count, ordering the items, checking for
     duplicates, and calculating the distance.
     """
-    log.debug(u'Candidate: {0} - {1} ({2})',
-              info.artist, info.album, info.album_id)
+    log.debug(
+        "Candidate: {0} - {1} ({2})", info.artist, info.album, info.album_id
+    )
 
     # Discard albums with zero tracks.
     if not info.tracks:
-        log.debug(u'No tracks.')
+        log.debug("No tracks.")
         return
 
-    # Don't duplicate.
-    if info.album_id in results:
-        log.debug(u'Duplicate.')
+    # Prevent duplicates.
+    if info.album_id and info.album_id in results:
+        log.debug("Duplicate.")
         return
 
     # Discard matches without required tags.
-    for req_tag in config['match']['required'].as_str_seq():
+    for req_tag in cast(
+        Sequence[str], config["match"]["required"].as_str_seq()
+    ):
         if getattr(info, req_tag) is None:
-            log.debug(u'Ignored. Missing required tag: {0}', req_tag)
+            log.debug("Ignored. Missing required tag: {0}", req_tag)
             return
 
     # Find mapping between the items and the track info.
@@ -378,18 +455,24 @@ def _add_candidate(items, results, info):
 
     # Skip matches with ignored penalties.
     penalties = [key for key, _ in dist]
-    for penalty in config['match']['ignored'].as_str_seq():
+    ignored = cast(Sequence[str], config["match"]["ignored"].as_str_seq())
+    for penalty in ignored:
         if penalty in penalties:
-            log.debug(u'Ignored. Penalty: {0}', penalty)
+            log.debug("Ignored. Penalty: {0}", penalty)
             return
 
-    log.debug(u'Success. Distance: {0}', dist)
-    results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
-                                              extra_items, extra_tracks)
+    log.debug("Success. Distance: {0}", dist)
+    results[info.album_id] = hooks.AlbumMatch(
+        dist, info, mapping, extra_items, extra_tracks
+    )
 
 
-def tag_album(items, search_artist=None, search_album=None,
-              search_ids=[]):
+def tag_album(
+    items,
+    search_artist: Optional[str] = None,
+    search_album: Optional[str] = None,
+    search_ids: List[str] = [],
+) -> Tuple[str, str, Proposal]:
     """Return a tuple of the current artist name, the current album
     name, and a `Proposal` containing `AlbumMatch` candidates.
 
@@ -409,20 +492,19 @@ def tag_album(items, search_artist=None, search_album=None,
     """
     # Get current metadata.
     likelies, consensus = current_metadata(items)
-    cur_artist = likelies['artist']
-    cur_album = likelies['album']
-    log.debug(u'Tagging {0} - {1}', cur_artist, cur_album)
+    cur_artist = cast(str, likelies["artist"])
+    cur_album = cast(str, likelies["album"])
+    log.debug("Tagging {0} - {1}", cur_artist, cur_album)
 
-    # The output result (distance, AlbumInfo) tuples (keyed by MB album
-    # ID).
-    candidates = {}
+    # The output result, keys are the MB album ID.
+    candidates: Dict[Any, AlbumMatch] = {}
 
     # Search by explicit ID.
     if search_ids:
         for search_id in search_ids:
-            log.debug(u'Searching for album ID: {0}', search_id)
-            for id_candidate in hooks.albums_for_id(search_id):
-                _add_candidate(items, candidates, id_candidate)
+            log.debug("Searching for album ID: {0}", search_id)
+            for album_info_for_id in hooks.albums_for_id(search_id):
+                _add_candidate(items, candidates, album_info_for_id)
 
     # Use existing metadata or text search.
     else:
@@ -431,44 +513,58 @@ def tag_album(items, search_artist=None, search_album=None,
         if id_info:
             _add_candidate(items, candidates, id_info)
             rec = _recommendation(list(candidates.values()))
-            log.debug(u'Album ID match recommendation is {0}', rec)
-            if candidates and not config['import']['timid']:
+            log.debug("Album ID match recommendation is {0}", rec)
+            if candidates and not config["import"]["timid"]:
                 # If we have a very good MBID match, return immediately.
                 # Otherwise, this match will compete against metadata-based
                 # matches.
                 if rec == Recommendation.strong:
-                    log.debug(u'ID match.')
-                    return cur_artist, cur_album, \
-                        Proposal(list(candidates.values()), rec)
+                    log.debug("ID match.")
+                    return (
+                        cur_artist,
+                        cur_album,
+                        Proposal(list(candidates.values()), rec),
+                    )
 
         # Search terms.
         if not (search_artist and search_album):
             # No explicit search terms -- use current metadata.
             search_artist, search_album = cur_artist, cur_album
-        log.debug(u'Search terms: {0} - {1}', search_artist, search_album)
+        log.debug("Search terms: {0} - {1}", search_artist, search_album)
+
+        extra_tags = None
+        if config["musicbrainz"]["extra_tags"]:
+            tag_list = config["musicbrainz"]["extra_tags"].get()
+            extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list}
+            log.debug("Additional search terms: {0}", extra_tags)
 
         # Is this album likely to be a "various artist" release?
-        va_likely = ((not consensus['artist']) or
-                     (search_artist.lower() in VA_ARTISTS) or
-                     any(item.comp for item in items))
-        log.debug(u'Album might be VA: {0}', va_likely)
+        va_likely = (
+            (not consensus["artist"])
+            or (search_artist.lower() in VA_ARTISTS)
+            or any(item.comp for item in items)
+        )
+        log.debug("Album might be VA: {0}", va_likely)
 
         # Get the results from the data sources.
-        for matched_candidate in hooks.album_candidates(items,
-                                                        search_artist,
-                                                        search_album,
-                                                        va_likely):
+        for matched_candidate in hooks.album_candidates(
+            items, search_artist, search_album, va_likely, extra_tags
+        ):
             _add_candidate(items, candidates, matched_candidate)
 
-    log.debug(u'Evaluating {0} candidates.', len(candidates))
+    log.debug("Evaluating {0} candidates.", len(candidates))
     # Sort and get the recommendation.
-    candidates = _sort_candidates(candidates.values())
-    rec = _recommendation(candidates)
-    return cur_artist, cur_album, Proposal(candidates, rec)
+    candidates_sorted = _sort_candidates(candidates.values())
+    rec = _recommendation(candidates_sorted)
+    return cur_artist, cur_album, Proposal(candidates_sorted, rec)
 
 
-def tag_item(item, search_artist=None, search_title=None,
-             search_ids=[]):
+def tag_item(
+    item,
+    search_artist: Optional[str] = None,
+    search_title: Optional[str] = None,
+    search_ids: Optional[List[str]] = None,
+) -> Proposal:
     """Find metadata for a single track. Return a `Proposal` consisting
     of `TrackMatch` objects.
 
@@ -480,26 +576,31 @@ def tag_item(item, search_artist=None, search_title=None,
     # Holds candidates found so far: keys are MBIDs; values are
     # (distance, TrackInfo) pairs.
     candidates = {}
+    rec: Optional[Recommendation] = None
 
     # First, try matching by MusicBrainz ID.
     trackids = search_ids or [t for t in [item.mb_trackid] if t]
     if trackids:
         for trackid in trackids:
-            log.debug(u'Searching for track ID: {0}', trackid)
+            log.debug("Searching for track ID: {0}", trackid)
             for track_info in hooks.tracks_for_id(trackid):
                 dist = track_distance(item, track_info, incl_artist=True)
-                candidates[track_info.track_id] = \
-                    hooks.TrackMatch(dist, track_info)
+                candidates[track_info.track_id] = hooks.TrackMatch(
+                    dist, track_info
+                )
                 # If this is a good match, then don't keep searching.
                 rec = _recommendation(_sort_candidates(candidates.values()))
-                if rec == Recommendation.strong and \
-                        not config['import']['timid']:
-                    log.debug(u'Track ID match.')
+                if (
+                    rec == Recommendation.strong
+                    and not config["import"]["timid"]
+                ):
+                    log.debug("Track ID match.")
                     return Proposal(_sort_candidates(candidates.values()), rec)
 
     # If we're searching by ID, don't proceed.
     if search_ids:
         if candidates:
+            assert rec is not None
             return Proposal(_sort_candidates(candidates.values()), rec)
         else:
             return Proposal([], Recommendation.none)
@@ -507,7 +608,7 @@ def tag_item(item, search_artist=None, search_title=None,
     # Search terms.
     if not (search_artist and search_title):
         search_artist, search_title = item.artist, item.title
-    log.debug(u'Item search terms: {0} - {1}', search_artist, search_title)
+    log.debug("Item search terms: {0} - {1}", search_artist, search_title)
 
     # Get and evaluate candidate metadata.
     for track_info in hooks.item_candidates(item, search_artist, search_title):
@@ -515,7 +616,7 @@ def tag_item(item, search_artist=None, search_title=None,
         candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
 
     # Sort by distance and return with recommendation.
-    log.debug(u'Found {0} candidates.', len(candidates))
-    candidates = _sort_candidates(candidates.values())
-    rec = _recommendation(candidates)
-    return Proposal(candidates, rec)
+    log.debug("Found {0} candidates.", len(candidates))
+    candidates_sorted = _sort_candidates(candidates.values())
+    rec = _recommendation(candidates_sorted)
+    return Proposal(candidates_sorted, rec)

lib/beets/config_default.yaml

@@ -1,71 +1,153 @@
+# --------------- Main ---------------
+
 library: library.db
 directory: ~/Music
+statefile: state.pickle
 
-import:
-    write: yes
-    copy: yes
-    move: no
-    link: no
-    hardlink: no
-    delete: no
-    resume: ask
-    incremental: no
-    quiet_fallback: skip
-    none_rec_action: ask
-    timid: no
-    log:
-    autotag: yes
-    quiet: no
-    singletons: no
-    default_action: apply
-    languages: []
-    detail: no
-    flat: no
-    group_albums: no
-    pretend: false
-    search_ids: []
-    duplicate_action: ask
+# --------------- Plugins ---------------
+
+plugins: []
+pluginpath: []
+
+# --------------- Import ---------------
 
 clutter: ["Thumbs.DB", ".DS_Store"]
 ignore: [".*", "*~", "System Volume Information", "lost+found"]
 ignore_hidden: yes
 
-replace:
-    '[\\/]': _
-    '^\.': _
-    '[\x00-\x1f]': _
-    '[<>:"\?\*\|]': _
-    '\.$': _
-    '\s+$': ''
-    '^\s+': ''
+import:
+    # common options
+    write: yes
+    copy: yes
+    move: no
+    timid: no
+    quiet: no
+    log:
+    # other options
+    default_action: apply
+    languages: []
+    quiet_fallback: skip
+    none_rec_action: ask
+    # rare options
+    link: no
+    hardlink: no
+    reflink: no
+    delete: no
+    resume: ask
+    incremental: no
+    incremental_skip_later: no
+    from_scratch: no
+    autotag: yes
+    singletons: no
+    detail: no
+    flat: no
+    group_albums: no
+    pretend: false
+    search_ids: []
+    duplicate_keys:
+        album: albumartist album
+        item: artist title
+    duplicate_action: ask
+    duplicate_verbose_prompt: no
+    bell: no
+    set_fields: {}
+    ignored_alias_types: []
+    singleton_album_disambig: yes
+
+# --------------- Paths ---------------
+
 path_sep_replace: _
+drive_sep_replace: _
 asciify_paths: false
 art_filename: cover
 max_filename_length: 0
+replace:
+    # Replace bad characters with _
+    # prohibited in many filesystem paths
+    '[<>:\?\*\|]': _
+    # double quotation mark "
+    '\"': _
+    # path separators: \ or /
+    '[\\/]': _
+    # starting and closing periods
+    '^\.': _
+    '\.$': _
+    # control characters
+    '[\x00-\x1f]': _
+    # dash at the start of a filename (causes command line ambiguity)
+    '^-': _
+    # Replace bad characters with nothing
+    # starting and closing whitespace
+    '\s+$': ''
+    '^\s+': ''
+
+aunique:
+    keys: albumartist album
+    disambiguators: albumtype year label catalognum albumdisambig releasegroupdisambig
+    bracket: '[]'
+
+sunique:
+    keys: artist title
+    disambiguators: year trackdisambig
+    bracket: '[]'
+
+# --------------- Tagging ---------------
 
-plugins: []
-pluginpath: []
-threaded: yes
-timeout: 5.0
 per_disc_numbering: no
-verbose: 0
-terminal_encoding:
 original_date: no
+artist_credit: no
 id3v23: no
 va_name: "Various Artists"
+paths:
+    default: $albumartist/$album%aunique{}/$track $title
+    singleton: Non-Album/$artist/$title
+    comp: Compilations/$album%aunique{}/$track $title
+
+# --------------- Performance ---------------
+
+threaded: yes
+timeout: 5.0
+
+# --------------- UI ---------------
+
+verbose: 0
+terminal_encoding:
 
 ui:
     terminal_width: 80
     length_diff_thresh: 10.0
     color: yes
     colors:
-        text_success: green
-        text_warning: yellow
-        text_error: red
-        text_highlight: red
-        text_highlight_minor: lightgray
-        action_default: turquoise
-        action: blue
+        text_success: ['bold', 'green']
+        text_warning: ['bold', 'yellow']
+        text_error: ['bold', 'red']
+        text_highlight: ['bold', 'red']
+        text_highlight_minor: ['white']
+        action_default: ['bold', 'cyan']
+        action: ['bold', 'cyan']
+        # New Colors
+        text: ['normal']
+        text_faint: ['faint']
+        import_path: ['bold', 'blue']
+        import_path_items: ['bold', 'blue']
+        added:   ['green']
+        removed: ['red']
+        changed: ['yellow']
+        added_highlight:   ['bold', 'green']
+        removed_highlight: ['bold', 'red']
+        changed_highlight: ['bold', 'yellow']
+        text_diff_added:   ['bold', 'red']
+        text_diff_removed: ['bold', 'red']
+        text_diff_changed: ['bold', 'red']
+        action_description: ['white']
+    import:
+        indentation:
+            match_header: 2
+            match_details: 2
+            match_tracklist: 5
+        layout: column
+
+# --------------- Search ---------------
 
 format_item: $artist - $album - $title
 format_album: $albumartist - $album
@@ -76,18 +158,27 @@ sort_album: albumartist+ album+
 sort_item: artist+ album+ disc+ track+
 sort_case_insensitive: yes
 
-paths:
-    default: $albumartist/$album%aunique{}/$track $title
-    singleton: Non-Album/$artist/$title
-    comp: Compilations/$album%aunique{}/$track $title
-
-statefile: state.pickle
+# --------------- Autotagger ---------------
 
+overwrite_null:
+  album: []
+  track: []
 musicbrainz:
+    enabled: yes
     host: musicbrainz.org
+    https: no
     ratelimit: 1
     ratelimit_interval: 1.0
     searchlimit: 5
+    extra_tags: []
+    genres: no
+    external_ids:
+        discogs: no
+        bandcamp: no
+        spotify: no
+        deezer: no
+        beatport: no
+        tidal: no
 
 match:
     strong_rec_thresh: 0.04
@@ -122,5 +213,10 @@ match:
         original_year: no
     ignored: []
     required: []
+    ignored_media: []
+    ignore_data_tracks: yes
+    ignore_video_tracks: yes
     track_length_grace: 10
     track_length_max: 30
+    album_disambig_fields: data_source media year country label catalognum albumdisambig
+    singleton_disambig_fields: data_source index track_alt album

lib/beets/dbcore/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -16,14 +15,21 @@
 """DBCore is an abstract database package that forms the basis for beets'
 Library.
 """
-from __future__ import division, absolute_import, print_function
 
-from .db import Model, Database
-from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery
+from .db import Database, Model, Results
+from .query import (
+    AndQuery,
+    FieldQuery,
+    InvalidQueryError,
+    MatchQuery,
+    OrQuery,
+    Query,
+)
+from .queryparse import (
+    parse_sorted_query,
+    query_from_strings,
+    sort_from_strings,
+)
 from .types import Type
-from .queryparse import query_from_strings
-from .queryparse import sort_from_strings
-from .queryparse import parse_sorted_query
-from .query import InvalidQueryError
 
 # flake8: noqa

lib/beets/dbcore/queryparse.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -13,32 +12,33 @@
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 
-"""Parsing of strings into DBCore queries.
-"""
-from __future__ import division, absolute_import, print_function
+"""Parsing of strings into DBCore queries."""
 
-import re
 import itertools
-from . import query
-import beets
+import re
+from typing import Collection, Dict, List, Optional, Sequence, Tuple, Type
+
+from . import Model, query
+from .query import Sort
 
 PARSE_QUERY_PART_REGEX = re.compile(
     # Non-capturing optional segment for the keyword.
-    r'(-|\^)?'   # Negation prefixes.
-
-    r'(?:'
-    r'(\S+?)'    # The field key.
-    r'(?<!\\):'  # Unescaped :
-    r')?'
-
-    r'(.*)',         # The term itself.
-
-    re.I  # Case-insensitive.
+    r"(-|\^)?"  # Negation prefixes.
+    r"(?:"
+    r"(\S+?)"  # The field key.
+    r"(?<!\\):"  # Unescaped :
+    r")?"
+    r"(.*)",  # The term itself.
+    re.I,  # Case-insensitive.
 )
 
 
-def parse_query_part(part, query_classes={}, prefixes={},
-                     default_class=query.SubstringQuery):
+def parse_query_part(
+    part: str,
+    query_classes: Dict[str, Type[query.FieldQuery]] = {},
+    prefixes: Dict = {},
+    default_class: Type[query.SubstringQuery] = query.SubstringQuery,
+) -> Tuple[Optional[str], str, Type[query.FieldQuery], bool]:
     """Parse a single *query part*, which is a chunk of a complete query
     string representing a single criterion.
 
@@ -89,13 +89,13 @@ def parse_query_part(part, query_classes={}, prefixes={},
     assert match  # Regex should always match
     negate = bool(match.group(1))
     key = match.group(2)
-    term = match.group(3).replace('\:', ':')
+    term = match.group(3).replace("\\:", ":")
 
     # Check whether there's a prefix in the query and use the
     # corresponding query type.
     for pre, query_class in prefixes.items():
         if term.startswith(pre):
-            return key, term[len(pre):], query_class, negate
+            return key, term[len(pre) :], query_class, negate
 
     # No matching prefix, so use either the query class determined by
     # the field or the default as a fallback.
@@ -103,7 +103,11 @@ def parse_query_part(part, query_classes={}, prefixes={},
     return key, term, query_class, negate
 
 
-def construct_query_part(model_cls, prefixes, query_part):
+def construct_query_part(
+    model_cls: Type[Model],
+    prefixes: Dict,
+    query_part: str,
+) -> query.Query:
     """Parse a *query part* string and return a :class:`Query` object.
 
     :param model_cls: The :class:`Model` class that this is a query for.
@@ -119,47 +123,59 @@ def construct_query_part(model_cls, prefixes, query_part):
     if not query_part:
         return query.TrueQuery()
 
-    # Use `model_cls` to build up a map from field names to `Query`
-    # classes.
-    query_classes = {}
-    for k, t in itertools.chain(model_cls._fields.items(),
-                                model_cls._types.items()):
+    out_query: query.Query
+
+    # Use `model_cls` to build up a map from field (or query) names to
+    # `Query` classes.
+    query_classes: Dict[str, Type[query.FieldQuery]] = {}
+    for k, t in itertools.chain(
+        model_cls._fields.items(), model_cls._types.items()
+    ):
         query_classes[k] = t.query
+    query_classes.update(model_cls._queries)  # Non-field queries.
 
     # Parse the string.
-    key, pattern, query_class, negate = \
-        parse_query_part(query_part, query_classes, prefixes)
+    key, pattern, query_class, negate = parse_query_part(
+        query_part, query_classes, prefixes
+    )
 
     # If there's no key (field name) specified, this is a "match
     # anything" query.
     if key is None:
-        if issubclass(query_class, query.FieldQuery):
-            # The query type matches a specific field, but none was
-            # specified. So we use a version of the query that matches
-            # any field.
-            q = query.AnyFieldQuery(pattern, model_cls._search_fields,
-                                    query_class)
-            if negate:
-                return query.NotQuery(q)
-            else:
-                return q
-        else:
-            # Non-field query type.
-            if negate:
-                return query.NotQuery(query_class(pattern))
-            else:
-                return query_class(pattern)
+        # The query type matches a specific field, but none was
+        # specified. So we use a version of the query that matches
+        # any field.
+        out_query = query.AnyFieldQuery(
+            pattern, model_cls._search_fields, query_class
+        )
 
-    # Otherwise, this must be a `FieldQuery`. Use the field name to
-    # construct the query object.
-    key = key.lower()
-    q = query_class(key.lower(), pattern, key in model_cls._fields)
+    # Field queries get constructed according to the name of the field
+    # they are querying.
+    else:
+        field = table = key.lower()
+        if field in model_cls.shared_db_fields:
+            # This field exists in both tables, so SQLite will encounter
+            # an OperationalError if we try to query it in a join.
+            # Using an explicit table name resolves this.
+            table = f"{model_cls._table}.{field}"
+
+        field_in_db = field in model_cls.all_db_fields
+        out_query = query_class(table, pattern, field_in_db)
+
+    # Apply negation.
     if negate:
-        return query.NotQuery(q)
-    return q
+        return query.NotQuery(out_query)
+    else:
+        return out_query
 
 
-def query_from_strings(query_cls, model_cls, prefixes, query_parts):
+# TYPING ERROR
+def query_from_strings(
+    query_cls: Type[query.CollectionQuery],
+    model_cls: Type[Model],
+    prefixes: Dict,
+    query_parts: Collection[str],
+) -> query.Query:
     """Creates a collection query of type `query_cls` from a list of
     strings in the format used by parse_query_part. `model_cls`
     determines how queries are constructed from strings.
@@ -172,23 +188,29 @@ def query_from_strings(query_cls, model_cls, prefixes, query_parts):
     return query_cls(subqueries)
 
 
-def construct_sort_part(model_cls, part):
+def construct_sort_part(
+    model_cls: Type[Model],
+    part: str,
+    case_insensitive: bool = True,
+) -> Sort:
     """Create a `Sort` from a single string criterion.
 
     `model_cls` is the `Model` being queried. `part` is a single string
-    ending in ``+`` or ``-`` indicating the sort.
+    ending in ``+`` or ``-`` indicating the sort. `case_insensitive`
+    indicates whether or not the sort should be performed in a case
+    sensitive manner.
     """
     assert part, "part must be a field name and + or -"
     field = part[:-1]
     assert field, "field is missing"
     direction = part[-1]
-    assert direction in ('+', '-'), "part must end with + or -"
-    is_ascending = direction == '+'
+    assert direction in ("+", "-"), "part must end with + or -"
+    is_ascending = direction == "+"
 
-    case_insensitive = beets.config['sort_case_insensitive'].get(bool)
     if field in model_cls._sorts:
-        sort = model_cls._sorts[field](model_cls, is_ascending,
-                                       case_insensitive)
+        sort = model_cls._sorts[field](
+            model_cls, is_ascending, case_insensitive
+        )
     elif field in model_cls._fields:
         sort = query.FixedFieldSort(field, is_ascending, case_insensitive)
     else:
@@ -197,21 +219,31 @@ def construct_sort_part(model_cls, part):
     return sort
 
 
-def sort_from_strings(model_cls, sort_parts):
-    """Create a `Sort` from a list of sort criteria (strings).
-    """
+def sort_from_strings(
+    model_cls: Type[Model],
+    sort_parts: Sequence[str],
+    case_insensitive: bool = True,
+) -> Sort:
+    """Create a `Sort` from a list of sort criteria (strings)."""
     if not sort_parts:
-        sort = query.NullSort()
+        return query.NullSort()
     elif len(sort_parts) == 1:
-        sort = construct_sort_part(model_cls, sort_parts[0])
+        return construct_sort_part(model_cls, sort_parts[0], case_insensitive)
     else:
         sort = query.MultipleSort()
         for part in sort_parts:
-            sort.add_sort(construct_sort_part(model_cls, part))
-    return sort
+            sort.add_sort(
+                construct_sort_part(model_cls, part, case_insensitive)
+            )
+        return sort
 
 
-def parse_sorted_query(model_cls, parts, prefixes={}):
+def parse_sorted_query(
+    model_cls: Type[Model],
+    parts: List[str],
+    prefixes: Dict = {},
+    case_insensitive: bool = True,
+) -> Tuple[query.Query, Sort]:
     """Given a list of strings, create the `Query` and `Sort` that they
     represent.
     """
@@ -222,29 +254,29 @@ def parse_sorted_query(model_cls, parts, prefixes={}):
     # Split up query in to comma-separated subqueries, each representing
     # an AndQuery, which need to be joined together in one OrQuery
     subquery_parts = []
-    for part in parts + [u',']:
-        if part.endswith(u','):
+    for part in parts + [","]:
+        if part.endswith(","):
             # Ensure we can catch "foo, bar" as well as "foo , bar"
             last_subquery_part = part[:-1]
             if last_subquery_part:
                 subquery_parts.append(last_subquery_part)
             # Parse the subquery in to a single AndQuery
             # TODO: Avoid needlessly wrapping AndQueries containing 1 subquery?
-            query_parts.append(query_from_strings(
-                query.AndQuery, model_cls, prefixes, subquery_parts
-            ))
+            query_parts.append(
+                query_from_strings(
+                    query.AndQuery, model_cls, prefixes, subquery_parts
+                )
+            )
             del subquery_parts[:]
         else:
             # Sort parts (1) end in + or -, (2) don't have a field, and
             # (3) consist of more than just the + or -.
-            if part.endswith((u'+', u'-')) \
-                    and u':' not in part \
-                    and len(part) > 1:
+            if part.endswith(("+", "-")) and ":" not in part and len(part) > 1:
                 sort_parts.append(part)
             else:
                 subquery_parts.append(part)
 
     # Avoid needlessly wrapping single statements in an OR
     q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0]
-    s = sort_from_strings(model_cls, sort_parts)
+    s = sort_from_strings(model_cls, sort_parts, case_insensitive)
     return q, s

lib/beets/dbcore/types.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -15,33 +14,46 @@
 
 """Representation of type information for DBCore model fields.
 """
-from __future__ import division, absolute_import, print_function
+import typing
+from abc import ABC
+from typing import Any, Generic, List, TypeVar, Union, cast
 
-from . import query
 from beets.util import str2bool
-import six
 
-if not six.PY2:
-    buffer = memoryview  # sqlite won't accept memoryview in python 2
+from .query import BooleanQuery, FieldQuery, NumericQuery, SubstringQuery
 
 
-# Abstract base.
+class ModelType(typing.Protocol):
+    """Protocol that specifies the required constructor for model types,
+    i.e. a function that takes any argument and attempts to parse it to the
+    given type.
+    """
 
-class Type(object):
+    def __init__(self, value: Any = None): ...
+
+
+# Generic type variables, used for the value type T and null type N (if
+# nullable, else T and N are set to the same type for the concrete subclasses
+# of Type).
+N = TypeVar("N")
+T = TypeVar("T", bound=ModelType)
+
+
+class Type(ABC, Generic[T, N]):
     """An object encapsulating the type of a model field. Includes
     information about how to store, query, format, and parse a given
     field.
     """
 
-    sql = u'TEXT'
+    sql: str = "TEXT"
     """The SQLite column type for the value.
     """
 
-    query = query.SubstringQuery
+    query: typing.Type[FieldQuery] = SubstringQuery
     """The `Query` subclass to be used when querying the field.
     """
 
-    model_type = six.text_type
+    model_type: typing.Type[T]
     """The Python type that is used to represent the value in the model.
 
     The model is guaranteed to return a value of this type if the field
@@ -50,12 +62,14 @@ class Type(object):
     """
 
     @property
-    def null(self):
-        """The value to be exposed when the underlying value is None.
-        """
-        return self.model_type()
+    def null(self) -> N:
+        """The value to be exposed when the underlying value is None."""
+        # Note that this default implementation only makes sense for T = N.
+        # It would be better to implement `null()` only in subclasses, or
+        # have a field null_type similar to `model_type` and use that here.
+        return cast(N, self.model_type())
 
-    def format(self, value):
+    def format(self, value: Union[N, T]) -> str:
         """Given a value of this type, produce a Unicode string
         representing the value. This is used in template evaluation.
         """
@@ -63,13 +77,13 @@ class Type(object):
             value = self.null
         # `self.null` might be `None`
         if value is None:
-            value = u''
-        if isinstance(value, bytes):
-            value = value.decode('utf-8', 'ignore')
+            return ""
+        elif isinstance(value, bytes):
+            return value.decode("utf-8", "ignore")
+        else:
+            return str(value)
 
-        return six.text_type(value)
-
-    def parse(self, string):
+    def parse(self, string: str) -> Union[T, N]:
         """Parse a (possibly human-written) string and return the
         indicated value of this type.
         """
@@ -78,40 +92,44 @@ class Type(object):
         except ValueError:
             return self.null
 
-    def normalize(self, value):
+    def normalize(self, value: Any) -> Union[T, N]:
         """Given a value that will be assigned into a field of this
         type, normalize the value to have the appropriate type. This
         base implementation only reinterprets `None`.
         """
+        # TYPING ERROR
         if value is None:
             return self.null
         else:
             # TODO This should eventually be replaced by
             # `self.model_type(value)`
-            return value
+            return cast(T, value)
 
-    def from_sql(self, sql_value):
+    def from_sql(
+        self,
+        sql_value: Union[None, int, float, str, bytes],
+    ) -> Union[T, N]:
         """Receives the value stored in the SQL backend and return the
         value to be stored in the model.
 
         For fixed fields the type of `value` is determined by the column
         type affinity given in the `sql` property and the SQL to Python
         mapping of the database adapter. For more information see:
-        http://www.sqlite.org/datatype3.html
+        https://www.sqlite.org/datatype3.html
         https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types
 
         Flexible fields have the type affinity `TEXT`. This means the
-        `sql_value` is either a `buffer`/`memoryview` or a `unicode` object`
+        `sql_value` is either a `memoryview` or a `unicode` object`
         and the method must handle these in addition.
         """
-        if isinstance(sql_value, buffer):
-            sql_value = bytes(sql_value).decode('utf-8', 'ignore')
-        if isinstance(sql_value, six.text_type):
+        if isinstance(sql_value, memoryview):
+            sql_value = bytes(sql_value).decode("utf-8", "ignore")
+        if isinstance(sql_value, str):
             return self.parse(sql_value)
         else:
             return self.normalize(sql_value)
 
-    def to_sql(self, model_value):
+    def to_sql(self, model_value: Any) -> Union[None, int, float, str, bytes]:
         """Convert a value as stored in the model object to a value used
         by the database adapter.
         """
@@ -120,87 +138,178 @@ class Type(object):
 
 # Reusable types.
 
-class Default(Type):
-    null = None
+
+class Default(Type[str, None]):
+    model_type = str
+
+    @property
+    def null(self):
+        return None
 
 
-class Integer(Type):
-    """A basic integer type.
-    """
-    sql = u'INTEGER'
-    query = query.NumericQuery
+class BaseInteger(Type[int, N]):
+    """A basic integer type."""
+
+    sql = "INTEGER"
+    query = NumericQuery
     model_type = int
 
+    def normalize(self, value: Any) -> Union[int, N]:
+        try:
+            return self.model_type(round(float(value)))
+        except ValueError:
+            return self.null
+        except TypeError:
+            return self.null
 
-class PaddedInt(Integer):
+
+class Integer(BaseInteger[int]):
+    @property
+    def null(self) -> int:
+        return 0
+
+
+class NullInteger(BaseInteger[None]):
+    @property
+    def null(self) -> None:
+        return None
+
+
+class BasePaddedInt(BaseInteger[N]):
     """An integer field that is formatted with a given number of digits,
     padded with zeroes.
     """
-    def __init__(self, digits):
+
+    def __init__(self, digits: int):
         self.digits = digits
 
-    def format(self, value):
-        return u'{0:0{1}d}'.format(value or 0, self.digits)
+    def format(self, value: Union[int, N]) -> str:
+        return "{0:0{1}d}".format(value or 0, self.digits)
+
+
+class PaddedInt(BasePaddedInt[int]):
+    pass
+
+
+class NullPaddedInt(BasePaddedInt[None]):
+    """Same as `PaddedInt`, but does not normalize `None` to `0`."""
+
+    @property
+    def null(self) -> None:
+        return None
 
 
 class ScaledInt(Integer):
     """An integer whose formatting operation scales the number by a
     constant and adds a suffix. Good for units with large magnitudes.
     """
-    def __init__(self, unit, suffix=u''):
+
+    def __init__(self, unit: int, suffix: str = ""):
         self.unit = unit
         self.suffix = suffix
 
-    def format(self, value):
-        return u'{0}{1}'.format((value or 0) // self.unit, self.suffix)
+    def format(self, value: int) -> str:
+        return "{}{}".format((value or 0) // self.unit, self.suffix)
 
 
-class Id(Integer):
+class Id(NullInteger):
     """An integer used as the row id or a foreign key in a SQLite table.
     This type is nullable: None values are not translated to zero.
     """
-    null = None
 
-    def __init__(self, primary=True):
+    @property
+    def null(self) -> None:
+        return None
+
+    def __init__(self, primary: bool = True):
         if primary:
-            self.sql = u'INTEGER PRIMARY KEY'
+            self.sql = "INTEGER PRIMARY KEY"
 
 
-class Float(Type):
-    """A basic floating-point type.
+class BaseFloat(Type[float, N]):
+    """A basic floating-point type. The `digits` parameter specifies how
+    many decimal places to use in the human-readable representation.
     """
-    sql = u'REAL'
-    query = query.NumericQuery
+
+    sql = "REAL"
+    query: typing.Type[FieldQuery[Any]] = NumericQuery
     model_type = float
 
-    def format(self, value):
-        return u'{0:.1f}'.format(value or 0.0)
+    def __init__(self, digits: int = 1):
+        self.digits = digits
+
+    def format(self, value: Union[float, N]) -> str:
+        return "{0:.{1}f}".format(value or 0, self.digits)
 
 
-class NullFloat(Float):
-    """Same as `Float`, but does not normalize `None` to `0.0`.
+class Float(BaseFloat[float]):
+    """Floating-point type that normalizes `None` to `0.0`."""
+
+    @property
+    def null(self) -> float:
+        return 0.0
+
+
+class NullFloat(BaseFloat[None]):
+    """Same as `Float`, but does not normalize `None` to `0.0`."""
+
+    @property
+    def null(self) -> None:
+        return None
+
+
+class BaseString(Type[T, N]):
+    """A Unicode string type."""
+
+    sql = "TEXT"
+    query = SubstringQuery
+
+    def normalize(self, value: Any) -> Union[T, N]:
+        if value is None:
+            return self.null
+        else:
+            return self.model_type(value)
+
+
+class String(BaseString[str, Any]):
+    """A Unicode string type."""
+
+    model_type = str
+
+
+class DelimitedString(BaseString[List[str], List[str]]):
+    """A list of Unicode strings, represented in-database by a single string
+    containing delimiter-separated values.
     """
-    null = None
 
+    model_type = list
 
-class String(Type):
-    """A Unicode string type.
-    """
-    sql = u'TEXT'
-    query = query.SubstringQuery
+    def __init__(self, delimiter: str):
+        self.delimiter = delimiter
+
+    def format(self, value: List[str]):
+        return self.delimiter.join(value)
+
+    def parse(self, string: str):
+        if not string:
+            return []
+        return string.split(self.delimiter)
+
+    def to_sql(self, model_value: List[str]):
+        return self.delimiter.join(model_value)
 
 
 class Boolean(Type):
-    """A boolean type.
-    """
-    sql = u'INTEGER'
-    query = query.BooleanQuery
+    """A boolean type."""
+
+    sql = "INTEGER"
+    query = BooleanQuery
     model_type = bool
 
-    def format(self, value):
-        return six.text_type(bool(value))
+    def format(self, value: bool) -> str:
+        return str(bool(value))
 
-    def parse(self, string):
+    def parse(self, string: str) -> bool:
         return str2bool(string)
 
 
@@ -213,3 +322,7 @@ FLOAT = Float()
 NULL_FLOAT = NullFloat()
 STRING = String()
 BOOLEAN = Boolean()
+SEMICOLON_SPACE_DSV = DelimitedString(delimiter="; ")
+
+# Will set the proper null char in mediafile
+MULTI_VALUE_DSV = DelimitedString(delimiter="\\␀")