Merge pull request #3367 from confor/fix-deps

Add imghdr and cgi

.travis.yml

@@ -11,7 +11,6 @@ cache:
     - lib
 
 python:
-  - "2.6"
   - "2.7"
 
 install:

CHANGELOG.md

@@ -1,5 +1,108 @@
 # Changelog
 
+## v0.6.3
+ Released 26 May 2024
+
+Highlights:
+* Hotfix for searcher not returning results
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.2...v0.6.3).
+
+## v0.6.2
+ Released 26 May 2024
+
+Highlights:
+* Added soulseek support
+* Added bandcamp support
+* Changes and dependency updates to work with Python >= 3.12
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.1...v0.6.2).
+
+## v0.6.1
+R eleased 26 November 2023
+
+Highlights:
+* Dependency updates to work with > Python 3.11
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.0...v0.6.1).
+
+## v0.6.0
+Released 13 November 2022
+
+Highlights:
+* Updated to python 3
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.20...v0.6.0).
+
+## v0.5.20
+Released 15 October 2021
+
+Highlights:
+* Improved: Last.fm & metacrtic api fixes
+* Improved: Various database fixes
+* Added: qBitTorrent
+* Added: Telegram notifications
+* Improved: Many more since the last release. Check the list
+
+Many thanks to @AdeHub for keeping the ship afloat
+and to everyone else who contributed.
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.19...v0.5.20).
+
+## v0.5.19
+Released 27 March 2018
+
+Highlights:
+* Improved: Windows encoding fixes
+* Improved: Performance enhancements
+* Improved: Many more since the last release. Check the list
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.18...v0.5.19).
+
+## v0.5.18
+Released 01 December 2016
+
+Highlights:
+* Added: PassTheHeadphones support
+* Fixed: Special characters in password fields breaking on config page
+* Improved: Updated t411 url
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.17...v0.5.18).
+
+
+## v0.5.17
+Released 10 November 2016
+
+Highlights:
+* Added: t411 support
+* Fixed: Rutracker login
+* Fixed: Deluge empty password
+* Fixed: FreeBSD init script
+* Improved: Musicbrainz searching
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.16...v0.5.17).
+
+## v0.5.16
+Released 10 June 2016
+
+Hotfix update
+
+Highlights:
+* Fixed: Waffles url
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.15...v0.5.16).
+
+## v0.5.15
+Released 07 June 2016
+
+Hotfix update
+
+Highlights:
+* Fixed: Update vip mirror url to point to new server
+* Fixed: Update waffles url to .ch
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.14...v0.5.15).
+
 ## v0.5.14
 Released 02 June 2016
 

Headphones.py

@@ -17,6 +17,10 @@
 import os
 import sys
 
+if sys.version_info <= (3, 6):
+    sys.stdout.write("Headphones requires Python >= 3.7\n")
+    sys.exit(1)
+
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib/'))
 
@@ -54,7 +58,10 @@ def main():
 
     try:
         locale.setlocale(locale.LC_ALL, "")
-        headphones.SYS_ENCODING = locale.getpreferredencoding()
+        if headphones.SYS_PLATFORM == 'win32':
+            headphones.SYS_ENCODING = sys.getdefaultencoding().upper()
+        else:
+            headphones.SYS_ENCODING = locale.getpreferredencoding()
     except (locale.Error, IOError):
         pass
 
@@ -81,6 +88,8 @@ def main():
                         help='Prevent browser from launching on startup')
     parser.add_argument(
         '--pidfile', help='Create a pid file (only relevant when running as a daemon)')
+    parser.add_argument(
+        '--host', help='Specify a host (default - localhost)')
 
     args = parser.parse_args()
 
@@ -170,6 +179,13 @@ def main():
     else:
         http_port = int(headphones.CONFIG.HTTP_PORT)
 
+    # Force the http host if neccessary
+    if args.host:
+        http_host = args.host
+        logger.info('Using forced web server host: %s', http_host)
+    else:
+        http_host = headphones.CONFIG.HTTP_HOST
+
     # Check if pyOpenSSL is installed. It is required for certificate generation
     # and for CherryPy.
     if headphones.CONFIG.ENABLE_HTTPS:
@@ -183,7 +199,7 @@ def main():
     # Try to start the server. Will exit here is address is already in use.
     web_config = {
         'http_port': http_port,
-        'http_host': headphones.CONFIG.HTTP_HOST,
+        'http_host': http_host,
         'http_root': headphones.CONFIG.HTTP_ROOT,
         'http_proxy': headphones.CONFIG.HTTP_PROXY,
         'enable_https': headphones.CONFIG.ENABLE_HTTPS,

README.md

@@ -1,4 +1,4 @@
-##![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
+## ![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
 
 [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=master)](https://travis-ci.org/rembo10/headphones)
 [![Build Status](https://img.shields.io/travis/rembo10/headphones/develop.svg?label=develop%20branch%20build)](https://travis-ci.org/rembo10/headphones)

contrib/sni_test.py

@@ -6,7 +6,7 @@ import sys
 # Ensure that we use the Headphones provided libraries.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib"))
 
-import urlparse
+import urllib.parse
 
 
 def can_import(module):
@@ -89,7 +89,7 @@ def main():
         url = sys.argv[1]
 
     # Check if it is a HTTPS website.
-    parts = urlparse.urlparse(url)
+    parts = urllib.parse.urlparse(url)
 
     if parts.scheme.lower() != "https":
         sys.stderr.write(

data/interfaces/default/album.html

@@ -324,7 +324,7 @@
                 if (++x === 10) {
                     clearInterval(refreshInterval);
                 }
-                var sts = $("#artistname").text();
+                var sts = $("#artistname").text().trim();
                 if (wasLoading == true || sts == "Loading"){
                     location.reload();
                 $('#albumnamespinner').remove()

data/interfaces/default/config.html

@@ -56,7 +56,7 @@
                                     <label title="Password for web server authentication. Leave empty to disable.">
                                         HTTP Password
                                     </label>
-                                    <input type="password" name="http_password" value="${config['http_password']}" size="30">
+                                    <input type="password" name="http_password" value="${config['http_password'] | h}" size="30">
                                 </div>
                                 <div class="row checkbox">
                                     <input type="checkbox" name="launch_browser" value="1" ${config['launch_browser']} />
@@ -133,6 +133,12 @@
                                     </label>
                                     <input type="text" name="mb_ignore_age" value="${config['mb_ignore_age']}" size="4">days
                                 </div>
+                                <div class="row checkbox">
+                                    <label title="Ignore MusicBrainz album updates missing a release date.">
+                                        Ignore Undated Releases
+                                    </label>
+                                    <input type="checkbox" id="mb_ignore_age_missing" name="mb_ignore_age_missing" value="1" ${config['mb_ignore_age_missing']}
+                                </div>                                
                             </fieldset>
                         </td>
                     </tr>
@@ -173,7 +179,7 @@
                                     <label title="SABnzbd password. Leave empty if not applicable.">
                                         SABnzbd Password
                                     </label>
-                                    <input type="password" name="sab_password" value="${config['sab_password']}" size="20">
+                                    <input type="password" name="sab_password" value="${config['sab_password'] | h}" size="20">
                                 </div>
                                 <div class="row">
                                     <label title="SABnzbd API key. Can be found in SABnzbd settings.">
@@ -207,7 +213,7 @@
                                     <label title="NZBGet password. Leave empty if not applicable">
                                         NZBget Password
                                     </label>
-                                    <input type="password" name="nzbget_password" value="${config['nzbget_password']}" size="20">
+                                    <input type="password" name="nzbget_password" value="${config['nzbget_password'] | h}" size="20">
                                 </div>
                                 <div class="row">
                                     <label title="Name of NZBget category to add downloads to.">
@@ -304,6 +310,16 @@
                                     <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="5">
                                 </div>
                             </fieldset>
+                            <fieldset title="Method for downloading Bandcamp.com files.">
+                                <legend>Bandcamp</legend>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can store raw downloads from Bandcamp.com.">
+                                        Bandcamp Directory
+                                    </label>
+                                    <input type="text" name="bandcamp_dir" value="${config['bandcamp_dir']}" size="50">
+                                    <small>Full path where raw MP3s will be stored, e.g. /Users/name/Downloads/bandcamp</small>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset title="Method for downloading torrent files.">
@@ -311,7 +327,8 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_blackhole" value="0" ${config['torrent_downloader_blackhole']}> Black Hole
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
-                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_qbittorrent" value="4" ${config['torrent_downloader_qbittorrent']}> QBitTorrent
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
                                 <div class="row">
@@ -359,7 +376,7 @@
                                 </div>
                                 <div class="row">
                                     <label>Transmission Password</label>
-                                    <input type="password" name="transmission_password" value="${config['transmission_password']}" size="30">
+                                    <input type="password" name="transmission_password" value="${config['transmission_password'] | h}" size="30">
                                 </div>
                                 <div class="row">
                                     <small>Note: With Transmission, you can specify a different download directory for downloads sent from Headphones.
@@ -379,13 +396,33 @@
                                 </div>
                                 <div class="row">
                                     <label>uTorrent Password</label>
-                                    <input type="password" name="utorrent_password" value="${config['utorrent_password']}" size="30">
+                                    <input type="password" name="utorrent_password" value="${config['utorrent_password'] | h}" size="30">
                                 </div>
                                 <div class="row">
                                     <label>uTorrent Label</label>
                                     <input type="text" name="utorrent_label" value="${config['utorrent_label']}" size="30">
                                 </div>
                             </fieldset>
+                            <fieldset id="qbittorrent_options">
+                                <small class="heading"><i class="fa fa-info-circle"></i> Note: Works with WebAPI Rev 11 and later (QBitTorrent 3.4.0 and later) </small>
+                                <div class="row">
+                                    <label>QBitTorrent Host</label>
+                                    <input type="text" name="qbittorrent_host" value="${config['qbittorrent_host']}" size="30">
+                                    <small>usually http://localhost:8081</small>
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Username</label>
+                                    <input type="text" name="qbittorrent_username" value="${config['qbittorrent_username']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Password</label>
+                                    <input type="password" name="qbittorrent_password" value="${config['qbittorrent_password']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Label</label>
+                                    <input type="text" name="qbittorrent_label" value="${config['qbittorrent_label']}" size="30">
+				</div>
+                            </fieldset>
                             <fieldset id="deluge_options">
                                 <div class="row">
                                     <label>Deluge WebUI Host and Port</label>
@@ -400,7 +437,7 @@
                                 </div>
                                 <div class="row">
                                     <label>Deluge Password</label>
-                                    <input type="password" name="deluge_password" value="${config['deluge_password']}" size="30">
+                                    <input type="password" name="deluge_password" value="${config['deluge_password'] | h}" size="30">
                                 </div>
                                 <div class="row">
                                     <small>Note: With Deluge, you can specify a different download directory for downloads sent from Headphones.
@@ -411,6 +448,11 @@
                                     <input type="text" name="deluge_label" value="${config['deluge_label']}" size="30">
                                     <small>Labels shouldn't contain spaces (requires Label plugin)</small>
                                 </div>
+                                <div class="row">
+                                    <label>Download Directory</label>
+                                    <input type="text" name="deluge_download_directory" value="${config['deluge_download_directory']}" size="30">
+                                    <small>Directory where Deluge should download to</small>
+                                </div>
                                 <div class="row">
                                     <label>Move When Completed</label>
                                     <input type="text" name="deluge_done_directory" value="${config['deluge_done_directory']}" size="30">
@@ -440,7 +482,33 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
-                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>Soulseek
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>No Preference
+                                </div>
+                            </fieldset>
+                        </td>
+                        <td>
+                            <fieldset>
+                                <legend>Soulseek</legend>
+                                <div class="row">
+                                    <label>Soulseek API URL</label>
+                                    <input type="text" name="soulseek_api_url" value="${config['soulseek_api_url']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label>Soulseek API KEY</label>
+                                    <input type="text" name="soulseek_api_key" value="${config['soulseek_api_key']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_download_dir" value="${config['soulseek_download_dir']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Incomplete Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_incomplete_download_dir" value="${config['soulseek_incomplete_download_dir']}" size="50">
                                 </div>
                             </fieldset>
                         </td>
@@ -466,7 +534,7 @@
                                     </div>
                                     <div class="row">
                                         <label>Password</label>
-                                        <input class="hppass" type="password" value="${config['hppass']}" size="20">
+                                        <input class="hppass" type="password" value="${config['hppass'] | h}" size="20">
                                     </div>
                                     <div class="row">
                                         <a href="https://headphones.codeshy.com/vip" id="vipserver" target="_blank">Don't have an account? Sign up!</a>
@@ -552,6 +620,19 @@
                                     </div>
                                 </div>
                             </fieldset>
+                            <fieldset>
+                                <legend>Other</legend>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_bandcamp" type="checkbox" class="bigcheck" name="use_bandcamp" value="1" ${config['use_bandcamp']} /><label for="use_bandcamp"><span class="option">Bandcamp</span></label>
+                                    </div>
+                                </fieldset>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_soulseek" type="checkbox" class="bigcheck" name="use_soulseek" value="1" ${config['use_soulseek']} /><label for="use_soulseek"><span class="option">Soulseek</span></label>
+                                    </div>
+                                </fieldset>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -574,43 +655,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_kat" type="checkbox" class="bigcheck" name="use_kat" value="1" ${config['use_kat']} /><label for="use_kat"><span class="option">Kick Ass Torrents</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Proxy URL</label>
-                                            <input type="text" name="kat_proxy_url" value="${config['kat_proxy_url']}" size="36">
-                                            <small>Optional. Leave empty for default.</small>
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="kat_ratio" value="${config['kat_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_waffles" type="checkbox" class="bigcheck" name="use_waffles" value="1" ${config['use_waffles']} /><label for="use_waffles"><span class="option">Waffles.fm</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>UID Number</label>
-                                            <input type="text" name="waffles_uid" value="${config['waffles_uid']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Passkey</label>
-                                            <input type="password" name="waffles_passkey" value="${config['waffles_passkey']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="waffles_ratio" value="${config['waffles_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                                 <fieldset>
                                     <div class="row checkbox left">
                                         <input id="use_rutracker" type="checkbox" class="bigcheck" name="use_rutracker" value="1" ${config['use_rutracker']} /><label for="use_rutracker"><span class="option">rutracker.org</span></label>
@@ -622,62 +666,90 @@
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="rutracker_password" value="${config['rutracker_password']}" size="36">
+                                            <input type="password" name="rutracker_password" value="${config['rutracker_password'] | h}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
                                             <input type="text" class="override-float" name="rutracker_ratio" value="${config['rutracker_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
+                                        <div class="row">
+                                            <label>Session Cookie (Optional - Advanced)</label>
+                                            <input type="text" class="override-float" name="rutracker_cookie" value="${config['rutracker_cookie']}" size="10" title="bb_session cookie (Advanced)">
+                                        </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_whatcd" type="checkbox" class="bigcheck" name="use_whatcd" value="1" ${config['use_whatcd']} /><label for="use_whatcd"><span class="option">What.cd</span></label>
+                                        <input id="use_orpheus" type="checkbox" class="bigcheck" name="use_orpheus" value="1" ${config['use_orpheus']} /><label for="use_orpheus"><span class="option">Orpheus.network</span></label>
                                     </div>
                                     <div class="config">
                                         <div class="row">
                                             <label>Username</label>
-                                            <input type="text" name="whatcd_username" value="${config['whatcd_username']}" size="36">
+                                            <input type="text" name="orpheus_username" value="${config['orpheus_username']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="whatcd_password" value="${config['whatcd_password']}" size="36">
+                                            <input type="password" name="orpheus_password" value="${config['orpheus_password'] | h}" size="36">
+                                        </div>
+                                        <div class="row">
+                                            <label>URL</label>
+                                            <input type="text" name="orpheus_url" value="${config['orpheus_url']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="whatcd_ratio" value="${config['whatcd_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                            <input type="text" class="override-float" name="orpheus_ratio" value="${config['orpheus_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_strike" type="checkbox" class="bigcheck" name="use_strike" value="1" ${config['use_strike']} /><label for="use_strike"><span class="option">Strike Search</span></label>
+                                        <input id="use_redacted" type="checkbox" class="bigcheck" name="use_redacted" value="1" ${config['use_redacted']} /><label for="use_redacted"><span class="option">Redacted</span></label>
                                     </div>
                                     <div class="config">
+                                        <div class="row">
+                                            <label>Api Key</label>
+                                            <input type="text" name="redacted_apikey" value="${config['redacted_apikey']}" size="36">
+                                        </div>
+                                        <div class="row">
+                                            <label>Username</label>
+                                            <input type="text" name="redacted_username" value="${config['redacted_username']}" size="36">
+                                        </div>
+                                        <div class="row">
+                                            <label>Password</label>
+                                            <input type="password" name="redacted_password" value="${config['redacted_password'] | h}" size="36">
+                                        </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="strike_ratio" value="${config['strike_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            <input type="text" class="override-float" name="redacted_ratio" value="${config['redacted_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                        </div>
+                                        <div class="row">
+                                            <label>Use freeleech tokens when available</label>
+                                            <input type="checkbox" name="redacted_use_fltoken" value="1" ${config['redacted_use_fltoken']}>
                                         </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_torznab" type="checkbox" class="bigcheck" name="use_torznab" value="1" ${config['use_torznab']} /><label for="use_torznab"><span class="option">Jackett / Torznab Providers</span></label>
+                                        <input id="use_torznab" type="checkbox" class="bigcheck" name="use_torznab" value="1" ${config['use_torznab']} /><label for="use_torznab"><span class="option">Torznab Providers</span></label>
                                     </div>
                                     <div id="torznab_providers">
                                         <div class="config" id="torznab1">
                                             <div class="row">
                                                 <label>Torznab Host</label>
-                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="30">
+                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="36">
-                                                <small>e.g. http://localhost:9117/torznab/iptorrents</small>
+                                                <small>e.g. http://localhost:9117/api/v2.0/indexers/demonoid/results/torznab/</small>
                                             </div>
                                             <div class="row">
                                                 <label>Torznab API</label>
                                                 <input type="text" name="torznab_apikey" value="${config['torznab_apikey']}" size="36">
                                             </div>
+                                            <div class="row">
+                                                <label>Torznab Seed Ratio</label>
+                                                <input type="text" class="override-float" name="torznab_ratio" value="${config['torznab_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            </div>
                                             <div class="row checkbox">
                                                 <input id="torznab_enabled" type="checkbox" name="torznab_enabled" value="1" ${config['torznab_enabled']} /><label>Enabled</label>
                                             </div>
@@ -687,7 +759,7 @@
                                         %>
                                         %for torznab in config['extra_torznabs']:
                                             <%
-                                                if torznab[2] == '1' or torznab[2] == 1:
+                                                if torznab[3] == '1' or torznab[3] == 1:
                                                     torznab_enabled = "checked"
                                                 else:
                                                     torznab_enabled = ""
@@ -695,12 +767,16 @@
                                             <div class="config" id="torznab${torznab_number}">
                                                 <div class="row">
                                                     <label>Torznab Host</label>
-                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="30">
+                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="36">
                                                 </div>
                                                 <div class="row">
                                                     <label>Torznab API</label>
                                                     <input type="text" name="torznab_api${torznab_number}" value="${torznab[1]}" size="36">
                                                 </div>
+                                                <div class="row">
+                                                    <label>Torznab Seed Ratio</label>
+                                                    <input type="text" class="override-float" name="torznab_ratio${torznab_number}" value="${torznab[2]}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                                </div>
                                                 <div class="row checkbox">
                                                     <input id="torznab_enabled" type="checkbox" name="torznab_enabled${torznab_number}" value="1" ${torznab_enabled} /><label>Enabled</label>
                                                 </div>
@@ -716,34 +792,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_oldpiratebay" type="checkbox" class="bigcheck" name="use_oldpiratebay" value="1" ${config['use_oldpiratebay']} /><label for="use_oldpiratebay"><span class="option">Old Pirate Bay</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>URL</label>
-                                            <input type="text" name="oldpiratebay_url" value="${config['oldpiratebay_url']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="oldpiratebay_ratio" value="${config['oldpiratebay_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
-                                <fieldset>
-                                     <div class="row checkbox left">
-                                        <input id="use_mininova" type="checkbox" class="bigcheck" name="use_mininova" value="1" ${config['use_mininova']} /><label for="use_mininova"><span class="option">Mininova</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="mininova_ratio" value="${config['mininova_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                             </fieldset>
                         </td>
                     </tr>
@@ -884,30 +932,42 @@
                                         <input type="checkbox" name="keep_nfo" value="1" ${config['keep_nfo']} />
                                     </label>
                                 </div>
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed lyrics
+                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
+                                    </label>
+                                </div>
+
+
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed album art in each file
+                                        <input type="checkbox" name="embed_album_art" id="embed_album_art" value="1" ${config['embed_album_art']}>
+                                    </label>
+                                </div>
+
                                 <div class="row checkbox left clearfix nopad">
                                     <label>
                                         Add album art jpeg to album folder
                                         <input type="checkbox" name="add_album_art" id="add_album_art" value="1" ${config['add_album_art']}>
                                     </label>
                                 </div>
+
                                 <div id="album_art_options" style="padding-left: 20px">
                                     <div class="row">
                                         as <input type="text" class="override-float" name="album_art_format" value="${config['album_art_format']}" size="10">.jpg
                                     </div>
                                     <small>Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
                                 </div>
-                                <div class="row checkbox left clearfix nopad">
+
-                                    <label>
+                                <div id="album_art_size_options" style="padding-left: 20px">
-                                        Embed album art in each file
+                                    <div class="row">
-                                        <input type="checkbox" name="embed_album_art" value="1" ${config['embed_album_art']}>
+                                        Album art min width <input type="text" class="override-float" name="album_art_min_width" value="${config['album_art_min_width']}" size="4" title="Only images with a pixel width greater or equal are considered as valid album art candidates. Default: 0.">\
-                                    </label>
+                                        Album art max width <input type="text" class="override-float" name="album_art_max_width" value="${config['album_art_max_width']}" size="4" title="A maximum image width to downscale fetched images if they are too big.">
-                                </div>
+                                    </div>
-                                <div class="row checkbox left clearfix nopad">
-                                    <label>
-                                        Embed lyrics
-                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
-                                    </label>
                                 </div>
+
                                 <div class="row">
                                     <label>
                                         Destination Directory:
@@ -964,7 +1024,7 @@
                                         <label>SMTP User</label><input type="text" name="email_smtp_user" value="${config['email_smtp_user']}" size="254">
                                     </div>
                                     <div class="row">
-                                        <label>SMTP Password</label><input type="password" name="email_smtp_password" value="${config['email_smtp_password']}" size="50">
+                                        <label>SMTP Password</label><input type="password" name="email_smtp_password" value="${config['email_smtp_password'] | h}" size="50">
                                     </div>
                                     <div class="row checkbox">
                                         <input type="text" class="override-float" name="email_smtp_port" value="${config['email_smtp_port']}" size="4"><label>SMTP Port</label>
@@ -990,7 +1050,7 @@
                                         <label>Growl Host:Port</label><input type="text" name="growl_host" value="${config['growl_host']}" size="30">
                                     </div>
                                     <div class="row">
-                                        <label>Growl Password</label><input type="password" name="growl_password" value="${config['growl_password']}" size="30">
+                                        <label>Growl Password</label><input type="password" name="growl_password" value="${config['growl_password'] | h}" size="30">
                                     </div>
                                     <div class="row checkbox">
                                         <input type="checkbox" name="growl_onsnatch" value="1" ${config['growl_onsnatch']} /><label>Notify on snatch?</label>
@@ -1012,7 +1072,7 @@
                                         <label>Username</label><input type="text" name="xbmc_username" value="${config['xbmc_username']}" size="30">
                                     </div>
                                     <div class="row">
-                                        <label>Password</label><input type="password" name="xbmc_password" value="${config['xbmc_password']}" size="30">
+                                        <label>Password</label><input type="password" name="xbmc_password" value="${config['xbmc_password'] | h}" size="30">
                                     </div>
                                     <div class="checkbox row">
                                         <input type="checkbox" name="xbmc_update" value="1" ${config['xbmc_update']} /><label>Update Library</label>
@@ -1126,7 +1186,7 @@
                                         <small>Username of your Plex client API (blank for none)</small>
                                     </div>
                                     <div class="row">
-                                        <label>Plex Password</label><input type="password" name="plex_password" value="${config['plex_password']}" size="30">
+                                        <label>Plex Password</label><input type="password" name="plex_password" value="${config['plex_password'] | h}" size="30">
                                         <small>Password of your Plex client API (blank for none)</small>
                                     </div>
                                     <div class="row">
@@ -1226,7 +1286,7 @@
                                         <label>Subsonic Username</label><input type="text" name="subsonic_username" value="${config['subsonic_username']}" size="30">
                                     </div>
                                     <div class="row">
-                                        <label>Subsonic Password</label><input type="password" name="subsonic_password" value="${config['subsonic_password']}" size="30">
+                                        <label>Subsonic Password</label><input type="password" name="subsonic_password" value="${config['subsonic_password'] | h}" size="30">
                                     </div>
                                 </div>
                             </fieldset>
@@ -1236,7 +1296,7 @@
                                     <input type="checkbox" name="synoindex_enabled" id="synoindex" value="1" ${config['synoindex_enabled']} /><label for="synoindex"><span class="option">Synology NAS</span></label>
                                 </div>
                             </fieldset>
-
+<!--
                             <fieldset>
                                 <div class="row checkbox left">
                                     <input type="checkbox" class="bigcheck" name="twitter_enabled" id="twitter" value="1" ${config['twitter_enabled']} /><label for="twitter"><span class="option">Twitter</span></label>
@@ -1257,6 +1317,26 @@
                                     </div>
                                 </div>
                             </fieldset>
+-->
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="slack_enabled" id="slack" value="1" ${config['slack_enabled']} /><label for="slack"><span class="option">Slack</span></label>
+                                </div>
+                                <div id="slackoptions">
+                                    <div class="row">
+                                        <label>Slack Webhook</label><input type="text" name="slack_url" value="${config['slack_url']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Channel</label><input type="text" name="slack_channel" value="${config['slack_channel']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <label>Emoji</label><input type="text" name="slack_emoji" value="${config['slack_emoji']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="slack_onsnatch" value="1" ${config['slack_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
 
                             <fieldset>
                                 <div class="row checkbox left">
@@ -1275,6 +1355,23 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="join_enabled" id="join" value="1" ${config['join_enabled']} /><label for="join"><span class="option">Join</span></label>
+                                </div>
+                                <div id="joinoptions">
+                                    <div class="row">
+                                        <label>Join API Key</label><input type="text" name="join_apikey" value="${config['join_apikey']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Device ID(s)</label><input type="text" name="join_deviceid" value="${config['join_deviceid']}" size="50"><small>Comma separated list.  Leave blank to send to all devices</small>
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="join_onsnatch" value="1" ${config['join_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
+
                         </td>
                     </tr>
                 </table>
@@ -1295,17 +1392,20 @@
                             <div class="row">
                                 <label>File Format</label>
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
-                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year. Put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
+                                <small>Use: In addition to the above, there is also $Title/$title (track title), $Track (track #), $Disc (disc #), $DiscTotal.</small>
                             </div>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
                             </div>
+                            <div class="checkbox row left clearfix nopad">
+                                <input type="checkbox" name="rename_single_disc_ignore" id="rename_single_disc_ignore" value="1" ${config['rename_single_disc_ignore']}/><label>Don't include disc# for single disc albums</label>
+                            </div>
                         </fieldset>
 
                         <fieldset>
                             <legend>Re-Encoding Options</legend>
                             <small class="heading"><i class="fa fa-info-circle"></i> Note: this option requires the lame, ffmpeg or xld encoder</small>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="music_encoder" id="music_encoder" value="1" ${config['music_encoder']}/><label>Re-encode downloads during postprocessing</label>
                             </div>
                             <div id="encoderoptions" class="row clearfix checkbox">
@@ -1570,6 +1670,20 @@
                                     <label>Cache Directory</label>
                                     <input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
                                 </div>
+                                <div class="row">
+                                    <label>Post Processing Temporary Directory</label>
+                                    <input type="text" name="keep_torrent_files_dir" value="${config['keep_torrent_files_dir']}" size="50" title="Enter the directory to preserve files for seeding, default is system temp directory">
+                                </div>
+                            </fieldset>
+
+                            <fieldset>
+                                <legend>Last.fm</legend>
+                                <div id="lastfmoptions">
+                                    <div class="row">
+                                        <label>API Key</label>
+                                        <input type="text" name="lastfm_apikey" value="${config['lastfm_apikey']}" size="40" />
+                                    </div>
+                                </div>
                             </fieldset>
 
                             <fieldset>
@@ -1626,7 +1740,7 @@
                                             <label>Username</label><input type="text" class="customuser" name="customuser" value="${config['customuser']}" size="20">
                                         </div>
                                         <div class="row">
-                                            <label>Password</label><input type="password" class="custompass" name="custompass" value="${config['custompass']}" size="15"><br>
+                                            <label>Password</label><input type="password" class="custompass" name="custompass" value="${config['custompass'] | h}" size="15"><br>
                                         </div>
                                     </div>
                                     <div class="row">
@@ -1639,7 +1753,7 @@
                                         <label>Username</label><input type="text" class="hpuser" name="hpuser" value="${config['hpuser']}" size="20">
                                     </div>
                                     <div class="row">
-                                        <label>Password</label><input type="password" class="hppass" name="hppass" value="${config['hppass']}" size="20"><br>
+                                        <label>Password</label><input type="password" class="hppass" name="hppass" value="${config['hppass'] | h}" size="20"><br>
                                         <a href="https://headphones.codeshy.com/vip" id="vipserver" target="_blank">Get an Account!</a>
                                     </div>
                                 </div>
@@ -1814,9 +1928,36 @@
                         }
                 });
 
+                if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").show();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").hide();
+                              }
+                        }
+
+                $("#embed_album_art").click(function(){
+                        if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").slideDown();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
+                        }
+                });
+
                 if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").show();
+                            $("#album_art_size_options").show();
                         }
                 else
                         {
@@ -1827,10 +1968,15 @@
                         if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").slideDown();
+                            $("#album_art_size_options").slideDown();
                         }
                         else
                         {
                             $("#album_art_options").slideUp();
+                            if (!$("#embed_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
                         }
                 });
 
@@ -2035,6 +2181,27 @@
                         }
                 });
 
+                if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").show();
+                        }
+                else
+                        {
+                            $("#joinoptions").hide();
+                        }
+
+
+                $("#join").click(function(){
+                        if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#joinoptions").slideUp();
+                        }
+                });
+
                 if ($("#twitter").is(":checked"))
                         {
                             $("#twitteroptions").show();
@@ -2055,6 +2222,27 @@
                         }
                 });
 
+                if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").show();
+                        }
+                else
+                        {
+                            $("#slackoptions").hide();
+                        }
+
+                $("#slack").click(function(){
+                        if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#slackoptions").slideUp();
+                        }
+                });
+
+
                 if ($("#telegram").is(":checked"))
                         {
                             $("#telegramoptions").show();
@@ -2231,26 +2419,30 @@
 
                 if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#torrent_blackhole_options").show();
                         }
                 if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#transmission_options").show();
                         }
                 if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").hide();
                             $("#utorrent_options").show();
                         }
+                if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#qbittorrent_options").show();
+                        }
                 if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#qbittorrent_options").hide();
                             $("#deluge_options").show();
                         }
 
-
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
                         {
@@ -2286,19 +2478,23 @@
                         }
                         if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
                         }
+                        if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#qbittorrent_options").fadeIn() });
+			}
                         if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
                 });
 
@@ -2360,7 +2556,7 @@
 
                 $("#add_torznab").click(function() {
                     var intId = $("#torznab_providers > div").size() + deletedTorznabs + 1;
-                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
+                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row\"><label>Torznab Seed Ratio</label><input type=\"text\" class=\"override-float\" name=\"torznab_ratio" + intId + "\" size=\"10\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
                     var removeTorznabButton = $("<div class=\"row\"><input type=\"button\" class=\"remove_torznab\" value=\"Remove\" /></div>");
                     removeTorznabButton.click(function() {
                         $(this).parent().remove();
@@ -2389,18 +2585,14 @@
                 initConfigCheckbox("#use_nzbsorg");
                 initConfigCheckbox("#use_omgwtfnzbs");
                 initConfigCheckbox("#use_torznab");
-                initConfigCheckbox("#use_kat");
                 initConfigCheckbox("#use_piratebay");
-                initConfigCheckbox("#use_oldpiratebay");
-                initConfigCheckbox("#use_mininova");
-                initConfigCheckbox("#use_waffles");
                 initConfigCheckbox("#use_rutracker");
-                initConfigCheckbox("#use_whatcd");
+                initConfigCheckbox("#use_orpheus");
-                initConfigCheckbox("#use_strike");
+                initConfigCheckbox("#use_redacted");
                 initConfigCheckbox("#api_enabled");
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");
-
+                initConfigCheckbox("#mb_ignore_age_missing");
 
                 $('#twitterStep1').click(function () {
                     $.get("/twitterStep1", function (data) {window.open(data); })

data/interfaces/default/history.html

@@ -1,7 +1,7 @@
 <%inherit file="base.html"/>
 <%!
 	from headphones import helpers
-	import cgi
+	from html import escape as html_escape 
 %>
 
 <%def name="headerIncludes()">
@@ -56,17 +56,19 @@
                 fileid = 'torrent'
             if item['URL'].find('codeshy') != -1:
                 fileid = 'nzb'
+            if item['URL'].find('bandcamp') != -1:
+                fileid = 'bandcamp'
 
             folder = 'Folder: ' + item['FolderName']
 
         %>
 			<tr class="grade${grade}">
 				<td id="dateadded">${item['DateAdded']}</td>
-				<td id="filename">${cgi.escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
+				<td id="filename">${html_escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
 				<td id="size">${helpers.bytes_to_mb(item['Size'])}</td>
 				<td title="${folder}" id="status">${item['Status']}</td>
-				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${cgi.escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${cgi.escape(item['Title'], quote=True)}'">new</a>]</td>
+				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${html_escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${html_escape(item['Title'], quote=True)}'">new</a>]</td>
-				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${cgi.escape(item['Title'], quote=True)}',$(this),'table')" data-success="${cgi.escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
+				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${html_escape(item['Title'], quote=True)}',$(this),'table')" data-success="${html_escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
 			</tr>
 		%endfor
 		</tbody>

data/interfaces/default/index.html

@@ -36,7 +36,7 @@
                         "aTargets": [0],
                         "mData":"ArtistID",
                         "mRender": function ( data, type, full ) {
-                            return '<div id="artistImg"><img class="albumArt" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
+                            return '<div id="artistImg"><img class="albumArt" height="50" width="50" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
                         }
                     },
                     {

data/interfaces/default/js/script.js

@@ -312,6 +312,7 @@ function doAjaxCall(url,elem,reload,form) {
 	$.ajax({
 	  url: url,
 	  data: dataString,
+	  type: 'POST',
 	  beforeSend: function(jqXHR, settings) {
 	  	// Start loader etc.
 	  	feedback.prepend(loader);

headphones/__init__.py

@@ -86,8 +86,8 @@ CURRENT_VERSION = None
 LATEST_VERSION = None
 COMMITS_BEHIND = None
 
-LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"]
+LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma", "opus"]
-LOSSLESS_MEDIA_FORMATS = ["flac", "aiff"]
+LOSSLESS_MEDIA_FORMATS = ["flac", "aiff", "aif"]
 MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS
 
 MIRRORLIST = ["musicbrainz.org", "headphones", "custom"]
@@ -218,7 +218,7 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno)
 
     os.setsid()
@@ -232,10 +232,10 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno)
 
-    dev_null = file('/dev/null', 'r')
+    dev_null = open('/dev/null', 'r')
     os.dup2(dev_null.fileno(), sys.stdin.fileno())
 
     si = open('/dev/null', "r")
@@ -251,7 +251,7 @@ def daemonize():
 
     if CREATEPID:
         logger.info("Writing PID %d to %s", pid, PIDFILE)
-        with file(PIDFILE, 'w') as fp:
+        with open(PIDFILE, 'w') as fp:
             fp.write("%s\n" % pid)
 
 
@@ -306,9 +306,10 @@ def initialize_scheduler():
                          minutes=minutes)
 
         # Remove Torrent + data if Post Processed and finished Seeding
-        minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
+        if headphones.CONFIG.TORRENT_DOWNLOADER != 0:
-        schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
+            minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
-                     minutes=minutes)
+            schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
+                         minutes=minutes)
 
         # Start scheduler
         if start_jobs and len(SCHED.get_jobs()):
@@ -359,6 +360,8 @@ def sig_handler(signum=None, frame=None):
 
 
 def dbcheck():
+    logger.debug("SQLite Version: %s", sqlite3.sqlite_version)
+    logger.debug("DB-API Version: %s", sqlite3.version)
     conn = sqlite3.connect(DB_FILE)
     c = conn.cursor()
     c.execute(
@@ -376,7 +379,7 @@ def dbcheck():
     c.execute(
         'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)')
     c.execute(
-        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)')
+        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT, TorrentHash TEXT)')
     # Matched is a temporary value used to see if there was a match found in
     # alltracks
     c.execute(
@@ -416,6 +419,33 @@ def dbcheck():
         'CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)')
     c.execute(
         'CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)')
+    c.execute(
+        'CREATE INDEX IF NOT EXISTS tracks_artistid ON tracks(ArtistID ASC)')
+
+    # Speed up album page
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_albumid ON allalbums(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_albumid ON alltracks(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS releases_albumid ON releases(ReleaseGroupID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_albumid ON descriptions(ReleaseGroupID ASC)')
+
+    # Speed up artist deletion
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_artistid ON allalbums(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_artistid ON alltracks(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_artistid ON descriptions(ArtistID ASC)')
+
+    # Speed up Artist refresh hybrid release
+    c.execute('CREATE INDEX IF NOT EXISTS albums_releaseid ON albums(ReleaseID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_releaseid ON tracks(ReleaseID ASC)')
+
+    # Speed up scanning and track matching
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistname ON artists(ArtistName COLLATE NOCASE ASC)')
+
+    # General speed up
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistsortname ON artists(ArtistSortName COLLATE NOCASE ASC)')
+
+    c.execute(
+        """CREATE INDEX IF NOT EXISTS have_matched_artist_album ON have(Matched ASC, ArtistName COLLATE NOCASE ASC, AlbumTitle COLLATE NOCASE ASC)""")
+    c.execute('DROP INDEX IF EXISTS have_matched')
 
     try:
         c.execute('SELECT IncludeExtras from artists')
@@ -613,6 +643,20 @@ def dbcheck():
     except sqlite3.OperationalError:
         c.execute('ALTER TABLE artists ADD COLUMN MetaCritic TEXT DEFAULT NULL')
 
+    try:
+        c.execute('SELECT TorrentHash from snatched')
+    except sqlite3.OperationalError:
+        c.execute('ALTER TABLE snatched ADD COLUMN TorrentHash TEXT')
+        c.execute('UPDATE snatched SET TorrentHash = FolderName WHERE Status LIKE "Seed_%"')
+
+    # One off script to set CleanName to lower case
+    clean_name_mixed = c.execute('SELECT CleanName FROM have ORDER BY Date Desc').fetchone()
+    if clean_name_mixed and clean_name_mixed[0] != clean_name_mixed[0].lower():
+        logger.info("Updating track clean name, this could take some time...")
+        c.execute('UPDATE tracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE alltracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE have SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+
     conn.commit()
     c.close()
 

headphones/albumart.py

@@ -13,16 +13,196 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import request, db, logger
+import struct
+from six.moves.urllib.parse import urlencode
+from io import BytesIO
+
+import headphones
+from headphones import db, request, logger
 
 
 def getAlbumArt(albumid):
-    myDB = db.DBConnection()
-    asin = myDB.action(
-        'SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0]
 
-    if asin:
+    artwork_path = None
-        return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
+    artwork = None
+
+    # CAA
+    logger.info("Searching for artwork at CAA")
+    #artwork_path = 'https://coverartarchive.org/release-group/%s/front' % albumid
+    artwork_path = 'https://coverartarchive.org/release-group/%s' % albumid
+
+    data = request.request_json(artwork_path, timeout=20, whitelist_status_code=404)
+
+    image_url = None
+    if data:
+        for item in data.get("images", []):
+            try:
+                if "Front" not in item["types"]:
+                    continue
+
+                # Use desired size
+                image_url = item["image"]
+                if headphones.CONFIG.ALBUM_ART_MAX_WIDTH:
+                    if isinstance(item.get("thumbnails"), dict):
+                        image_url = item["thumbnails"].get(
+                            headphones.CONFIG.ALBUM_ART_MAX_WIDTH, image_url
+                        )
+                break
+            except KeyError:
+                pass
+
+    if image_url:
+        artwork = getartwork(image_url)
+        if artwork:
+            logger.info("Artwork found at CAA")
+            return artwork_path, artwork
+
+    # Amazon
+    logger.info("Searching for artwork at Amazon")
+    myDB = db.DBConnection()
+    dbalbum = myDB.action(
+        'SELECT ArtistName, AlbumTitle, ReleaseID, AlbumASIN FROM albums WHERE AlbumID=?',
+        [albumid]).fetchone()
+    if dbalbum['AlbumASIN']:
+        artwork_path = 'https://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % dbalbum['AlbumASIN']
+        artwork = getartwork(artwork_path)
+        if artwork:
+            logger.info("Artwork found at Amazon")
+            return artwork_path, artwork
+
+    # last.fm
+    from headphones import lastfm
+    logger.info("Searching for artwork at last.fm")
+    if dbalbum['ReleaseID'] != albumid:
+        data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'])
+        if not data:
+            data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                         album=dbalbum['AlbumTitle'])
+    else:
+        data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                     album=dbalbum['AlbumTitle'])
+
+    if data:
+        try:
+            images = data['album']['image']
+            for image in images:
+                if image['size'] == 'extralarge':
+                    artwork_path = image['#text']
+                elif image['size'] == 'mega':
+                    artwork_path = image['#text']
+                    break
+        except KeyError:
+            artwork_path = None
+
+        if artwork_path:
+            artwork = getartwork(artwork_path)
+            if artwork:
+                logger.info("Artwork found at last.fm")
+                return artwork_path, artwork
+
+    logger.info("No suitable album art found.")
+    return None, None
+
+
+def jpeg(bites):
+    fhandle = BytesIO(bites)
+    try:
+        fhandle.seek(0)
+        size = 2
+        ftype = 0
+        while not 0xc0 <= ftype <= 0xcf:
+            fhandle.seek(size, 1)
+            byte = fhandle.read(1)
+            while ord(byte) == 0xff:
+                byte = fhandle.read(1)
+            ftype = ord(byte)
+            size = struct.unpack('>H', fhandle.read(2))[0] - 2
+        fhandle.seek(1, 1)
+        height, width = struct.unpack('>HH', fhandle.read(4))
+        return width, height
+    except struct.error:
+        return None, None
+    except TypeError:
+        return None, None
+
+
+def png(bites):
+    try:
+        check = struct.unpack('>i', bites[4:8])[0]
+        if check != 0x0d0a1a0a:
+            return None, None
+        return struct.unpack('>ii', bites[16:24])
+    except struct.error:
+        return None, None
+
+
+def get_image_data(bites):
+    type = None
+    width = None
+    height = None
+    if len(bites) < 24:
+        return None, None, None
+
+    peek = bites[0:2]
+    if peek == b'\xff\xd8':
+        width, height = jpeg(bites)
+        type = 'jpg'
+    elif peek == b'\x89P':
+        width, height = png(bites)
+        type = 'png'
+    return type, width, height
+
+
+def getartwork(artwork_path):
+    artwork = bytes()
+    minwidth = 0
+    maxwidth = 0
+    if headphones.CONFIG.ALBUM_ART_MIN_WIDTH:
+        minwidth = int(headphones.CONFIG.ALBUM_ART_MIN_WIDTH)
+    if headphones.CONFIG.ALBUM_ART_MAX_WIDTH:
+        maxwidth = int(headphones.CONFIG.ALBUM_ART_MAX_WIDTH)
+
+    resp = request.request_response(artwork_path, timeout=20, stream=True, whitelist_status_code=404)
+
+    if resp:
+        img_width = None
+        for chunk in resp.iter_content(chunk_size=1024):
+            artwork += chunk
+            if not img_width and (minwidth or maxwidth):
+                img_type, img_width, img_height = get_image_data(artwork)
+            # Check min/max
+            if img_width and (minwidth or maxwidth):
+                if minwidth and img_width < minwidth:
+                    logger.info("Artwork is too small. Type: %s. Width: %s. Height: %s",
+                                img_type, img_width, img_height)
+                    artwork = None
+                    break
+                elif maxwidth and img_width > maxwidth:
+                    # Downsize using proxy service to max width
+                    artwork = bytes()
+                    url = "https://images.weserv.nl"
+                    params = {
+                        "url": artwork_path,
+                        "w": maxwidth
+                    }
+                    headers = {"User-Agent": "Headphones"}
+                    r = request.request_response(
+                        url,
+                        params=params,
+                        timeout=20,
+                        stream=True,
+                        whitelist_status_code=404,
+                        headers=headers
+                    )
+                    if r:
+                        for chunk in r.iter_content(chunk_size=1024):
+                            artwork += chunk
+                        r.close()
+                        logger.info("Artwork is greater than the maximum width, downsized using proxy service")
+                    break
+        resp.close()
+
+    return artwork
 
 
 def getCachedArt(albumid):
@@ -34,7 +214,7 @@ def getCachedArt(albumid):
     if not artwork_path:
         return
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith("http"):
         artwork = request.request_content(artwork_path, timeout=20)
 
         if not artwork:

headphones/albumswitcher.py

@@ -85,8 +85,8 @@ def switch(AlbumID, ReleaseID):
 
     # Update have track counts on index
     totaltracks = len(myDB.select(
-        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")',
+        'SELECT TrackTitle from tracks AS tr INNER JOIN albums AS al ON al.AlbumID = tr.AlbumID WHERE al.ArtistID=? '
-        [newalbumdata['ArtistID']]))
+        'AND al.Status != "Ignored"', [newalbumdata['ArtistID']]))
     havetracks = len(myDB.select(
         'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
         [newalbumdata['ArtistID']]))

headphones/api.py

@@ -82,11 +82,11 @@ class Api(object):
     def fetchData(self):
 
         if self.data == 'OK':
-            logger.info('Recieved API command: %s', self.cmd)
+            logger.info('Received API command: %s', self.cmd)
             methodToCall = getattr(self, "_" + self.cmd)
             methodToCall(**self.kwargs)
             if 'callback' not in self.kwargs:
-                if isinstance(self.data, basestring):
+                if isinstance(self.data, str):
                     return self.data
                 else:
                     return json.dumps(self.data)
@@ -106,7 +106,7 @@ class Api(object):
         rows_as_dic = []
 
         for row in rows:
-            row_as_dic = dict(zip(row.keys(), row))
+            row_as_dic = dict(list(zip(list(row.keys()), row)))
             rows_as_dic.append(row_as_dic)
 
         return rows_as_dic
@@ -474,17 +474,17 @@ class Api(object):
         # Handle situations where the torrent url contains arguments that are
         # parsed
         if kwargs:
-            import urllib
+            import urllib.request, urllib.parse, urllib.error
-            import urllib2
+            import urllib.request, urllib.error, urllib.parse
-            url = urllib2.quote(
+            url = urllib.parse.quote(
-                url, safe=":?/=&") + '&' + urllib.urlencode(kwargs)
+                url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
 
         try:
             result = [(title, int(size), url, provider, kind)]
         except ValueError:
             result = [(title, float(size), url, provider, kind)]
 
-        logger.info(u"Making sure we can download the chosen result")
+        logger.info("Making sure we can download the chosen result")
         (data, bestqual) = searcher.preprocess(result)
 
         if data and bestqual:

headphones/bandcamp.py

@@ -0,0 +1,166 @@
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>
+
+import headphones
+import json
+import os
+import re
+
+from headphones import logger, helpers, metadata, request
+from headphones.common import USER_AGENT
+from headphones.types import Result
+
+from mediafile import MediaFile, UnreadableFileError
+from bs4 import BeautifulSoup
+from bs4 import FeatureNotFound
+
+
+def search(album, albumlength=None, page=1, resultlist=None):
+    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ',
+           '"': '', ',': '', '*': '', '.': '', ':': ''}
+    if resultlist is None:
+        resultlist = []
+
+    cleanalbum = helpers.latinToAscii(
+        helpers.replace_all(album['AlbumTitle'], dic)
+        ).strip()
+    cleanartist = helpers.latinToAscii(
+        helpers.replace_all(album['ArtistName'], dic)
+        ).strip()
+
+    headers = {'User-Agent': USER_AGENT}
+    params = {
+        "page": page,
+        "q": cleanalbum,
+    }
+    logger.info("Looking up https://bandcamp.com/search with {}".format(
+        params))
+    content = request.request_content(
+        url='https://bandcamp.com/search',
+        params=params,
+        headers=headers
+        ).decode('utf8')
+    try:
+        soup = BeautifulSoup(content, "html5lib")
+    except FeatureNotFound:
+        soup = BeautifulSoup(content, "html.parser")
+
+    for item in soup.find_all("li", class_="searchresult"):
+        type = item.find('div', class_='itemtype').text.strip().lower()
+        if type == "album":
+            data = parse_album(item)
+
+            cleanartist_found = helpers.latinToAscii(data['artist'])
+            cleanalbum_found = helpers.latinToAscii(data['album'])
+
+            logger.debug(u"{} - {}".format(data['album'], cleanalbum_found))
+
+            logger.debug("Comparing {} to {}".format(
+                cleanalbum, cleanalbum_found))
+            if (cleanartist.lower() == cleanartist_found.lower() and
+                    cleanalbum.lower() == cleanalbum_found.lower()):
+                resultlist.append(Result(
+                    data['title'], data['size'], data['url'],
+                    'bandcamp', 'bandcamp', True))
+        else:
+            continue
+
+    if(soup.find('a', class_='next')):
+        page += 1
+        logger.debug("Calling next page ({})".format(page))
+        search(album, albumlength=albumlength,
+               page=page, resultlist=resultlist)
+
+    return resultlist
+
+
+def download(album, bestqual):
+    html = request.request_content(url=bestqual.url).decode('utf-8')
+    trackinfo = []
+    try:
+        trackinfo = json.loads(
+            re.search(r"trackinfo&quot;:(\[.*?\]),", html)
+            .group(1)
+            .replace('&quot;', '"'))
+    except ValueError as e:
+        logger.warn("Couldn't load json: {}".format(e))
+
+    directory = os.path.join(
+        headphones.CONFIG.BANDCAMP_DIR,
+        u'{} - {}'.format(
+            album['ArtistName'].replace('/', '_'),
+            album['AlbumTitle'].replace('/', '_')))
+    directory = helpers.latinToAscii(directory)
+
+    if not os.path.exists(directory):
+        try:
+            os.makedirs(directory)
+        except Exception as e:
+            logger.warn("Could not create directory ({})".format(e))
+
+    index = 1
+    for track in trackinfo:
+        filename = helpers.replace_illegal_chars(
+                    u'{:02d} - {}.mp3'.format(index, track['title']))
+        fullname = os.path.join(directory.encode('utf-8'),
+                                filename.encode('utf-8'))
+        logger.debug("Downloading to {}".format(fullname))
+
+        if 'file' in track and track['file'] != None and 'mp3-128' in track['file']:
+            content = request.request_content(track['file']['mp3-128'])
+            open(fullname, 'wb').write(content)
+            try:
+                f = MediaFile(fullname)
+                date, year = metadata._date_year(album)
+                f.update({
+                    'artist': album['ArtistName'].encode('utf-8'),
+                    'album': album['AlbumTitle'].encode('utf-8'),
+                    'title': track['title'].encode('utf-8'),
+                    'track': track['track_num'],
+                    'tracktotal': len(trackinfo),
+                    'year': year,
+                })
+                f.save()
+            except UnreadableFileError as ex:
+                logger.warn("MediaFile couldn't parse: %s (%s)",
+                            fullname,
+                            str(ex))
+
+        index += 1
+
+    return directory
+
+
+def parse_album(item):
+    album = item.find('div', class_='heading').text.strip()
+    artist = item.find('div', class_='subhead').text.strip().replace("by ", "")
+    released = item.find('div', class_='released').text.strip().replace(
+        "released ", "")
+    year = re.search(r"(\d{4})", released).group(1)
+
+    url = item.find('div', class_='heading').find('a')['href'].split("?")[0]
+
+    length = item.find('div', class_='length').text.strip()
+    tracks, minutes = length.split(",")
+    tracks = tracks.replace(" tracks", "").replace(" track", "").strip()
+    minutes = minutes.replace(" minutes", "").strip()
+    # bandcamp offers mp3 128b with should be 960KB/minute
+    size = int(minutes) * 983040
+
+    data = {"title": u'{} - {} [{}]'.format(artist, album, year),
+            "artist": artist, "album": album,
+            "url": url, "size": size}
+
+    return data

headphones/cache.py

@@ -14,11 +14,16 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from six.moves.urllib.parse import urlencode
 
 import headphones
-from headphones import db, helpers, logger, lastfm, request
+from headphones import db, helpers, logger, lastfm, request, mb
 
-LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
+LASTFM_API_KEY = "8d983789c771afaeb7412ac358d4bad0"
+
+FANART_URL = 'https://webservice.fanart.tv/v3/music/'
+FANART_PROJECT_KEY = '22b73c9603eba09d0c855f2d2bdba31c'
+FANART_CLIENT_KEY = '919b389a18a3f0b2c916090022ab3c7a'
 
 
 class Cache(object):
@@ -96,6 +101,11 @@ class Cache(object):
     def _is_current(self, filename=None, date=None):
 
         if filename:
+
+            # 2019 last.fm no longer allows access to artist images, for now we'll keep the cached artist image if it exists and get new from fanart.tv
+            if self.id_type == 'artist' and 'fanart' not in filename:
+                return True
+
             base_filename = os.path.basename(filename)
             date = base_filename.split('.')[1]
 
@@ -211,20 +221,34 @@ class Cache(object):
         if ArtistID:
 
             self.id_type = 'artist'
-            data = lastfm.request_lastfm("artist.getinfo", mbid=ArtistID, api_key=LASTFM_API_KEY)
+
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + ArtistID, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
 
             if not data:
                 return
 
-            try:
+            if data.get('artistthumb'):
-                image_url = data['artist']['image'][-1]['#text']
+                image_url = data['artistthumb'][0]['url']
-            except (KeyError, IndexError):
+            elif data.get('artistbackground'):
-                logger.debug('No artist image found')
+                image_url = data['artistbackground'][0]['url']
-                image_url = None
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
 
-            thumb_url = self._get_thumb_url(data)
+            # fallback to 1st album cover if none of the above
-            if not thumb_url:
+            elif 'albums' in data:
-                logger.debug('No artist thumbnail image found')
+                for mbid, art in list(data.get('albums', dict()).items()):
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            if image_url:
+                thumb_url = image_url
+            else:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
 
@@ -283,6 +307,7 @@ class Cache(object):
         """
 
         myDB = db.DBConnection()
+        fanart = False
 
         # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums
         # Exception is when adding albums manually, then we should use release id
@@ -290,6 +315,14 @@ class Cache(object):
 
             data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY)
 
+            # Try with name if not found
+            if not data:
+                dbartist = myDB.action('SELECT ArtistName, Type FROM artists WHERE ArtistID=?', [self.id]).fetchone()
+                if dbartist:
+                    data = lastfm.request_lastfm("artist.getinfo",
+                                                 artist=helpers.clean_musicbrainz_name(dbartist['ArtistName']),
+                                                 api_key=LASTFM_API_KEY)
+
             if not data:
                 return
 
@@ -303,30 +336,70 @@ class Cache(object):
             except KeyError:
                 logger.debug('No artist bio found')
                 self.info_content = None
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except KeyError:
-                logger.debug('No artist image found')
-                image_url = None
 
-            thumb_url = self._get_thumb_url(data)
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
-            if not thumb_url:
+            image_url = None
-                logger.debug('No artist thumbnail image found')
+            thumb_url = None
+            data = request.request_json(FANART_URL + self.id, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
+
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
+
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in list(data.get('albums', dict()).items()):
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            # finally, use 1st album cover from last.fm
+            if image_url:
+                fanart = True
+                thumb_url = image_url
+            else:
+                dbalbum = myDB.action(
+                    'SELECT ArtworkURL, ThumbURL FROM albums WHERE ArtworkURL IS NOT NULL AND ArtistID=?',
+                    [self.id]).fetchone()
+                if dbalbum:
+                    fanart = True
+                    image_url = dbalbum['ArtworkURL']
+                    thumb_url = dbalbum['ThumbURL']
+
+            if not image_url:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
             dbalbum = myDB.action(
-                'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?',
+                'SELECT ArtistName, AlbumTitle, ReleaseID, Type FROM albums WHERE AlbumID=?',
                 [self.id]).fetchone()
             if dbalbum['ReleaseID'] != self.id:
                 data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'],
                                              api_key=LASTFM_API_KEY)
                 if not data:
-                    data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                    data = lastfm.request_lastfm("album.getinfo",
-                                                 album=dbalbum['AlbumTitle'],
+                                                 artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                 album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
                                                  api_key=LASTFM_API_KEY)
             else:
-                data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                if dbalbum['Type'] != "part of":
-                                             album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
+                    data = lastfm.request_lastfm("album.getinfo",
+                                                artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                api_key=LASTFM_API_KEY)
+                else:
+
+                    # Series, use actual artist for the release-group
+                    artist = mb.getArtistForReleaseGroup(self.id)
+                    if artist:
+                        data = lastfm.request_lastfm("album.getinfo",
+                                                     artist=helpers.clean_musicbrainz_name(artist),
+                                                     album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                     api_key=LASTFM_API_KEY)
 
             if not data:
                 return
@@ -406,7 +479,11 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                artwork_path = os.path.join(self.path_to_art_cache,
+                if fanart:
+                    artwork_path = os.path.join(self.path_to_art_cache,
+                                                self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    artwork_path = os.path.join(self.path_to_art_cache,
                                             self.id + '.' + helpers.today() + ext)
                 try:
                     with open(artwork_path, 'wb') as f:
@@ -422,7 +499,9 @@ class Cache(object):
         # as it's missing/outdated.
         if thumb_url and self.query_type in ['thumb', 'artwork'] and not (
                 self.thumb_files and self._is_current(self.thumb_files[0])):
-            artwork = request.request_content(thumb_url, timeout=20)
+
+            if not (self.query_type == 'artwork' and 'fanart' in thumb_url and artwork):
+                artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
                 # Make sure the artwork dir exists:
@@ -445,11 +524,41 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                thumb_path = os.path.join(self.path_to_art_cache,
+                if fanart:
-                                          'T_' + self.id + '.' + helpers.today() + ext)
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '.' + helpers.today() + ext)
                 try:
-                    with open(thumb_path, 'wb') as f:
+                    if self.id_type != 'artist':
-                        f.write(artwork)
+                        with open(thumb_path, 'wb') as f:
+                            f.write(artwork)
+                    else:
+
+                        # 2019 last.fm no longer allows access to artist images, use the fanart.tv image to create a thumb
+                        artwork_thumb = None
+                        if 'fanart' in thumb_url:
+                            # Create thumb using image resizing service
+                            url = "https://images.weserv.nl"
+                            params = {
+                                "url": thumb_url,
+                                "w": 300
+                            }
+                            headers = {"User-Agent": "Headphones"}
+                            artwork_thumb = request.request_content(
+                                url,
+                                params=params,
+                                timeout=20,
+                                whitelist_status_code=404,
+                                headers=headers
+                            )
+                        if artwork_thumb:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork_thumb)
+                        else:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork)
 
                     os.chmod(thumb_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8))
                 except (OSError, IOError) as e:
@@ -465,7 +574,7 @@ def getArtwork(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         artwork_file = os.path.basename(artwork_path)
@@ -479,7 +588,7 @@ def getThumb(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         thumbnail_file = os.path.basename(artwork_path)

headphones/classes.py

@@ -18,12 +18,12 @@
 #######################################
 
 
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
-from common import USER_AGENT
+from .common import USER_AGENT
 
 
-class HeadphonesURLopener(urllib.FancyURLopener):
+class HeadphonesURLopener(urllib.request.FancyURLopener):
     version = USER_AGENT
 
 
@@ -44,7 +44,7 @@ class AuthURLOpener(HeadphonesURLopener):
         self.numTries = 0
 
         # call the base class
-        urllib.FancyURLopener.__init__(self)
+        urllib.request.FancyURLopener.__init__(self)
 
     def prompt_user_passwd(self, host, realm):
         """

headphones/common.py

@@ -24,6 +24,7 @@ import operator
 import os
 import re
 from headphones import version
+from functools import reduce
 
 
 # Identify Our Application
@@ -74,7 +75,7 @@ class Quality:
     @staticmethod
     def _getStatusStrings(status):
         toReturn = {}
-        for x in Quality.qualityStrings.keys():
+        for x in list(Quality.qualityStrings.keys()):
             toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + \
                                                            Quality.qualityStrings[x] + ")"
         return toReturn
@@ -93,7 +94,7 @@ class Quality:
     def splitQuality(quality):
         anyQualities = []
         bestQualities = []
-        for curQual in Quality.qualityStrings.keys():
+        for curQual in list(Quality.qualityStrings.keys()):
             if curQual & quality:
                 anyQualities.append(curQual)
             if curQual << 16 & quality:
@@ -101,36 +102,6 @@ class Quality:
 
         return (anyQualities, bestQualities)
 
-    @staticmethod
-    def nameQuality(name):
-
-        def checkName(list, func):
-            return func([re.search(x, name, re.I) for x in list])
-
-        name = os.path.basename(name)
-
-        # if we have our exact text then assume we put it there
-        for x in Quality.qualityStrings:
-            if x == Quality.UNKNOWN:
-                continue
-
-            regex = '\W' + Quality.qualityStrings[x].replace(' ', '\W') + '\W'
-            regex_match = re.search(regex, name, re.I)
-            if regex_match:
-                return x
-
-        # TODO: fix quality checking here
-        if checkName(["mp3", "192"], any) and not checkName(["flac"], all):
-            return Quality.B192
-        elif checkName(["mp3", "256"], any) and not checkName(["flac"], all):
-            return Quality.B256
-        elif checkName(["mp3", "vbr"], any) and not checkName(["flac"], all):
-            return Quality.VBR
-        elif checkName(["mp3", "320"], any) and not checkName(["flac"], all):
-            return Quality.B320
-        else:
-            return Quality.UNKNOWN
-
     @staticmethod
     def assumeQuality(name):
         if name.lower().endswith(".mp3"):
@@ -151,28 +122,21 @@ class Quality:
     @staticmethod
     def splitCompositeStatus(status):
         """Returns a tuple containing (status, quality)"""
-        for x in sorted(Quality.qualityStrings.keys(), reverse=True):
+        for x in sorted(list(Quality.qualityStrings.keys()), reverse=True):
             if status > x * 100:
                 return (status - x * 100, x)
 
         return (Quality.NONE, status)
 
-    @staticmethod
-    def statusFromName(name, assume=True):
-        quality = Quality.nameQuality(name)
-        if assume and quality == Quality.UNKNOWN:
-            quality = Quality.assumeQuality(name)
-        return Quality.compositeStatus(DOWNLOADED, quality)
-
     DOWNLOADED = None
     SNATCHED = None
     SNATCHED_PROPER = None
 
 
-Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in Quality.qualityStrings.keys()]
+Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in list(Quality.qualityStrings.keys())]
-Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in Quality.qualityStrings.keys()]
+Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in list(Quality.qualityStrings.keys())]
 Quality.SNATCHED_PROPER = [Quality.compositeStatus(SNATCHED_PROPER, x) for x in
-                           Quality.qualityStrings.keys()]
+                           list(Quality.qualityStrings.keys())]
 
 MP3 = Quality.combineQualities([Quality.B192, Quality.B256, Quality.B320, Quality.VBR], [])
 LOSSLESS = Quality.combineQualities([Quality.FLAC], [])

headphones/config.py

@@ -2,15 +2,16 @@ import itertools
 
 import os
 import re
+import ast
+from configparser import ConfigParser
 import headphones.logger
-from configobj import ConfigObj
 
 
 def bool_int(value):
     """
     Casts a config value into a 0 or 1
     """
-    if isinstance(value, basestring):
+    if isinstance(value, str):
         if value.lower() in ('', '0', 'false', 'f', 'no', 'n', 'off'):
             value = 0
     return int(bool(value))
@@ -34,11 +35,18 @@ _CONFIG_DEFINITIONS = {
     'ADD_ALBUM_ART': (int, 'General', 0),
     'ADVANCEDENCODER': (str, 'General', ''),
     'ALBUM_ART_FORMAT': (str, 'General', 'folder'),
+    'ALBUM_ART_MIN_WIDTH': (str, 'General', ''),
+    'ALBUM_ART_MAX_WIDTH': (str, 'General', ''),
     # This is used in importer.py to determine how complete an album needs to
     # be - to be considered "downloaded". Percentage from 0-100
     'ALBUM_COMPLETION_PCT': (int, 'Advanced', 80),
     'API_ENABLED': (int, 'General', 0),
     'API_KEY': (str, 'General', ''),
+    'ORPHEUS': (int, 'Orpheus.network', 0),
+    'ORPHEUS_PASSWORD': (str, 'Orpheus.network', ''),
+    'ORPHEUS_RATIO': (str, 'Orpheus.network', ''),
+    'ORPHEUS_USERNAME': (str, 'Orpheus.network', ''),
+    'ORPHEUS_URL': (str, 'Orpheus.network', 'https://orpheus.network'),
     'AUTOWANT_ALL': (int, 'General', 0),
     'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1),
     'AUTOWANT_UPCOMING': (int, 'General', 1),
@@ -72,6 +80,7 @@ _CONFIG_DEFINITIONS = {
     'DELUGE_PASSWORD': (str, 'Deluge', ''),
     'DELUGE_LABEL': (str, 'Deluge', ''),
     'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''),
+    'DELUGE_DOWNLOAD_DIRECTORY': (str, 'Deluge', ''),
     'DELUGE_PAUSED': (int, 'Deluge', 0),
     'DESTINATION_DIR': (str, 'General', ''),
     'DETECT_BITRATE': (int, 'General', 0),
@@ -138,16 +147,19 @@ _CONFIG_DEFINITIONS = {
     'IGNORED_FILES': (list, 'Advanced', []),    # path
     'INCLUDE_EXTRAS': (int, 'General', 0),
     'INTERFACE': (str, 'General', 'default'),
+    'JOIN_APIKEY': (str, 'Join', ''),
+    'JOIN_DEVICEID': (str, 'Join', ''),
+    'JOIN_ENABLED': (int, 'Join', 0),
+    'JOIN_ONSNATCH': (int, 'Join', 0),
     'JOURNAL_MODE': (str, 'Advanced', 'wal'),
-    'KAT': (int, 'Kat', 0),
-    'KAT_PROXY_URL': (str, 'Kat', ''),
-    'KAT_RATIO': (str, 'Kat', ''),
     'KEEP_NFO': (int, 'General', 0),
     'KEEP_TORRENT_FILES': (int, 'General', 0),
+    'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
+    'LASTFM_APIKEY': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
-    'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
+    'LIBRARYSCAN_INTERVAL': (int, 'General', 24),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
     'LOG_DIR': (path, 'General', ''),
@@ -155,8 +167,7 @@ _CONFIG_DEFINITIONS = {
     'LOSSLESS_BITRATE_TO': (int, 'General', 0),
     'LOSSLESS_DESTINATION_DIR': (path, 'General', ''),
     'MB_IGNORE_AGE': (int, 'General', 365),
-    'MININOVA': (int, 'Mininova', 0),
+    'MB_IGNORE_AGE_MISSING': (int, 'General', 0),
-    'MININOVA_RATIO': (str, 'Mininova', ''),
     'MIRROR': (str, 'General', 'musicbrainz.org'),
     'MOVE_FILES': (int, 'General', 0),
     'MPC_ENABLED': (bool_int, 'MPC', False),
@@ -192,9 +203,6 @@ _CONFIG_DEFINITIONS = {
     'PIRATEBAY': (int, 'Piratebay', 0),
     'PIRATEBAY_PROXY_URL': (str, 'Piratebay', ''),
     'PIRATEBAY_RATIO': (str, 'Piratebay', ''),
-    'OLDPIRATEBAY': (int, 'Old Piratebay', 0),
-    'OLDPIRATEBAY_URL': (str, 'Old Piratebay', ''),
-    'OLDPIRATEBAY_RATIO': (str, 'Old Piratebay', ''),
     'PLEX_CLIENT_HOST': (str, 'Plex', ''),
     'PLEX_ENABLED': (int, 'Plex', 0),
     'PLEX_NOTIFY': (int, 'Plex', 0),
@@ -226,7 +234,12 @@ _CONFIG_DEFINITIONS = {
     'PUSHOVER_KEYS': (str, 'Pushover', ''),
     'PUSHOVER_ONSNATCH': (int, 'Pushover', 0),
     'PUSHOVER_PRIORITY': (int, 'Pushover', 0),
+    'QBITTORRENT_HOST': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_LABEL': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
+    'RENAME_SINGLE_DISC_IGNORE': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
     'REPLACE_EXISTING_FOLDERS': (int, 'General', 0),
@@ -236,6 +249,7 @@ _CONFIG_DEFINITIONS = {
     'RUTRACKER_PASSWORD': (str, 'Rutracker', ''),
     'RUTRACKER_RATIO': (str, 'Rutracker', ''),
     'RUTRACKER_USER': (str, 'Rutracker', ''),
+    'RUTRACKER_COOKIE': (str, 'Rutracker', ''),
     'SAB_APIKEY': (str, 'SABnzbd', ''),
     'SAB_CATEGORY': (str, 'SABnzbd', ''),
     'SAB_HOST': (str, 'SABnzbd', ''),
@@ -243,13 +257,21 @@ _CONFIG_DEFINITIONS = {
     'SAB_USERNAME': (str, 'SABnzbd', ''),
     'SAMPLINGFREQUENCY': (int, 'General', 44100),
     'SEARCH_INTERVAL': (int, 'General', 1440),
+    'SLACK_ENABLED': (int, 'Slack', 0),
+    'SLACK_URL': (str, 'Slack', ''),
+    'SLACK_CHANNEL': (str, 'Slack', ''),
+    'SLACK_EMOJI': (str, 'Slack', ''),
+    'SLACK_ONSNATCH': (int, 'Slack', 0),
     'SOFT_CHROOT': (path, 'General', ''),
     'SONGKICK_APIKEY': (str, 'Songkick', 'nd1We7dFW2RqxPw8'),
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
-    'STRIKE': (int, 'Strike', 0),
+    'SOULSEEK_API_URL': (str, 'Soulseek', ''),
-    'STRIKE_RATIO': (str, 'Strike', ''),
+    'SOULSEEK_API_KEY': (str, 'Soulseek', ''),
+    'SOULSEEK_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK_INCOMPLETE_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK': (int, 'Soulseek', 0),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
@@ -266,6 +288,7 @@ _CONFIG_DEFINITIONS = {
     'TORZNAB_APIKEY': (str, 'Torznab', ''),
     'TORZNAB_ENABLED': (int, 'Torznab', 1),
     'TORZNAB_HOST': (str, 'Torznab', ''),
+    'TORZNAB_RATIO': (str, 'Torznab', ''),
     'TRANSMISSION_HOST': (str, 'Transmission', ''),
     'TRANSMISSION_PASSWORD': (str, 'Transmission', ''),
     'TRANSMISSION_USERNAME': (str, 'Transmission', ''),
@@ -282,21 +305,21 @@ _CONFIG_DEFINITIONS = {
     'UTORRENT_USERNAME': (str, 'uTorrent', ''),
     'VERIFY_SSL_CERT': (bool_int, 'Advanced', 1),
     'WAIT_UNTIL_RELEASE_DATE': (int, 'General', 0),
-    'WAFFLES': (int, 'Waffles', 0),
+    'REDACTED': (int, 'Redacted', 0),
-    'WAFFLES_PASSKEY': (str, 'Waffles', ''),
+    'REDACTED_APIKEY': (str, 'Redacted', ''),
-    'WAFFLES_RATIO': (str, 'Waffles', ''),
+    'REDACTED_USERNAME': (str, 'Redacted', ''),
-    'WAFFLES_UID': (str, 'Waffles', ''),
+    'REDACTED_PASSWORD': (str, 'Redacted', ''),
-    'WHATCD': (int, 'What.cd', 0),
+    'REDACTED_RATIO': (str, 'Redacted', ''),
-    'WHATCD_PASSWORD': (str, 'What.cd', ''),
+    'REDACTED_USE_FLTOKEN': (int, 'Redacted', 0),
-    'WHATCD_RATIO': (str, 'What.cd', ''),
-    'WHATCD_USERNAME': (str, 'What.cd', ''),
     'XBMC_ENABLED': (int, 'XBMC', 0),
     'XBMC_HOST': (str, 'XBMC', ''),
     'XBMC_NOTIFY': (int, 'XBMC', 0),
     'XBMC_PASSWORD': (str, 'XBMC', ''),
     'XBMC_UPDATE': (int, 'XBMC', 0),
     'XBMC_USERNAME': (str, 'XBMC', ''),
-    'XLDPROFILE': (str, 'General', '')
+    'XLDPROFILE': (str, 'General', ''),
+    'BANDCAMP': (int, 'General', 0),
+    'BANDCAMP_DIR': (path, 'General', '')
 }
 
 
@@ -308,8 +331,9 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigObj(self._config_file, encoding='utf-8')
+        self._config = ConfigParser(interpolation=None)
-        for key in _CONFIG_DEFINITIONS.keys():
+        self._config.read(self._config_file)
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
         self.ENCODER_MULTICORE_COUNT = max(0, self.ENCODER_MULTICORE_COUNT)
         self._upgrade()
@@ -326,7 +350,7 @@ class Config(object):
 
     def check_section(self, section):
         """ Check if INI section exists, if not create it """
-        if section not in self._config:
+        if not self._config.has_section(section):
             self._config[section] = {}
             return True
         else:
@@ -336,28 +360,38 @@ class Config(object):
         """ Cast any value in the config to the right type or use the default """
         key, definition_type, section, ini_key, default = self._define(key)
         self.check_section(section)
+
+        # ConfigParser values are strings, so need to convert to actual list
+        if definition_type == list:
+            definition_type = ast.literal_eval
+
         try:
             my_val = definition_type(self._config[section][ini_key])
+            # ConfigParser interprets quotes in the config
+            # literally, so we need to sanitize it. It's not really
+            # a config upgrade, since a user can at any time put 
+            # some_key = 'some_val'
+            if type(my_val) in [str, path]:
+                my_val = my_val.strip('"').strip("'")
         except Exception:
-            my_val = definition_type(default)
+            my_val = default
-            self._config[section][ini_key] = my_val
+            self._config[section][ini_key] = str(my_val)
         return my_val
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigObj(encoding="UTF-8")
+        new_config = ConfigParser(interpolation=None)
-        new_config.filename = self._config_file
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data
-        for key, subkeys in self._config.items():
+        for key, subkeys in list(self._config.items()):
             if key not in new_config:
                 new_config[key] = {}
-            for subkey, value in subkeys.items():
+            for subkey, value in list(subkeys.items()):
                 new_config[key][subkey] = value
 
         # next make sure that everything we expect to have defined is so
-        for key in _CONFIG_DEFINITIONS.keys():
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             key, definition_type, section, ini_key, default = self._define(key)
             self.check_setting(key)
             if section not in new_config:
@@ -368,14 +402,15 @@ class Config(object):
         headphones.logger.info("Writing configuration to file")
 
         try:
-            new_config.write()
+            with open(self._config_file, 'w') as configfile:
+                new_config.write(configfile)
         except IOError as e:
             headphones.logger.error("Error writing configuration file: %s", e)
 
     def get_extra_newznabs(self):
         """ Return the extra newznab tuples """
         extra_newznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
+            zip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
                              for i in range(3)])
         )
         return extra_newznabs
@@ -394,8 +429,8 @@ class Config(object):
     def get_extra_torznabs(self):
         """ Return the extra torznab tuples """
         extra_torznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+            zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
-                             for i in range(3)])
+                             for i in range(4)])
         )
         return extra_torznabs
 
@@ -430,20 +465,21 @@ class Config(object):
             return value
         else:
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
             return self._config[section][ini_key]
 
     def process_kwargs(self, kwargs):
         """
         Given a big bunch of key value pairs, apply them to the ini.
         """
-        for name, value in kwargs.items():
+        for name, value in list(kwargs.items()):
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
 
     def _upgrade(self):
         """
-        Bring old configs up to date
+        Bring old configs up to date. Although this is kind of a dumb
+        way to do it because it doesn't handle multi-step upgrades
         """
         if self.CONFIG_VERSION == '2':
             # Update the config to use direct path to the encoder rather than the encoder folder
@@ -466,4 +502,17 @@ class Config(object):
         if self.CONFIG_VERSION == '5':
             if self.OPEN_MAGNET_LINKS:
                 self.MAGNET_LINKS = 2
-            self.CONFIG_VERSION = '5'
+
+            # Add Seed Ratio to Torznabs
+            if self.EXTRA_TORZNABS:
+                extra_torznabs = list(
+                    zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+                                     for i in range(3)])
+                )
+                new_torznabs = []
+                for torznab in extra_torznabs:
+                    new_torznabs.extend([torznab[0], torznab[1], '', torznab[2]])
+                if new_torznabs:
+                    self.EXTRA_TORZNABS = new_torznabs
+
+            self.CONFIG_VERSION = '6'

headphones/config_test.py

@@ -2,8 +2,8 @@ import mock
 from mock import MagicMock
 import headphones.config
 import re
-import unittestcompat
+from . import unittestcompat
-from unittestcompat import TestCase, TestArgs
+from .unittestcompat import TestCase, TestArgs
 
 
 class ConfigApiTest(TestCase):
@@ -101,7 +101,7 @@ class ConfigApiTest(TestCase):
         # call methods
         c = headphones.config.Config(path)
         # assertions:
-        with self.assertRaisesRegexp(KeyError, exc_regex):
+        with self.assertRaisesRegex(KeyError, exc_regex):
             c.check_setting(setting_name)
         pass
 
@@ -355,86 +355,89 @@ class ConfigApiTest(TestCase):
     #   TORZNABS
     # TODO : here is copypaste from of NEZNABS tests. Make tests better, plz refactor them
     #
-    @TestArgs(
+    # TODO: Fix tests for following:
-        ('', []),
+    #   CONFIG_VERSION == '5' each entry = 'host, api, enabled'
-        ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
+    #   CONFIG_VERSION > '5' each entry = 'host, api, seed ratio, enabled'
-        (['ABC', 'DEF'], []),
+    # @TestArgs(
-        ([1], []),
+    #     ('', []),
-        ([1, 2], []),
+    #     ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
-        ([1, 2, 3], [(1, 2, 3)]),
+    #     (['ABC', 'DEF'], []),
-
+    #     ([1], []),
-        ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
+    #     ([1, 2], []),
-        ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
+    #     ([1, 2, 3], [(1, 2, 3)]),
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    #
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    #     ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
-    )
+    #     ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
-    def test_get_extra_torznabs(self, conf_value, expected):
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-        """ Config: get_extra_torznabs """
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_get_extra_torznabs(self, conf_value, expected):
-        # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
+    #     """ Config: get_extra_torznabs """
-        # set up mocks:
+    #     path = '/tmp/notexist'
-        # 'EXTRA_TORZNABS': (list, '', ''),
+    #
-        self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
+    #     # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
-
+    #     # set up mocks:
-        # call methods
+    #     # 'EXTRA_TORZNABS': (list, '', ''),
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
-        res = c.get_extra_torznabs()
+    #
-
+    #     # call methods
-        # assertions:
+    #     c = headphones.config.Config(path)
-        self.assertEqual(res, expected)
+    #     res = c.get_extra_torznabs()
-
+    #
-    def test_clear_extra_torznabs(self):
+    #     # assertions:
-        """ Config: clear_extra_torznabs """
+    #     self.assertEqual(res, expected)
-        path = '/tmp/notexist'
+    #
-
+    # def test_clear_extra_torznabs(self):
-        random_value = -1292721
+    #     """ Config: clear_extra_torznabs """
-        self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
+    #     path = '/tmp/notexist'
-        self.config_mock["Torznab"] = {"do_not_touch": random_value}
+    #
-
+    #     random_value = -1292721
-        # call methods
+    #     self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"do_not_touch": random_value}
-        res = c.clear_extra_torznabs()
+    #
-
+    #     # call methods
-        # assertions:
+    #     c = headphones.config.Config(path)
-        self.assertIsNone(res)
+    #     res = c.clear_extra_torznabs()
-        self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
+    #
-        self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
+    #     # assertions:
-
+    #     self.assertIsNone(res)
-    @TestArgs(
+    #     self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
-        ([], [''], ['']),
+    #     self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
-        ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
+    #
-
+    # @TestArgs(
-        ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
+    #     ([], [''], ['']),
-    )
+    #     ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
-    def test_add_extra_torznab(self, initial, added, expected):
+    #
-        """ Config: add_extra_torznab """
+    #     ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_add_extra_torznab(self, initial, added, expected):
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #     """ Config: add_extra_torznab """
-
+    #     path = '/tmp/notexist'
-        # call methods
+    #
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
-        c.add_extra_torznab(added)
+    #
-        act = self.config_mock["Torznab"]["extra_torznabs"]
+    #     # call methods
-
+    #     c = headphones.config.Config(path)
-        # assertions:
+    #     c.add_extra_torznab(added)
-        self.assertEqual(act, expected)
+    #     act = self.config_mock["Torznab"]["extra_torznabs"]
-
+    #
-    @TestArgs(
+    #     # assertions:
-        (None),
+    #     self.assertEqual(act, expected)
-        ([]),
+    #
-        ([1, 2, 3]),
+    # @TestArgs(
-        ([True]),
+    #     (None),
-    )
+    #     ([]),
-    def test_add_extra_torznab_raise_on_none(self, initial):
+    #     ([1, 2, 3]),
-        """ Config: add_extra_torznab should raise on None adding"""
+    #     ([True]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_add_extra_torznab_raise_on_none(self, initial):
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #     """ Config: add_extra_torznab should raise on None adding"""
-
+    #     path = '/tmp/notexist'
-        # call methods
+    #
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
-        with self.assertRaises(TypeError):
+    #
-            c.add_extra_torznab(None)
+    #     # call methods
-        pass
+    #     c = headphones.config.Config(path)
+    #     with self.assertRaises(TypeError):
+    #         c.add_extra_torznab(None)
+    #     pass

headphones/crier.py

@@ -0,0 +1,36 @@
+import pprint
+import sys
+import threading
+import traceback
+
+from headphones import logger
+
+
+def cry():
+    """
+    Logs thread traces.
+    """
+    tmap = {}
+    main_thread = None
+    # get a map of threads by their ID so we can print their names
+    # during the traceback dump
+    for t in threading.enumerate():
+        if t.ident:
+            tmap[t.ident] = t
+        else:
+            main_thread = t
+
+    # Loop over each thread's current frame, writing info about it
+    for tid, frame in sys._current_frames().items():
+        thread = tmap.get(tid, main_thread)
+
+        lines = []
+        lines.append('%s\n' % thread.getName())
+        lines.append('========================================\n')
+        lines += traceback.format_stack(frame)
+        lines.append('========================================\n')
+        lines.append('LOCAL VARIABLES:\n')
+        lines.append('========================================\n')
+        lines.append(pprint.pformat(frame.f_locals))
+        lines.append('\n\n')
+        logger.info("".join(lines))

headphones/cuesplit.py

@@ -87,7 +87,7 @@ def check_splitter(command):
 
 def split_baby(split_file, split_cmd):
     '''Let's split baby'''
-    logger.info('Splitting %s...', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Splitting {split_file}...")
     logger.debug(subprocess.list2cmdline(split_cmd))
 
     # Prevent Windows from opening a terminal window
@@ -108,16 +108,16 @@ def split_baby(split_file, split_cmd):
 
     process = subprocess.Popen(split_cmd, startupinfo=startupinfo,
                                stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE, env=env)
+                               stderr=subprocess.PIPE, env=env, text=True)
     stdout, stderr = process.communicate()
 
     if process.returncode:
-        logger.error('Split failed for %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.error(f"Split failed for {split_file}")
-        out = stdout if stdout else stderr
+        out = stdout or stderr
-        logger.error('Error details: %s', out.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.error(f"Error details: {out}")
         return False
     else:
-        logger.info('Split success %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.info(f"Split succeeded for {split_file}")
         return True
 
 
@@ -232,7 +232,7 @@ class Directory:
         for i in list_dir:
             if not check_match(i):
                 # music file
-                if os.path.splitext(i)[-1] in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+                if os.path.splitext(i)[-1] in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
                     track_nr = identify_track_number(i)
                     if track_nr:
                         self.content.append(WaveFile(self.path + os.sep + i, track_nr=track_nr))
@@ -378,7 +378,7 @@ class CueFile(File):
             except:
                 raise ValueError('Cant encode CUE Sheet.')
 
-        if self.content[0] == u'\ufeff':
+        if self.content[0] == '\ufeff':
             self.content = self.content[1:]
 
         header = header_parser()
@@ -581,7 +581,7 @@ def split(albumpath):
 
     # use xld profile to split cue
     if headphones.CONFIG.ENCODER == 'xld' and headphones.CONFIG.MUSIC_ENCODER and headphones.CONFIG.XLDPROFILE:
-        import getXldProfile
+        from . import getXldProfile
         xldprofile, xldformat, _ = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE)
         if not xldformat:
             raise ValueError(
@@ -601,7 +601,7 @@ def split(albumpath):
         raise ValueError('Command not found, ensure shntool or xld installed')
 
     # Determine if file can be split
-    if wave.name_ext not in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+    if wave.name_ext not in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
         raise ValueError('Cannot split, audio file has unsupported extension')
 
     # Split with xld

headphones/db.py

@@ -17,7 +17,9 @@
 # Stolen from Sick-Beard's db.py  #
 ###################################
 
-from __future__ import with_statement
+
+
+import time
 
 import sqlite3
 
@@ -45,36 +47,97 @@ class DBConnection:
         self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
         # don't wait for the disk to finish writing
         self.connection.execute("PRAGMA synchronous = OFF")
-        # journal disabled since we never do rollbacks
+        # default set to Write-Ahead Logging WAL
         self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE)
         # 64mb of cache memory,probably need to make it user configurable
         self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024))
         self.connection.row_factory = sqlite3.Row
 
-    def action(self, query, args=None):
+    def action(self, query, args=None, upsert_insert_qry=None):
 
         if query is None:
             return
 
         sqlResult = None
+        attempts = 0
+        dberror = None
 
-        try:
+        while attempts < 10:
-            with self.connection as c:
+            try:
-                if args is None:
+                with self.connection as c:
-                    sqlResult = c.execute(query)
+
+                    # log that previous attempt was locked and we're trying again
+                    if dberror:
+                        if args is None:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s', attempts + 1, query)
+                        else:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s. Args: %s', attempts + 1, query, args)
+
+                    # debugging
+                    # try:
+                    #     explain_query = 'EXPLAIN QUERY PLAN ' + query
+                    #     if not args:
+                    #         sql_results = c.execute(explain_query)
+                    #     else:
+                    #         sql_results = c.execute(explain_query, args)
+                    #     if not args:
+                    #         print(explain_query)
+                    #     else:
+                    #         print(explain_query + ' ' + str(args))
+                    #     explain_results = sql_results
+                    #     for row in explain_results:
+                    #         print row
+                    # except Exception as e:
+                    #     print(e)
+
+                    # Execute query
+
+                    # time0 = time.time()
+
+                    if args is None:
+                        sqlResult = c.execute(query)
+                        # logger.debug('SQL: ' + query)
+                    else:
+                        sqlResult = c.execute(query, args)
+                        # logger.debug('SQL: %s. Args: %s', query, args)
+
+                        # INSERT part of upsert query
+                        if upsert_insert_qry:
+                            sqlResult = c.execute(upsert_insert_qry, args)
+                            # logger.debug('SQL: %s. Args: %s', upsert_insert_qry, args)
+
+                    # debugging: loose test to log queries taking longer than 5 seconds
+                    # seconds = time.time() - time0
+                    # if seconds > 5:
+                    #     if args is None:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s", seconds, query)
+                    #     else:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s with args %s", seconds, query, args)
+
+                    break
+
+            except sqlite3.OperationalError as e:
+                if "unable to open database file" in str(e) or "database is locked" in str(e):
+                    dberror = e
+                    if args is None:
+                        logger.debug('Database error: %s. Query: %s', e, query)
+                    else:
+                        logger.debug('Database error: %s. Query: %s. Args: %s', e, query, args)
+                    attempts += 1
+                    time.sleep(1)
                 else:
-                    sqlResult = c.execute(query, args)
+                    logger.error('Database error: %s', e)
-
+                    raise
-        except sqlite3.OperationalError, e:
+            except sqlite3.DatabaseError as e:
-            if "unable to open database file" in e.message or "database is locked" in e.message:
+                logger.error('Fatal Error executing %s :: %s', query, e)
-                logger.warn('Database Error: %s', e)
-            else:
-                logger.error('Database error: %s', e)
                 raise
 
-        except sqlite3.DatabaseError, e:
+        # log if no results returned due to lock
-            logger.error('Fatal Error executing %s :: %s', query, e)
+        if not sqlResult and attempts:
-            raise
+            if args is None:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s', dberror, query)
+            else:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s. Args: %s', dberror, query, args)
 
         return sqlResult
 
@@ -88,24 +151,19 @@ class DBConnection:
         return sqlResults
 
     def upsert(self, tableName, valueDict, keyDict):
-
+        """
+        Transactions an Update or Insert to a table based on key.
+        If the table is not updated then the 'WHERE changes' will be 0 and the table inserted
+        """
         def genParams(myDict):
-            return [x + " = ?" for x in myDict.keys()]
+            return [x + " = ?" for x in list(myDict.keys())]
 
-        changesBefore = self.connection.total_changes
+        update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
 
-        update_query = "UPDATE " + tableName + " SET " + ", ".join(
+        insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(list(valueDict.keys()) + list(keyDict.keys())) + ")" + " SELECT " + ", ".join(
-            genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
+            ["?"] * len(list(valueDict.keys()) + list(keyDict.keys()))) + " WHERE changes()=0")
 
-        self.action(update_query, valueDict.values() + keyDict.values())
+        try:
-
+            self.action(update_query, list(valueDict.values()) + list(keyDict.values()), upsert_insert_qry=insert_query)
-        if self.connection.total_changes == changesBefore:
+        except sqlite3.IntegrityError:
-            insert_query = (
+            logger.info('Queries failed: %s and %s', update_query, insert_query)
-                "INSERT INTO " + tableName + " (" + ", ".join(
-                    valueDict.keys() + keyDict.keys()) + ")" +
-                " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
-            )
-            try:
-                self.action(insert_query, valueDict.values() + keyDict.values())
-            except sqlite3.IntegrityError:
-                logger.info('Queries failed: %s and %s', update_query, insert_query)

headphones/deluge.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 #  This file is part of Headphones.
 #
 #  Headphones is free software: you can redistribute it and/or modify
@@ -32,7 +34,7 @@
 # You should have received a copy of the GNU General Public License
 # along with SickRage. If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import unicode_literals
+
 
 from headphones import logger
 
@@ -49,18 +51,19 @@ delugeweb_auth = {}
 delugeweb_url = ''
 deluge_verify_cert = False
 scrub_logs = True
+headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 
 
 def _scrubber(text):
     if scrub_logs:
         try:
             # URL parameter values
-            text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text)
+            text = re.sub(r'=[0-9a-zA-Z]*', r'=REMOVED', text)
             # Local host with port
             # text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # just host
-            text = re.sub('\:\/\/.*\:[0-9]*', '://REMOVED:', text)
+            text = re.sub(r'\:\/\/.*\:[0-9]*', r'://REMOVED:', text)
             # Session cookie
-            text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text)
+            text = re.sub(r"_session_id'\: '.*'", r"_session_id': 'REMOVED'", text)
             # Local Windows user path
             if text.lower().startswith('c:\\users\\'):
                 k = text.split('\\')
@@ -80,8 +83,7 @@ def addTorrent(link, data=None, name=None):
 
         result = {}
         retid = False
-        url_what = ['https://what.cd/', 'http://what.cd/']
+        url_orpheus = ['https://orpheus.network/', 'http://orpheus.network/']
-        url_waffles = ['https://waffles.fm/', 'http://waffles.fm/']
 
         if link.lower().startswith('magnet:'):
             logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link))
@@ -91,10 +93,7 @@ def addTorrent(link, data=None, name=None):
 
         elif link.lower().startswith('http://') or link.lower().startswith('https://'):
             logger.debug('Deluge: Got a URL: %s' % _scrubber(link))
-            if link.lower().startswith(tuple(url_waffles)):
+            if link.lower().startswith(tuple(url_orpheus)):
-                if 'rss=' not in link:
-                    link = link + '&rss=1'
-            if link.lower().startswith(tuple(url_what)):
                 logger.debug('Deluge: Using different User-Agent for this site')
                 user_agent = 'Headphones'
                 # This method will make Deluge download the file
@@ -104,11 +103,11 @@ def addTorrent(link, data=None, name=None):
                 # return addTorrent(local_torrent_path)
             else:
                 user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
-            headers = {'User-Agent': user_agent}
+            get_headers = {'User-Agent': user_agent}
             torrentfile = ''
             logger.debug('Deluge: Trying to download (GET)')
             try:
-                r = requests.get(link, headers=headers)
+                r = requests.get(link, headers=get_headers)
                 if r.status_code == 200:
                     logger.debug('Deluge: 200 OK')
                     # .text will ruin the encoding for some torrents
@@ -125,9 +124,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -135,7 +134,10 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
-            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            try:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except:
+                logger.debug('Deluge: Sending Deluge torrent with problematic name and some content')
             result = {'type': 'torrent',
                         'name': name,
                         'content': torrentfile}
@@ -154,9 +156,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -164,7 +166,10 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
-            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            try:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except UnicodeDecodeError:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name.decode('utf-8'), str(torrentfile)[:40]))
             result = {'type': 'torrent',
                         'name': name,
                         'content': torrentfile}
@@ -199,7 +204,7 @@ def getTorrentFolder(result):
                                 ],
                                 "id": 21})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['total_done'] = json.loads(response.text)['result']['total_done']
 
         tries = 0
@@ -207,7 +212,7 @@ def getTorrentFolder(result):
             tries += 1
             time.sleep(5)
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             result['total_done'] = json.loads(response.text)['result']['total_done']
 
         post_data = json.dumps({"method": "web.get_torrent_status",
@@ -226,7 +231,7 @@ def getTorrentFolder(result):
                                 "id": 23})
 
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
 
         result['save_path'] = json.loads(response.text)['result']['save_path']
         result['name'] = json.loads(response.text)['result']['name']
@@ -242,18 +247,45 @@ def removeTorrent(torrentid, remove_data=False):
         _get_auth()
 
     try:
-        result = False
+        logger.debug('Deluge: Checking if torrent %s finished seeding' % str(torrentid))
-        post_data = json.dumps({"method": "core.remove_torrent",
+        post_data = json.dumps({"method": "web.get_torrent_status",
                                 "params": [
                                     torrentid,
-                                    remove_data
+                                    [
-                                    ],
+                                        "name",
-                                "id": 25})
+                                        "ratio",
-        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+                                        "state"
-            verify=deluge_verify_cert)
+                                    ]
-        result = json.loads(response.text)['result']
+                                ],
+                                "id": 26})
 
-        return result
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+            verify=deluge_verify_cert, headers=headers)
+
+        try:
+            state = json.loads(response.text)['result']['state']
+        except KeyError as e:
+            logger.debug('Deluge: "state" KeyError when trying to remove torrent %s' % str(torrentid))
+            return False
+
+        not_finished = ["queued", "seeding", "downloading", "checking", "error"]
+        result = False
+        if state.lower() in not_finished:
+            logger.debug('Deluge: Torrent %s is either queued or seeding, not removing yet' % str(torrentid))
+            return False
+        else:
+            logger.debug('Deluge: Removing torrent %s' % str(torrentid))
+            post_data = json.dumps({"method": "core.remove_torrent",
+                                    "params": [
+                                        torrentid,
+                                        remove_data
+                                        ],
+                                    "id": 25})
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+                verify=deluge_verify_cert, headers=headers)
+            result = json.loads(response.text)['result']
+
+            return result
     except Exception as e:
         logger.error('Deluge: Removing torrent failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -269,7 +301,8 @@ def _get_auth():
     delugeweb_host = headphones.CONFIG.DELUGE_HOST
     delugeweb_cert = headphones.CONFIG.DELUGE_CERT
     delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD
-    logger.debug('Deluge: Using password %s******%s' % (delugeweb_password[0], delugeweb_password[-1]))
+    if len(delugeweb_password) > 0:
+        logger.debug('Deluge: Using password %s******%s' % (delugeweb_password[0], delugeweb_password[-1]))
 
     if not delugeweb_host.startswith('http'):
         delugeweb_host = 'http://%s' % delugeweb_host
@@ -292,12 +325,12 @@ def _get_auth():
                             "id": 1})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
     except requests.ConnectionError:
         try:
             logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case')
             response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             # If the previous line didn't fail, change delugeweb_url for the rest of this session
             logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!')
             delugeweb_url = delugeweb_url.replace('http:', 'https:')
@@ -322,7 +355,7 @@ def _get_auth():
                             "id": 10})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
     except Exception as e:
         logger.error('Deluge: Authentication failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -339,7 +372,7 @@ def _get_auth():
                                 "id": 11})
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -347,7 +380,8 @@ def _get_auth():
             return None
 
         delugeweb_hosts = json.loads(response.text)['result']
-        if len(delugeweb_hosts) == 0:
+        # Check if delugeweb_hosts is None before checking its length
+        if not delugeweb_hosts or len(delugeweb_hosts) == 0:
             logger.error('Deluge: WebUI does not contain daemons')
             return None
 
@@ -357,7 +391,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -370,7 +404,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -395,7 +429,7 @@ def _add_torrent_magnet(result):
                                 "params": [result['url'], {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -415,7 +449,7 @@ def _add_torrent_url(result):
                                 "params": [result['url'], {}],
                                 "id": 32})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['location'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -428,35 +462,36 @@ def _add_torrent_url(result):
 
 def _add_torrent_file(result):
     logger.debug('Deluge: Adding file')
+
+    options = {}
+
+    if headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY:
+        options['download_location'] = headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY
+
+    if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
+        options['move_completed'] = 1
+        if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
+            options['move_completed_path'] = headphones.CONFIG.DELUGE_DONE_DIRECTORY
+        else:
+            options['move_completed_path'] = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
+
+    if headphones.CONFIG.DELUGE_PAUSED:
+        options['add_paused'] = headphones.CONFIG.DELUGE_PAUSED
+
     if not any(delugeweb_auth):
         _get_auth()
     try:
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
-                                "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}],
+                                "params": [result['name'] + '.torrent',
+                                b64encode(result['content']).decode(),
+                                options],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
-    except UnicodeDecodeError:
-        try:
-            # content is torrent file contents that needs to be encoded to base64
-            # this time let's try leaving the encoding as is
-            post_data = json.dumps({"method": "core.add_torrent_file",
-                                    "params": [result['name'] + '.torrent', b64encode(result['content']), {}],
-                                    "id": 22})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
-            result['hash'] = json.loads(response.text)['result']
-            logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
-            return json.loads(response.text)['result']
-        except Exception as e:
-            logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e))
-            formatted_lines = traceback.format_exc().splitlines()
-            logger.error('; '.join(formatted_lines))
-            return False
     except Exception as e:
         logger.error('Deluge: Adding torrent file failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -480,7 +515,7 @@ def setTorrentLabel(result):
                                 "params": [],
                                 "id": 3})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         labels = json.loads(response.text)['result']
 
         if labels is not None:
@@ -491,7 +526,7 @@ def setTorrentLabel(result):
                                             "params": [label],
                                             "id": 4})
                     response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                        verify=deluge_verify_cert)
+                        verify=deluge_verify_cert, headers=headers)
                     logger.debug('Deluge: %s label added to Deluge' % label)
                 except Exception as e:
                     logger.error('Deluge: Setting label failed: %s' % str(e))
@@ -503,7 +538,7 @@ def setTorrentLabel(result):
                                     "params": [result['hash'], label],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             logger.debug('Deluge: %s label added to torrent' % label)
         else:
             logger.debug('Deluge: Label plugin not detected')
@@ -527,12 +562,12 @@ def setSeedRatio(result):
                                     "params": [result['hash'], True],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             post_data = json.dumps({"method": "core.set_torrent_stop_ratio",
                                     "params": [result['hash'], float(ratio)],
                                     "id": 6})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -544,61 +579,3 @@ def setSeedRatio(result):
         return None
 
 
-def setTorrentPath(result):
-    logger.debug('Deluge: Setting download path')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
-            post_data = json.dumps({"method": "core.set_torrent_move_completed",
-                                    "params": [result['hash'], True],
-                                    "id": 7})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
-
-            if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
-                move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
-            else:
-                move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
-
-            if not os.path.exists(move_to):
-                logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to)
-                os.makedirs(move_to)
-            post_data = json.dumps({"method": "core.set_torrent_move_completed_path",
-                                    "params": [result['hash'], move_to],
-                                    "id": 8})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent move-to directory failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None
-
-
-def setTorrentPause(result):
-    logger.debug('Deluge: Pausing torrent')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_PAUSED:
-            post_data = json.dumps({"method": "core.pause_torrent",
-                                    "params": [[result['hash']]],
-                                    "id": 9})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent paused failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None

headphones/getXldProfile.py

@@ -1,6 +1,6 @@
 import os.path
 
-import biplist
+import plistlib
 from headphones import logger
 
 
@@ -14,8 +14,9 @@ def getXldProfile(xldProfile):
 
     # Get xld preferences plist
     try:
-        preferences = biplist.readPlist(expanded)
+        with open(expanded, 'rb') as _f:
-    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException), e:
+            preferences = plistlib.load(_f)
+    except Exception as e:
         logger.error("Error reading xld preferences plist: %s", e)
         return (xldProfileNotFound, None, None)
 

headphones/helpers.py

@@ -14,17 +14,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from operator import itemgetter
-import unicodedata
-import datetime
-import shutil
-import time
-import sys
-
-import fnmatch
-import re
 import os
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+import re
+import shutil
+import sys
+import tempfile
+import time
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, date
+from fnmatch import fnmatch
+from functools import cmp_to_key
+from glob import glob
+from operator import itemgetter
+
+from beets import logging as beetslogging
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from six import text_type
+from unidecode import unidecode
+
 import headphones
 
 
@@ -34,6 +42,24 @@ RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.")
 RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)?", re.I)
 RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
 
+def cmp(x, y):
+    """
+    Replacement for built-in function cmp that was removed in Python 3
+
+    Compare the two objects x and y and return an integer according to
+    the outcome. The return value is negative if x < y, zero if x == y
+    and strictly positive if x > y.
+
+    https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function
+    """
+    if x is None and y is None:
+        return 0
+    elif x is None:
+        return -1
+    elif y is None:
+        return 1
+    else:
+        return (x > y) - (x < y)
 
 def multikeysort(items, columns):
     comparers = [
@@ -48,7 +74,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, cmp=comparer)
+    return sorted(items, key=cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -130,28 +156,25 @@ def convert_seconds(s):
 
 
 def today():
-    today = datetime.date.today()
+    return date.isoformat(date.today())
-    yyyymmdd = datetime.date.isoformat(today)
-    return yyyymmdd
 
 
 def now():
-    now = datetime.datetime.now()
+    now = datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def get_age(date):
+def is_valid_date(d):
-    try:
+    if not d:
-        split_date = date.split('-')
-    except:
         return False
+    else:
+        return bool(re.match(r'\d{4}-\d{2}-\d{2}', d))
 
-    try:
-        days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2])
-    except (IndexError, ValueError):
-        days_old = False
 
-    return days_old
+def age(d):
+    '''Requires a valid date'''
+    delta = date.today() - date.fromisoformat(d)
+    return delta.days
 
 
 def bytes_to_mb(bytes):
@@ -161,7 +184,7 @@ def bytes_to_mb(bytes):
 
 
 def mb_to_bytes(mb_str):
-    result = re.search('^(\d+(?:\.\d+)?)\s?(?:mb)?', mb_str, flags=re.I)
+    result = re.search(r"^(\d+(?:\.\d+)?)\s?(?:mb)?", mb_str, flags=re.I)
     if result:
         return int(float(result.group(1)) * 1048576)
 
@@ -191,14 +214,20 @@ def piratesize(size):
     return size
 
 
-def replace_all(text, dic, normalize=False):
+def pattern_substitute(pattern, dic, normalize=False):
+    """
+    Execute path rendering/substitution based on replacement dictionary
+    e.g. pattern = $Artist/$Album
+         dic = {Artist: 'My artist', Album: 'My album'}
+         returns My artist/My album
+    """
     from headphones import pathrender
-    if not text:
+    if not pattern:
         return ''
 
     if normalize:
         new_dic = {}
-        for i, j in dic.iteritems():
+        for i, j in dic.items():
             if j is not None:
                 try:
                     if sys.platform == 'darwin':
@@ -210,20 +239,28 @@ def replace_all(text, dic, normalize=False):
                         j.decode(headphones.SYS_ENCODING, 'replace'))
             new_dic[i] = j
         dic = new_dic
-    return pathrender.render(text, dic)[0]
+    return pathrender.render(pattern, dic)[0]
+
+
+def replace_all(text, dic):
+    if not text:
+        return ''
+
+    for i, j in dic.items():
+        text = text.replace(i, j)
+    return text
 
 
 def replace_illegal_chars(string, type="file"):
     if type == "file":
-        string = re.sub('[\?"*:|<>/]', '_', string)
+        string = re.sub(r"[\?\"*:|<>/]", "_", string)
     if type == "folder":
-        string = re.sub('[:\?<>"|]', '_', string)
+        string = re.sub(r"[:\?<>\"|*]", "_", string)
-
     return string
 
 
-_CN_RE1 = re.compile(ur'[^\w]+', re.UNICODE)
+_CN_RE1 = re.compile(r'[^\w]+', re.UNICODE)
-_CN_RE2 = re.compile(ur'[\s_]+', re.UNICODE)
+_CN_RE2 = re.compile(r'[\s_]+', re.UNICODE)
 
 
 _XLATE_GRAPHICAL_AND_DIACRITICAL = {
@@ -233,25 +270,33 @@ _XLATE_GRAPHICAL_AND_DIACRITICAL = {
     # ©ª«®²³¹»¼½¾ÆÐØÞßæðøþĐđĦħıIJijĸĿŀŁłŒœŦŧDŽDždžLJLjljNJNjnjǤǥDZDzdzȤȥ. This
     # includes also some graphical symbols which can be easily replaced and
     # usually are written by people who don't have appropriate keyboard layout.
-    u'©': '(C)', u'ª': 'a.', u'«': '<<', u'®': '(R)', u'²': '2', u'³': '3',
+    '©': '(C)', 'ª': 'a.', '«': '<<', '®': '(R)', '²': '2', '³': '3',
-    u'¹': '1', u'»': '>>', u'¼': ' 1/4 ', u'½': ' 1/2 ', u'¾': ' 3/4 ',
+    '¹': '1', '»': '>>', '¼': ' 1/4 ', '½': ' 1/2 ', '¾': ' 3/4 ',
-    u'Æ': 'AE', u'Ð': 'D', u'Ø': 'O', u'Þ': 'Th', u'ß': 'ss', u'æ': 'ae',
+    'Æ': 'AE', 'Ð': 'D', 'Ø': 'O', 'Þ': 'Th', 'ß': 'ss', 'æ': 'ae',
-    u'ð': 'd', u'ø': 'o', u'þ': 'th', u'Đ': 'D', u'đ': 'd', u'Ħ': 'H',
+    'ð': 'd', 'ø': 'o', 'þ': 'th', 'Đ': 'D', 'đ': 'd', 'Ħ': 'H',
-    u'ħ': 'h', u'ı': 'i', u'IJ': 'IJ', u'ij': 'ij', u'ĸ': 'q', u'Ŀ': 'L',
+    'ħ': 'h', 'ı': 'i', 'IJ': 'IJ', 'ij': 'ij', 'ĸ': 'q', 'Ŀ': 'L',
-    u'ŀ': 'l', u'Ł': 'L', u'ł': 'l', u'Œ': 'OE', u'œ': 'oe', u'Ŧ': 'T',
+    'ŀ': 'l', 'Ł': 'L', 'ł': 'l', 'Œ': 'OE', 'œ': 'oe', 'Ŧ': 'T',
-    u'ŧ': 't', u'DŽ': 'DZ', u'Dž': 'Dz', u'LJ': 'LJ', u'Lj': 'Lj',
+    'ŧ': 't', 'DŽ': 'DZ', 'Dž': 'Dz', 'LJ': 'LJ', 'Lj': 'Lj',
-    u'lj': 'lj', u'NJ': 'NJ', u'Nj': 'Nj', u'nj': 'nj',
+    'lj': 'lj', 'NJ': 'NJ', 'Nj': 'Nj', 'nj': 'nj',
-    u'Ǥ': 'G', u'ǥ': 'g', u'DZ': 'DZ', u'Dz': 'Dz', u'dz': 'dz',
+    'Ǥ': 'G', 'ǥ': 'g', 'DZ': 'DZ', 'Dz': 'Dz', 'dz': 'dz',
-    u'Ȥ': 'Z', u'ȥ': 'z', u'№': 'No.',
+    'Ȥ': 'Z', 'ȥ': 'z', '№': 'No.',
-    u'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
+    'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
                        # this is 'masculine ordering indicator', not degree
 }
 
 _XLATE_SPECIAL = {
     # Translation table.
     # Cover additional special characters processing normalization.
-    u"'": '',         # replace apostrophe with nothing
+    "'": '',         # replace apostrophe with nothing
-    u'&': ' and ',     # expand & to ' and '
+    "’": '',         # replace musicbrainz style apostrophe with nothing
+    '&': ' and ',     # expand & to ' and '
+}
+
+_XLATE_MUSICBRAINZ = {
+    # Translation table for Musicbrainz.
+    "…": '...',     # HORIZONTAL ELLIPSIS (U+2026)
+    "’": "'",       # APOSTROPHE (U+0027)
+    "‐": "-",       # EN DASH (U+2013)
 }
 
 
@@ -286,10 +331,10 @@ def _transliterate(u, xlate):
     Perform transliteration using the specified dictionary
     """
     u = unicodedata.normalize('NFD', u)
-    u = u''.join([u'' if _is_unicode_combining(x) else x for x in u])
+    u = ''.join(['' if _is_unicode_combining(x) else x for x in u])
     u = _translate(u, xlate)
     # at this point output is either unicode, or plain ascii
-    return unicode(u)
+    return str(u)
 
 
 def clean_name(s):
@@ -299,10 +344,10 @@ def clean_name(s):
     :param s: string to clean up, possibly unicode one.
     :return: cleaned-up version of input string.
     """
-    if not isinstance(s, unicode):
+    if not isinstance(s, str):
         # ignore extended chars if someone was dumb enough to pass non-ascii
         # narrow string here, use only unicode for meaningful texts
-        u = unicode(s, 'ascii', 'replace')
+        u = str(s, 'ascii', 'replace')
     else:
         u = s
     # 1. don't bother doing normalization NFKC, rather transliterate
@@ -313,17 +358,35 @@ def clean_name(s):
     # 3. translate spacials
     u = _translate(u, _XLATE_SPECIAL)
     # 4. replace any non-alphanumeric character sequences by spaces
-    u = _CN_RE1.sub(u' ', u)
+    u = _CN_RE1.sub(' ', u)
     # 5. coalesce interleaved space/underscore sequences
-    u = _CN_RE2.sub(u' ', u)
+    u = _CN_RE2.sub(' ', u)
     # 6. trim
     u = u.strip()
     # 7. lowercase
+    u = u.lower()
     return u
 
 
+def clean_musicbrainz_name(s, return_as_string=True):
+    # type: (basestring)->unicode
+    """Substitute special Musicbrainz characters.
+    :param s: string to clean up, probably unicode.
+    :return: cleaned-up version of input string.
+    """
+    if not isinstance(s, str):
+        u = str(s, 'ascii', 'replace')
+    else:
+        u = s
+    u = _translate(u, _XLATE_MUSICBRAINZ)
+    if return_as_string:
+        return u.encode('utf-8')
+    else:
+        return u
+
+
 def cleanTitle(title):
-    title = re.sub('[\.\-\/\_]', ' ', title).lower()
+    title = re.sub(r"[\.\-\/\_]", " ", title).lower()
 
     # Strip out extra whitespace
     title = ' '.join(title.split())
@@ -406,8 +469,7 @@ def expand_subfolders(f):
 
     if difference > 0:
         logger.info(
-            "Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.",
+            f"Found {len(media_folders)} media folders, but depth difference between lowest and deepest media folder is {difference} (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.")
-            len(media_folders), difference)
 
         # While already failed, advice the user what he could try. We assume the
         # directory may contain separate CD's and maybe some extra's. The
@@ -419,8 +481,7 @@ def expand_subfolders(f):
             set([os.path.join(*media_folder) for media_folder in extra_media_folders]))
 
         logger.info(
-            "Please look at the following folder(s), since they cause the depth difference: %s",
+            f"Please look at the following folder(s), since they cause the depth difference: {extra_media_folders}")
-            extra_media_folders)
         return
 
     # Convert back to paths and remove duplicates, which may be there after
@@ -434,7 +495,7 @@ def expand_subfolders(f):
         logger.debug("Did not expand subfolder, as it resulted in one folder.")
         return
 
-    logger.debug("Expanded subfolders in folder: %s", media_folders)
+    logger.debug(f"Expanded subfolders in folder: {media_folders}")
     return media_folders
 
 
@@ -445,14 +506,14 @@ def path_match_patterns(path, patterns):
     """
 
     for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
+        if fnmatch(path, pattern):
             return True
 
     # No match
     return False
 
 
-def path_filter_patterns(paths, patterns, root=None):
+def path_filter_patterns(paths, patterns, root=''):
     """
     Scan for ignored paths based on glob patterns. Note that the whole path
     will be matched, therefore paths should only contain the relative paths.
@@ -466,8 +527,7 @@ def path_filter_patterns(paths, patterns, root=None):
 
     for path in paths[:]:
         if path_match_patterns(path, patterns):
-            logger.debug("Path ignored by pattern: %s",
+            logger.debug(f"Path ignored by pattern: {os.path.join(root, path)}")
-                         os.path.join(root or "", path))
 
             ignored += 1
             paths.remove(path)
@@ -549,7 +609,7 @@ def extract_metadata(f):
     count_ratio = 0.75
 
     if count < (count_ratio * len(results)):
-        logger.info("Counted %d media files, but only %d have tags, ignoring.", count, len(results))
+        logger.info(f"Counted {count} media files, but only {len(results)} have tags, ignoring.")
         return (None, None, None)
 
     # Count distinct values
@@ -567,8 +627,7 @@ def extract_metadata(f):
                 old_album = new_albums[index]
                 new_albums[index] = RE_CD_ALBUM.sub("", album).strip()
 
-                logger.debug("Stripped albumd number identifier: %s -> %s", old_album,
+                logger.debug(f"Stripped album number identifier: {old_album} -> {new_albums[index]}")
-                             new_albums[index])
 
         # Remove duplicates
         new_albums = list(set(new_albums))
@@ -586,7 +645,7 @@ def extract_metadata(f):
     if len(artists) > 1 and len(albums) == 1:
         split_artists = [RE_FEATURING.split(x) for x in artists]
         featurings = [len(split_artist) - 1 for split_artist in split_artists]
-        logger.info("Album seem to feature %d different artists", sum(featurings))
+        logger.info("Album seem to feature {sum(featurings)} different artists")
 
         if sum(featurings) > 0:
             # Find the artist of which the least splits have been generated.
@@ -598,9 +657,11 @@ def extract_metadata(f):
             return (artist, albums[0], years[0])
 
     # Not sure what to do here.
-    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists),
+    logger.info(
-                len(albums), len(years))
+        f"Found {len(artists)} artists, {len(albums)} albums and "
-    logger.debug("Artists: %s, Albums: %s, Years: %s", artists, albums, years)
+        f"{len(years)} years in metadata, so ignoring"
+    )
+    logger.debug("Artists: {artists}, Albums: {albums}, Years: {years}")
 
     return (None, None, None)
 
@@ -620,26 +681,71 @@ def get_downloaded_track_list(albumpath):
     return downloaded_track_list
 
 
-def preserve_torrent_directory(albumpath):
+def preserve_torrent_directory(albumpath, forced=False, single=False):
     """
-    Copy torrent directory to headphones-modified to keep files for seeding.
+    Copy torrent directory to temp headphones_ directory to keep files for seeding.
     """
     from headphones import logger
-    new_folder = os.path.join(albumpath,
+
-                              'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
+    # Create temp dir
-    logger.info(
+    if headphones.CONFIG.KEEP_TORRENT_FILES_DIR:
-        "Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
+        tempdir = headphones.CONFIG.KEEP_TORRENT_FILES_DIR
+    else:
+        tempdir = tempfile.gettempdir()
+
+    logger.info(f"Preparing to copy to a temporary directory for post processing: {albumpath}")
+
     try:
-        shutil.copytree(albumpath, new_folder)
+        file_name = os.path.basename(os.path.normpath(albumpath))
-        return new_folder
+        if not single:
+            prefix = "headphones_" + file_name + "_@hp@_"
+        else:
+            prefix = "headphones_" + os.path.splitext(file_name)[0] + "_@hp@_"
+        new_folder = tempfile.mkdtemp(prefix=prefix, dir=tempdir)
     except Exception as e:
-        logger.warn("Cannot copy/move files to temp folder: " +
+        logger.error(f"Cannot create temp directory: {tempdir}. Error: {e}")
-                    new_folder.decode(headphones.SYS_ENCODING, 'replace') +
+        return None
-                    ". Not continuing. Error: " + str(e))
+
+    # Attempt to stop multiple temp dirs being created for the same albumpath
+    if not forced:
+        try:
+            workdir = os.path.join(tempdir, prefix)
+            workdir = re.sub(r'\[', '[[]', workdir)
+            workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
+            if len(glob(workdir + '*/')) >= 3:
+                logger.error(
+                    "Looks like a temp directory has previously been created "
+                    "for this albumpath, not continuing "
+                )
+                shutil.rmtree(new_folder)
+                return None
+        except Exception as e:
+            logger.warn(
+                "Cannot determine if already copied/processed, will copy anyway. "
+                f"Warning: {e}"
+            )
+
+    # Copy to temp dir
+    try:
+        subdir = os.path.join(new_folder, "headphones")
+        logger.info(f"Copying files to {subdir}")
+        if not single:
+            shutil.copytree(albumpath, subdir)
+        else:
+            os.makedirs(subdir)
+            shutil.copy(albumpath, subdir)
+        # Update the album path with the new location
+        return subdir
+    except Exception as e:
+        logger.warn(
+            f"Cannot copy/move files to temp directory: {new_folder}. "
+            f"Not continuing. Error: {e}"
+        )
+        shutil.rmtree(new_folder)
         return None
 
 
-def cue_split(albumpath):
+def cue_split(albumpath, keep_original_folder=False):
     """
      Attempts to check and split audio files by a cue for the given directory.
      """
@@ -661,6 +767,15 @@ def cue_split(albumpath):
     # Split cue
     if cue_count and cue_count >= count and cue_dirs:
 
+        # Copy to temp directory
+        if keep_original_folder:
+            temppath = preserve_torrent_directory(albumpath)
+            if temppath:
+                cue_dirs = [cue_dir.replace(albumpath, temppath) for cue_dir in cue_dirs]
+                albumpath = temppath
+            else:
+                return None
+
         from headphones import logger, cuesplit
         logger.info("Attempting to split audio files by cue")
 
@@ -670,13 +785,13 @@ def cue_split(albumpath):
                 cuesplit.split(cue_dir)
             except Exception as e:
                 os.chdir(cwd)
-                logger.warn("Cue not split: " + str(e))
+                logger.warn(f"Cue not split. Error: {e}")
-                return False
+                return None
 
         os.chdir(cwd)
-        return True
+        return albumpath
 
-    return False
+    return None
 
 
 def extract_logline(s):
@@ -708,7 +823,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid default format", s)
+        logger.info(f"Couldn't parse {s} into a valid default format")
 
     # newzbin default format
     pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s\((?P<year>\d+?\))', re.VERBOSE)
@@ -719,7 +834,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid Newbin format", s)
+        logger.info(f"Couldn't parse {s} into a valid Newbin format")
         return (name, album, year)
 
 
@@ -728,9 +843,11 @@ def smartMove(src, dest, delete=True):
 
     source_dir = os.path.dirname(src)
     filename = os.path.basename(src)
+    source_path = os.path.join(source_dir, filename)
+    dest_path = os.path.join(dest, filename)
 
-    if os.path.isfile(os.path.join(dest, filename)):
+    if os.path.isfile(dest_path):
-        logger.info('Destination file exists: %s', os.path.join(dest, filename))
+        logger.info(f"Destination file exists: {dest_path}")
         title = os.path.splitext(filename)[0]
         ext = os.path.splitext(filename)[1]
         i = 1
@@ -739,24 +856,39 @@ def smartMove(src, dest, delete=True):
             if os.path.isfile(os.path.join(dest, newfile)):
                 i += 1
             else:
-                logger.info('Renaming to %s', newfile)
+                logger.info(f"Renaming to {newfile}")
                 try:
                     os.rename(src, os.path.join(source_dir, newfile))
                     filename = newfile
+                    source_path = os.path.join(source_dir, filename)
+                    dest_path = os.path.join(dest, filename)
                 except Exception as e:
-                    logger.warn('Error renaming %s: %s',
+                    logger.warn(f"Error renaming {src}: {e}")
-                                src.decode(headphones.SYS_ENCODING, 'replace'), e)
                 break
 
-    try:
+    if delete:
-        if delete:
+        try:
-            shutil.move(os.path.join(source_dir, filename), os.path.join(dest, filename))
+            logger.info('Moving "%s" to "%s"', source_path, dest_path)
-        else:
+            shutil.move(source_path, dest_path)
-            shutil.copy(os.path.join(source_dir, filename), os.path.join(dest, filename))
+        except Exception as e:
+            exists = os.path.exists(dest_path)
+            if exists and os.path.getsize(source_path) == os.path.getsize(dest_path):
+                logger.warn(
+                    f"Successfully moved {filename}, but something went wrong: {e}"
+                )
+                os.unlink(source_path)
+            else:
+                # remove faultly copied file
+                if exists:
+                    os.unlink(dest_path)
+                raise
+    else:
+        try:
+            logger.info(f"Copying {source_path} to {dest_path}")
+            shutil.copy(source_path, dest_path)
             return True
-    except Exception as e:
+        except Exception as e:
-        logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
+            logger.warn(f"Error copying {filename}: {e}") 
-                    e)
 
 
 def walk_directory(basedir, followlinks=True):
@@ -765,7 +897,7 @@ def walk_directory(basedir, followlinks=True):
     with care. In case a folder is already processed, don't traverse it again.
     """
 
-    import logger
+    from . import logger
 
     # Add the base path, because symlinks poiting to the basedir should not be
     # traversed again.
@@ -779,8 +911,10 @@ def walk_directory(basedir, followlinks=True):
                 real_path = os.path.abspath(os.readlink(path))
 
                 if real_path in traversed:
-                    logger.debug("Skipping '%s' since it is a symlink to "
+                    logger.debug(
-                                 "'%s', which is already visited.", path, real_path)
+                        f"Skipping {path} since it is a symlink to "
+                        f"{real_path}, which is already visited."
+                    )
                 else:
                     traversed.append(real_path)
 
@@ -822,22 +956,15 @@ def sab_sanitize_foldername(name):
     FL_ILLEGAL = CH_ILLEGAL + ':\x92"'
     FL_LEGAL = CH_LEGAL + "-''"
 
-    uFL_ILLEGAL = FL_ILLEGAL.decode('latin-1')
-    uFL_LEGAL = FL_LEGAL.decode('latin-1')
-
     if not name:
-        return name
+        return
-    if isinstance(name, unicode):
+
-        illegal = uFL_ILLEGAL
+    name = unidecode(name)
-        legal = uFL_LEGAL
-    else:
-        illegal = FL_ILLEGAL
-        legal = FL_LEGAL
 
     lst = []
     for ch in name.strip():
-        if ch in illegal:
+        if ch in FL_ILLEGAL:
-            ch = legal[illegal.find(ch)]
+            ch = FL_LEGAL[FL_ILLEGAL.find(ch)]
             lst.append(ch)
         else:
             lst.append(ch)
@@ -893,7 +1020,42 @@ def create_https_certificates(ssl_cert, ssl_key):
         with open(ssl_cert, "w") as fp:
             fp.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
     except IOError as e:
-        logger.error("Error creating SSL key and certificate: %s", e)
+        logger.error(f"Error creating SSL key and certificate: e")
         return False
 
     return True
+
+
+class BeetsLogCapture(beetslogging.Handler):
+
+    def __init__(self):
+        beetslogging.Handler.__init__(self)
+        self.messages = []
+
+    def emit(self, record):
+        self.messages.append(text_type(record.msg))
+
+
+@contextmanager
+def capture_beets_log(logger='beets'):
+    capture = BeetsLogCapture()
+    log = beetslogging.getLogger(logger)
+    log.addHandler(capture)
+    try:
+        yield capture.messages
+    finally:
+        log.removeHandler(capture)
+
+def have_pct_have_total(db_artist):
+    have_tracks = db_artist['HaveTracks'] or 0
+    total_tracks = db_artist['TotalTracks'] or 0 
+    have_pct = have_tracks / total_tracks if total_tracks else 0
+    return (have_pct, total_tracks)
+
+
+def has_token(title, token):
+    return bool(
+            re.search(rf'(?:\W|^)+{token}(?:\W|$)+', 
+            title,
+            re.IGNORECASE | re.UNICODE)
+        )

headphones/helpers_test.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from unittestcompat import TestCase
+from .unittestcompat import TestCase
-from headphones.helpers import clean_name
+from headphones.helpers import clean_name, is_valid_date, age, has_token
 
 
 class HelpersTest(TestCase):
@@ -8,28 +8,28 @@ class HelpersTest(TestCase):
     def test_clean_name(self):
         """helpers: check correctness of clean_name() function"""
         cases = {
-            u' Weiße & rose ': 'Weisse and rose',
+            ' Weiße & rose ': 'Weisse and rose',
-            u'Multiple / spaces': 'Multiple spaces',
+            'Multiple / spaces': 'Multiple spaces',
-            u'Kevin\'s m²': 'Kevins m2',
+            'Kevin\'s m²': 'Kevins m2',
-            u'Symphonęy Nº9': 'Symphoney No.9',
+            'Symphonęy Nº9': 'Symphoney No.9',
-            u'ÆæßðÞIJij': u'AeaessdThIJıj',
+            'ÆæßðÞIJij': 'AeaessdThIJıj',
-            u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
+            'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
                                                     'apoplexy remix',
-            u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
+            'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
                                                     'zbojow',
-            u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
+            'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
-            u'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
+            'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
-            u'L\'Arc~en~Ciel': 'larc en ciel',
+            'L\'Arc~en~Ciel': 'larc en ciel',
-            u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
+            'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
-                u'Orquesta de la Luz オルケスタ デ ラ ルス'
+                'Orquesta de la Luz オルケスタ デ ラ ルス'
 
         }
-        for first, second in cases.iteritems():
+        for first, second in cases.items():
             nf = clean_name(first).lower()
             ns = clean_name(second).lower()
             self.assertEqual(
-                nf, ns, u"check cleaning of case (%s,"
+                nf, ns, "check cleaning of case (%s,"
-                        u"%s)" % (nf, ns)
+                        "%s)" % (nf, ns)
             )
 
     def test_clean_name_nonunicode(self):
@@ -46,3 +46,28 @@ class HelpersTest(TestCase):
         self.assertEqual(
             test, expected, "check clean_name() with narrow non-ascii input"
         )
+
+    def test_is_valid_date(date):
+        test_cases = [
+            ('2021-11-12', True, "check is_valid_date returns True for valid date"),
+            (None, False, "check is_valid_date returns False for None"),
+            ('2021-11', False, "check is_valid_date returns False for incomplete"),
+            ('2021', False, "check is_valid_date returns False for incomplete")
+        ]
+        for input, expected, desc in test_cases:
+            self.assertEqual(is_valid_date(input), expected, desc)
+
+    def test_has_token(self):
+        """helpers: has_token()"""
+        self.assertEqual(
+            has_token("a cat ran", "cat"), 
+            True, 
+            "return True if token is in string"
+        )
+        self.assertEqual(
+            has_token("acatran", "cat"),
+            False,
+            "return False if token is part of another word"
+        )
+
+

headphones/importer.py

@@ -16,7 +16,7 @@
 import time
 
 from headphones import logger, helpers, db, mb, lastfm, metacritic
-from beets.mediafile import MediaFile
+from mediafile import MediaFile
 import headphones
 
 blacklisted_special_artist_names = ['[anonymous]', '[data]', '[no artist]',
@@ -39,7 +39,7 @@ def is_exists(artistid):
 
     if any(artistid in x for x in artistlist):
         logger.info(artistlist[0][
-                        1] + u" is already in the database. Updating 'have tracks', but not artist information")
+                        1] + " is already in the database. Updating 'have tracks', but not artist information")
         return True
     else:
         return False
@@ -53,7 +53,7 @@ def artistlist_to_mbids(artistlist, forced=False):
 
         # If adding artists through Manage New Artists, they're coming through as non-unicode (utf-8?)
         # and screwing everything up
-        if not isinstance(artist, unicode):
+        if not isinstance(artist, str):
             try:
                 artist = artist.decode('utf-8', 'replace')
             except Exception:
@@ -89,23 +89,20 @@ def artistlist_to_mbids(artistlist, forced=False):
             addArtisttoDB(artistid)
 
         # Just update the tracks if it does
-        else:
+
-            havetracks = len(
+        # not sure this is correct and we're updating during scanning in librarysync
-                myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+        # else:
-                myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+        #     havetracks = len(
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
+        #         myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+        #         myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+        #     myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
 
         # Delete it from the New Artists if the request came from there
         if forced:
             myDB.action('DELETE from newartists WHERE ArtistName=?', [artist])
 
     # Update the similar artist tag cloud:
-    logger.info('Updating artist information from Last.fm')
+    lastfm.getSimilar()
-
-    try:
-        lastfm.getSimilar()
-    except Exception as e:
-        logger.warn('Failed to update arist information from Last.fm: %s' % e)
 
 
 def addArtistIDListToDB(artistidlist):
@@ -182,7 +179,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     else:
         sortname = artist['artist_name']
 
-    logger.info(u"Now adding/updating: " + artist['artist_name'])
+    logger.info("Now adding/updating: " + artist['artist_name'])
     controlValueDict = {"ArtistID": artistid}
     newValueDict = {"ArtistName": artist['artist_name'],
                     "ArtistSortName": sortname,
@@ -243,7 +240,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         rgid = rg['id']
         skip_log = 0
         # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
-        pause_delta = headphones.CONFIG.MB_IGNORE_AGE
+        ignore_age = headphones.CONFIG.MB_IGNORE_AGE
 
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
 
@@ -261,25 +258,29 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 new_releases = mb.get_new_releases(rgid, includeExtras)
 
             else:
-                if check_release_date is None or check_release_date == u"None":
+                if check_release_date is None or check_release_date == "None":
-                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                    if not headphones.CONFIG.MB_IGNORE_AGE_MISSING:
-                    new_releases = mb.get_new_releases(rgid, includeExtras, True)
+                        logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                        new_releases = mb.get_new_releases(rgid, includeExtras, True)
+                    else:
+                        logger.info("[%s] Skipping update of: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                        new_releases = 0
                 else:
                     if len(check_release_date) == 10:
                         release_date = check_release_date
                     elif len(check_release_date) == 7:
-                        release_date = check_release_date + "-31"
+                        release_date = check_release_date + "-27"
                     elif len(check_release_date) == 4:
-                        release_date = check_release_date + "-12-31"
+                        release_date = check_release_date + "-12-27"
                     else:
                         release_date = today
-                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                    if helpers.age(release_date) < ignore_age:
                         logger.info("[%s] Now updating: %s (Release Date <%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
                         logger.info("[%s] Skipping: %s (Release Date >%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         skip_log = 1
                         new_releases = 0
 
@@ -305,7 +306,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
             # This will be used later to build a hybrid release
             fullreleaselist = []
             # Search for releases within a release group
-            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
+            find_hybrid_releases = myDB.select("SELECT * from allalbums WHERE AlbumID=?",
                                                [rg['id']])
 
             # Build the dictionary for the fullreleaselist
@@ -444,14 +445,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
                 if headphones.CONFIG.AUTOWANT_ALL:
                     newValueDict['Status'] = "Wanted"
-                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
+                elif headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
+                    if helpers.is_valid_date(album['ReleaseDate']) and helpers.age(album['ReleaseDate']) < 21:
-                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
+                        newValueDict['Status'] = "Wanted"
-                # The first test just makes sure we have year-month-day
-                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
-                        today) - helpers.get_age(
-                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
                 else:
                     newValueDict['Status'] = "Skipped"
 
@@ -511,8 +507,11 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     marked_as_downloaded = True
 
             logger.info(
-                u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+                "[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
-            cache.getThumb(AlbumID=rg['id'])
+            try:
+                cache.getThumb(AlbumID=rg['id'])
+            except Exception as e:
+                logger.error("Error getting album art: %s", e)
 
             # Start a search for the album if it's new, hasn't been marked as
             # downloaded and autowant_all is selected. This search is deferred,
@@ -521,17 +520,23 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 album_searches.append(rg['id'])
         else:
             if skip_log == 0:
-                logger.info(u"[%s] No new releases, so no changes made to %s" % (
+                logger.info("[%s] No new releases, so no changes made to %s" % (
                             artist['artist_name'], rg['title']))
 
     time.sleep(3)
     finalize_update(artistid, artist['artist_name'], errors)
 
-    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
+    logger.info("Seeing if we need album art for: %s" % artist['artist_name'])
-    cache.getThumb(ArtistID=artistid)
+    try:
+        cache.getThumb(ArtistID=artistid)
+    except Exception as e:
+        logger.error("Error getting album art: %s", e)
 
-    logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
+    logger.info("Fetching Metacritic reviews for: %s" % artist['artist_name'])
-    metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
+    try:
+        metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
+    except Exception as e:
+        logger.error("Error getting Metacritic reviews: %s", e)
 
     if errors:
         logger.info(
@@ -539,7 +544,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 artist['artist_name'], artist['artist_name']))
     else:
         myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
-        logger.info(u"Updating complete for: %s" % artist['artist_name'])
+        logger.info("Updating complete for: %s" % artist['artist_name'])
 
     # Start searching for newly added albums
     if album_searches:
@@ -648,7 +653,7 @@ def addReleaseById(rid, rgid=None):
             sortname = release_dict['artist_name']
 
         logger.info(
-            u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
+            "Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
         controlValueDict = {"ArtistID": release_dict['artist_id']}
         newValueDict = {"ArtistName": release_dict['artist_name'],
                         "ArtistSortName": sortname,
@@ -681,7 +686,7 @@ def addReleaseById(rid, rgid=None):
 
     if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
         # but if it is this will fail
-        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
+        logger.info("Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
         controlValueDict = {"AlbumID": rgid}
         if status != 'Loading':
             status = 'Wanted'
@@ -757,7 +762,7 @@ def addReleaseById(rid, rgid=None):
 
         # Start a search for the album
         if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
-            import searcher
+            from . import searcher
             searcher.searchforalbum(rgid, False)
 
     elif not rg_exists and not release_dict:

headphones/lastfm.py

@@ -22,8 +22,8 @@ from headphones import db, logger, request
 
 TIMEOUT = 60.0  # seconds
 REQUEST_LIMIT = 1.0 / 5  # seconds
-ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
+ENTRY_POINT = "https://ws.audioscrobbler.com/2.0/"
-API_KEY = "395e6ec6bb557382fc41fde867bce66f"
+APP_API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
 # Required for API request limit
 lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
@@ -31,7 +31,7 @@ lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
 
 def request_lastfm(method, **kwargs):
     """
-    Call a Last.FM API method. Automatically sets the method and API key. Method
+    Call a Last.fm API method. Automatically sets the method and API key. Method
     will return the result if no error occured.
 
     By default, this method will request the JSON format, since it is more
@@ -40,35 +40,42 @@ def request_lastfm(method, **kwargs):
 
     # Prepare request
     kwargs["method"] = method
-    kwargs.setdefault("api_key", API_KEY)
+    kwargs.setdefault("api_key", headphones.CONFIG.LASTFM_APIKEY or APP_API_KEY)
     kwargs.setdefault("format", "json")
 
     # Send request
-    logger.debug("Calling Last.FM method: %s", method)
+    logger.debug("Calling Last.fm method: %s", method)
-    logger.debug("Last.FM call parameters: %s", kwargs)
+    logger.debug("Last.fm call parameters: %s", kwargs)
 
     data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs, lock=lastfm_lock)
 
     # Parse response and check for errors.
     if not data:
-        logger.error("Error calling Last.FM method: %s", method)
+        logger.error("Error calling Last.fm method: %s", method)
         return
 
     if "error" in data:
-        logger.error("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.fm returned an error: %s", data["message"])
         return
 
     return data
 
 
 def getSimilar():
-    myDB = db.DBConnection()
+    if not headphones.CONFIG.LASTFM_APIKEY:
-    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC")
+        logger.info(
+            'To update the Similar Artists cloud tag, create a Last.fm application api key '
+            'and add it under the Advanced config tab'
+        )
+        return
 
-    logger.info("Fetching similar artists from Last.FM for tag cloud")
+    myDB = db.DBConnection()
+    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC LIMIT 10")
+
+    logger.info("Fetching similar artists from Last.fm for tag cloud")
     artistlist = []
 
-    for result in results[:12]:
+    for result in results:
         data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"])
 
         if data and "similarartists" in data:
@@ -85,13 +92,13 @@ def getSimilar():
                     artistlist.append((artist_name, artist_mbid))
 
     # Add new artists to tag cloud
-    logger.debug("Fetched %d artists from Last.FM", len(artistlist))
+    logger.debug("Fetched %d artists from Last.fm", len(artistlist))
     count = defaultdict(int)
 
     for artist, mbid in artistlist:
         count[artist, mbid] += 1
 
-    items = count.items()
+    items = list(count.items())
     top_list = sorted(items, key=lambda x: x[1], reverse=True)[:25]
 
     random.shuffle(top_list)
@@ -103,7 +110,7 @@ def getSimilar():
 
         myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count])
 
-    logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list))
+    logger.debug("Inserted %d artists into Last.fm tag cloud", len(top_list))
 
 
 def getArtists():
@@ -111,16 +118,16 @@ def getArtists():
     results = myDB.select("SELECT ArtistID from artists")
 
     if not headphones.CONFIG.LASTFM_USERNAME:
-        logger.warn("Last.FM username not set, not importing artists.")
+        logger.warn("Last.fm username not set, not importing artists.")
         return
 
-    logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME)
+    logger.info("Fetching artists from Last.fm for username: %s", headphones.CONFIG.LASTFM_USERNAME)
     data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME)
 
     if data and "artists" in data:
         artistlist = []
         artists = data["artists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             artist_mbid = artist["mbid"]
@@ -133,23 +140,26 @@ def getArtists():
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.info("Imported %d new artists from Last.FM", len(artistlist))
+        logger.info("Imported %d new artists from Last.fm", len(artistlist))
 
 
 def getTagTopArtists(tag, limit=50):
     myDB = db.DBConnection()
     results = myDB.select("SELECT ArtistID from artists")
 
-    logger.info("Fetching top artists from Last.FM for tag: %s", tag)
+    logger.info("Fetching top artists from Last.fm for tag: %s", tag)
     data = request_lastfm("tag.gettopartists", limit=limit, tag=tag)
 
     if data and "topartists" in data:
         artistlist = []
         artists = data["topartists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
-            artist_mbid = artist["mbid"]
+            try:
+                artist_mbid = artist["mbid"]
+            except KeyError:
+                continue
 
             if not any(artist_mbid in x for x in results):
                 artistlist.append(artist_mbid)
@@ -159,4 +169,4 @@ def getTagTopArtists(tag, limit=50):
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.debug("Added %d new artists from Last.FM", len(artistlist))
+        logger.debug("Added %d new artists from Last.fm", len(artistlist))

headphones/librarysync.py

@@ -17,7 +17,7 @@ import os
 import math
 
 import headphones
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from headphones import db, logger, helpers, importer, lastfm
 
 
@@ -30,56 +30,64 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     if not dir:
         if not headphones.CONFIG.MUSIC_DIR:
+            logger.info(
+                "No music directory configured. Add it under "
+                "Manage -> Scan Music Library"
+            )
             return
         else:
             dir = headphones.CONFIG.MUSIC_DIR
 
-    # If we're appending a dir, it's coming from the post processor which is
-    # already bytestring
-    if not append or artistScan:
-        dir = dir.encode(headphones.SYS_ENCODING)
-
     if not os.path.isdir(dir):
-        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING,
+        logger.warn(f"Cannot find music directory: {dir}")
-                                                                           'replace'))
         return
 
     myDB = db.DBConnection()
     new_artists = []
 
-    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Scanning music directory: {dir}")
 
     if not append:
-        # Clean up bad filepaths
-        tracks = myDB.select(
-            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
 
-        for track in tracks:
+        # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
+        if ArtistID:
-            if not os.path.isfile(encoded_track_string):
+            dbtracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
+                'IS NOT NULL',
+                [ArtistID, ArtistID])
+        else:
+            dbtracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL')
+
+        for track in dbtracks:
+            track_location = track['Location']
+            if not os.path.isfile(track_location):
                 myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, track_location])
                 myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, track_location])
 
-        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
+        if ArtistName:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName])
+        else:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have')
 
         for track in del_have_tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
+            if not os.path.isfile(track['Location']):
-            if not os.path.isfile(encoded_track_string):
                 if track['ArtistName']:
                     # Make sure deleted files get accounted for when updating artist track counts
                     new_artists.append(track['ArtistName'])
                 myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                 logger.info(
-                    'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
+                    f"{track['Location']} removed from Headphones, as it "
-                        headphones.SYS_ENCODING, 'replace'))
+                    f"is no longer on disk"
+                )
 
     bitrates = []
-    song_list = []
+    track_list = []
     latest_subdirectory = []
 
-    new_song_count = 0
+    new_track_count = 0
     file_count = 0
 
     for r, d, f in helpers.walk_directory(dir):
@@ -94,32 +102,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 subdirectory = r.replace(dir, '')
                 latest_subdirectory.append(subdirectory)
 
-                if file_count == 0 and r.replace(dir, '') != '':
+                track_path = os.path.join(r, files)
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-                elif latest_subdirectory[file_count] != latest_subdirectory[
-                            file_count - 1] and file_count != 0:
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-
-                song = os.path.join(r, files)
-
-                # We need the unicode path to use for logging, inserting into database
-                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')
 
                 # Try to read the metadata
                 try:
-                    f = MediaFile(song)
+                    f = MediaFile(track_path)
                 except (FileTypeError, UnreadableFileError):
-                    logger.warning(
+                    logger.warning(f"Cannot read `{track_path}`. It may be corrupted or not a media file.")
-                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
-                        unicode_song_path)
                     continue
                 except IOError:
-                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?",
+                    logger.warning(f"Cannnot read `{track_path}`. Does the file exists?")
-                                   unicode_song_path)
                     continue
 
                 # Grab the bitrates for the auto detect bit rate option
@@ -134,15 +126,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 else:
                     f_artist = None
 
-                # Add the song to our song list -
+                # Add the track to our track list -
-                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
+                # TODO: skip adding tracks without the minimum requisite information (just a matter of putting together the right if statements)
 
                 if f_artist and f.album and f.title:
                     CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title)
                 else:
                     CleanName = None
 
-                controlValueDict = {'Location': unicode_song_path}
+                controlValueDict = {'Location': track_path}
 
                 newValueDict = {'TrackID': f.mb_trackid,
                                 # 'ReleaseID' : f.mb_albumid,
@@ -158,24 +150,24 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                                 'CleanName': CleanName
                                 }
 
-                # song_list.append(song_dict)
+                # track_list.append(track_dict)
-                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?",
+                check_exist_track = myDB.action("SELECT * FROM have WHERE Location=?",
-                                               [unicode_song_path]).fetchone()
+                                               [track_path]).fetchone()
-                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
+                # Only attempt to match tracks that are new, haven't yet been matched, or metadata has changed.
-                if not check_exist_song:
+                if not check_exist_track:
                     # This is a new track
                     if f_artist:
                         new_artists.append(f_artist)
                     myDB.upsert("have", newValueDict, controlValueDict)
-                    new_song_count += 1
+                    new_track_count += 1
                 else:
-                    if check_exist_song['ArtistName'] != f_artist or check_exist_song[
+                    if check_exist_track['ArtistName'] != f_artist or check_exist_track[
-                            'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                            'AlbumTitle'] != f.album or check_exist_track['TrackTitle'] != f.title:
                         # Important track metadata has been modified, need to run matcher again
-                        if f_artist and f_artist != check_exist_song['ArtistName']:
+                        if f_artist and f_artist != check_exist_track['ArtistName']:
                             new_artists.append(f_artist)
-                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
+                        elif f_artist and f_artist == check_exist_track['ArtistName'] and \
-                                        check_exist_song['Matched'] != "Ignored":
+                                        check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
                         else:
                             continue
@@ -184,49 +176,59 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                         myDB.upsert("have", newValueDict, controlValueDict)
                         myDB.action(
                             'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
+                            [None, None, None, track_path])
                         myDB.action(
                             'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
+                            [None, None, None, track_path])
-                        new_song_count += 1
+                        new_track_count += 1
                     else:
                         # This track information hasn't changed
-                        if f_artist and check_exist_song['Matched'] != "Ignored":
+                        if f_artist and check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
 
                 file_count += 1
 
     # Now we start track matching
-    logger.info("%s new/modified songs found and added to the database" % new_song_count)
+    logger.info(f"{new_track_count} new/modified tracks found and added to the database")
-    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+    dbtracks = myDB.action(
-                            [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
+            "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-    total_number_of_songs = \
+            [f"{dir}%"]
-        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+        )
-                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
+    dbtracks_count = myDB.action(
-    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(
+            "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-        headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+            [f"{dir}%"]
+        ).fetchone()[0]
+    logger.info(f"Found {dbtracks_count} new/modified tracks in `{dir}`")
+    logger.info("Matching tracks to the appropriate releases....")
 
-    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
-    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
 
-    # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
-    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
 
-    # We'll use this to give a % completion, just because the track matching might take a while
+    # Sort the track_list by most vague (e.g. no trackid or releaseid)
-    song_count = 0
+    # to most specific (both trackid & releaseid)
-    latest_artist = []
+    # When we insert into the database, the tracks with the most 
+    # specific information will overwrite the more general matches
+
+    sorted_dbtracks = helpers.multikeysort(dbtracks, ['ArtistName', 'AlbumTitle'])
+
+
+    # We'll use this to give a % completion, just because the
+    # track matching might take a while
+    tracks_completed = 0
+    latest_artist = None
     last_completion_percentage = 0
+    prev_artist_name = None
+    artistid = None
 
-    for song in song_list:
+    for track in sorted_dbtracks:
 
-        latest_artist.append(song['ArtistName'])
+        if latest_artist != track['ArtistName']:
-        if song_count == 0:
+            logger.info(f"Now matching tracks by {track['ArtistName']}")
-            logger.info("Now matching songs by %s" % song['ArtistName'])
+            latest_artist = track['ArtistName']
-        elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0:
-            logger.info("Now matching songs by %s" % song['ArtistName'])
 
-        song_count += 1
+        tracks_completed += 1
-        completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10
+        completion_percentage = math.floor(
+                float(tracks_completed) / dbtracks_count * 1000
+            ) / 10
 
         if completion_percentage >= (last_completion_percentage + 10):
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
@@ -237,139 +239,184 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
         # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+        albumid = None
 
-            track = myDB.action(
+        if track['ArtistName'] and track['CleanName']:
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+            artist_name = track['ArtistName']
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            clean_name = track['CleanName']
-            have_updated = False
-            if track:
-                controlValueDict = {'ArtistName': track['ArtistName'],
-                                    'AlbumTitle': track['AlbumTitle'],
-                                    'TrackTitle': track['TrackTitle']}
-                newValueDict = {'Location': song['Location'],
-                                'BitRate': song['BitRate'],
-                                'Format': song['Format']}
-                myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                controlValueDict2 = {'Location': song['Location']}
+            # Only update if artist is in the db
-                newValueDict2 = {'Matched': track['AlbumID']}
+            if artist_name != prev_artist_name:
-                myDB.upsert("have", newValueDict2, controlValueDict2)
+                prev_artist_name = artist_name
-                have_updated = True
+                artistid = None
-            else:
-                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
-                                    [song['CleanName']]).fetchone()
-                if track:
-                    controlValueDict = {'CleanName': track['CleanName']}
-                    newValueDict = {'Location': song['Location'],
-                                    'BitRate': song['BitRate'],
-                                    'Format': song['Format']}
-                    myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                    controlValueDict2 = {'Location': song['Location']}
+                artist_lookup = "\"" + artist_name.replace("\"", "\"\"") + "\""
-                    newValueDict2 = {'Matched': track['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
-                else:
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': "Failed"}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
 
-            alltrack = myDB.action(
+                try:
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+                except:
-            if alltrack:
+                    dbartist = None
-                controlValueDict = {'ArtistName': alltrack['ArtistName'],
+                if not dbartist:
-                                    'AlbumTitle': alltrack['AlbumTitle'],
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name])
-                                    'TrackTitle': alltrack['TrackTitle']}
+                    if not dbartist:
-                newValueDict = {'Location': song['Location'],
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name])
-                                'BitRate': song['BitRate'],
+                        if not dbartist:
-                                'Format': song['Format']}
+                            clean_artist = helpers.clean_name(artist_name)
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                            if clean_artist:
+                                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                                       [clean_artist, clean_artist + '{'])
+                                if not dbartist:
+                                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                                           [clean_artist, clean_artist + '{'])
 
-                controlValueDict2 = {'Location': song['Location']}
+                if dbartist:
-                newValueDict2 = {'Matched': alltrack['AlbumID']}
+                    artistid = dbartist[0][0]
-                myDB.upsert("have", newValueDict2, controlValueDict2)
+
-            else:
+            if artistid:
-                alltrack = myDB.action(
+
-                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
+                # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index
-                    [song['CleanName']]).fetchone()
+                # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like)
+                # Now selects/updates using CleanName index (may have to revert if not working)
+
+                # matching on CleanName should be enough, ensure it's the same artist just in case
+
+                # Update tracks
+                dbtrack = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
+                if dbtrack:
+                    albumid = dbtrack['AlbumID']
+                    myDB.action(
+                        'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
+
+                # Update alltracks
+                alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
                 if alltrack:
-                    controlValueDict = {'CleanName': alltrack['CleanName']}
+                    albumid = alltrack['AlbumID']
-                    newValueDict = {'Location': song['Location'],
+                    myDB.action(
-                                    'BitRate': song['BitRate'],
+                        'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
-                                    'Format': song['Format']}
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
-                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': alltrack['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                else:
-                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
-                    if not have_updated:
-                        controlValueDict2 = {'Location': song['Location']}
-                        newValueDict2 = {'Matched': "Failed"}
-                        myDB.upsert("have", newValueDict2, controlValueDict2)
 
+        # Update have
+        controlValueDict2 = {'Location': track['Location']}
+        if albumid:
+            newValueDict2 = {'Matched': albumid}
         else:
-            controlValueDict2 = {'Location': song['Location']}
             newValueDict2 = {'Matched': "Failed"}
-            myDB.upsert("have", newValueDict2, controlValueDict2)
+        myDB.upsert("have", newValueDict2, controlValueDict2)
 
-            # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+        # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [track['ArtistName'], track['AlbumTitle'], track['TrackNumber'], track['TrackTitle'], track['TrackLength'], track['BitRate'], track['Genre'], track['Date'], track['TrackID'], track['Location'], CleanName, track['Format']])
 
-    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
+    logger.info(f"Completed matching tracks from `{dir}`")
-                                                                            'replace'))
 
     if not append or artistScan:
         logger.info('Updating scanned artist track counts')
 
         # Clean up the new artist list
-        unique_artists = {}.fromkeys(new_artists).keys()
+        unique_artists = list({}.fromkeys(new_artists).keys())
-        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
 
-        # There was a bug where artists with special characters (-,') would show up in new artists.
+        # # Don't think we need to do this, check the db instead below
-        artist_list = [
+        #
-            x for x in unique_artists
+        # # artist scan
-            if helpers.clean_name(x).lower() not in [
+        # if ArtistName:
-                helpers.clean_name(y[0]).lower()
+        #     current_artists = [[ArtistName]]
-                for y in current_artists
+        # # directory scan
-                ]
+        # else:
-            ]
+        #     current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL')
-        artists_checked = [
+        #
-            x for x in unique_artists
+        # # There was a bug where artists with special characters (-,') would show up in new artists.
-            if helpers.clean_name(x).lower() in [
+        #
-                helpers.clean_name(y[0]).lower()
+        # # artist_list = scanned artists not in the db
-                for y in current_artists
+        # artist_list = [
-                ]
+        #     x for x in unique_artists
-            ]
+        #     if helpers.clean_name(x).lower() not in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
+        #
+        # # artists_checked = scanned artists that exist in the db
+        # artists_checked = [
+        #     x for x in unique_artists
+        #     if helpers.clean_name(x).lower() in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
 
-        # Update track counts
+        new_artist_list = []
-        for artist in artists_checked:
-            # Have tracks are selected from tracks table and not all tracks because of duplicates
-            # We update the track count upon an album switch to compliment this
-            havetracks = (
-                len(myDB.select(
-                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
-                    [artist])) + len(myDB.select(
-                        'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
-                        [artist]))
-            )
-            # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
-            # (can fix by getting rid of second len statement)
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
 
-        logger.info('Found %i new artists' % len(artist_list))
+        for artist in unique_artists:
 
-        if artist_list:
+            if not artist:
+                continue
+
+            logger.info('Processing artist: %s' % artist)
+
+            # check if artist is already in the db
+            artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\""
+
+            try:
+                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
+            except:
+                dbartist = None
+            if not dbartist:
+                clean_artist = helpers.clean_name(artist)
+                if clean_artist:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                           [clean_artist, clean_artist + '{'])
+                    if not dbartist:
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                               [clean_artist, clean_artist + '{'])
+
+            # new artist not in db, add to list
+            if not dbartist:
+                new_artist_list.append(artist)
+            else:
+
+                # artist in db, update have track counts
+                artistid = dbartist[0][0]
+
+                # Have tracks are selected from tracks table and not all tracks because of duplicates
+                # We update the track count upon an album switch to compliment this
+
+                # havetracks = (
+                #     len(myDB.select(
+                #         'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
+                #         [artist])) + len(myDB.select(
+                #             'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                #             [artist]))
+                # )
+
+                try:
+                    havetracks = (
+                        len(myDB.select(
+                            'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
+                            [artistid])) + len(myDB.select(
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+                    )
+                except Exception as e:
+                    logger.warn('Error updating counts for artist: %s: %s' % (artist, e))
+
+                # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
+                # (can fix by getting rid of second len statement)
+
+                if havetracks:
+                    myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
+
+                    # Update albums to downloaded
+                    update_album_status(ArtistID=artistid)
+
+        logger.info('Found %i new artists' % len(new_artist_list))
+
+        # Add scanned artists not in the db
+        if new_artist_list:
             if headphones.CONFIG.AUTO_ADD_ARTISTS:
-                logger.info('Importing %i new artists' % len(artist_list))
+                logger.info('Importing %i new artists' % len(new_artist_list))
-                importer.artistlist_to_mbids(artist_list)
+                importer.artistlist_to_mbids(new_artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
                 # myDB.action('DELETE from newartists')
-                for artist in artist_list:
+                for artist in new_artist_list:
                     myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
 
         if headphones.CONFIG.DETECT_BITRATE and bitrates:
@@ -379,50 +426,78 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
 
-        havetracks = len(
+        artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\""
-            myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
+        try:
-                        [ArtistID])) + len(myDB.select(
+            havetracks = len(
-                            'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
-                            [ArtistName]))
+                            [ArtistID])) + len(myDB.select(
-        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+        except Exception as e:
+            logger.warn('Error updating counts for artist: %s: %s' % (ArtistName, e))
 
-    if not append:
+        if havetracks:
-        update_album_status()
+            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
+
+    # Moved above to call for each artist
+    # if not append:
+    #     update_album_status()
 
     if not append and not artistScan:
         lastfm.getSimilar()
 
-    logger.info('Library scan complete')
+    if ArtistName:
+        logger.info('Scanning complete for artist: %s', ArtistName)
+    else:
+        logger.info('Library scan complete')
 
 
 # ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
 
+# Think the above comment relates to calling from Manage Unmatched
 
-def update_album_status(AlbumID=None):
+# This used to select and update all albums and would clobber the db, changed to run by ArtistID.
+
+def update_album_status(AlbumID=None, ArtistID=None):
     myDB = db.DBConnection()
-    logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+    # logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+
     if AlbumID:
         album_status_updater = myDB.action(
-            'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+            'SELECT'
+            ' a.AlbumID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
+            'FROM'
+            ' albums AS a '
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
+            'WHERE'
+            ' a.AlbumID = ? AND a.Status != "Downloaded" '
+            'GROUP BY'
+            ' a.AlbumID '
+            'HAVING'
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
+            [AlbumID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
+        )
     else:
-        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
+        album_status_updater = myDB.action(
-    for album in album_status_updater:
+            'SELECT'
-        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?',
+            ' a.AlbumID, a.ArtistID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
-                                    [album['AlbumID']])
+            'FROM'
-        total_tracks = 0
+            ' albums AS a '
-        have_tracks = 0
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
-        for track in track_counter:
+            'WHERE'
-            total_tracks += 1
+            ' a.ArtistID = ? AND a.Status != "Downloaded" '
-            if track['Location']:
+            'GROUP BY'
-                have_tracks += 1
+            ' a.AlbumID '
-        if total_tracks != 0:
+            'HAVING'
-            album_completion = float(float(have_tracks) / float(total_tracks)) * 100
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
-        else:
+            [ArtistID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
-            album_completion = 0
+        )
-            logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
 
-        if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT:
+    new_album_status = "Downloaded"
-            new_album_status = "Downloaded"
+
+    albums = []
+    for album in album_status_updater:
+        albums.append([album['AlbumID'], album['ArtistName'], album['AlbumTitle']])
+    for album in albums:
 
         # I don't think we want to change Downloaded->Skipped.....
         # I think we can only automatically change Skipped->Downloaded when updating
@@ -433,10 +508,13 @@ def update_album_status(AlbumID=None):
         #        new_album_status = "Skipped"
         #    else:
         #        new_album_status = album['Status']
-        else:
+        #     else:
-            new_album_status = album['Status']
+        #         new_album_status = album['Status']
+        #
+        #     myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
+        #     if new_album_status != album['Status']:
+        #         logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
+        # logger.info('Album status update complete')
 
-        myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
+        myDB.action('UPDATE albums SET Status = ? WHERE AlbumID = ?', [new_album_status, album[0]])
-        if new_album_status != album['Status']:
+        logger.info('Album: %s - %s. Status updated to %s' % (album[1], album[2], new_album_status))
-            logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
-    logger.info('Album status update complete')

headphones/lock.py

@@ -4,7 +4,7 @@ Locking-related classes
 
 import time
 import threading
-import Queue
+import queue
 
 import headphones.logger
 
@@ -29,7 +29,7 @@ class TimedLock(object):
         self.lock = threading.Lock()
         self.last_used = 0
         self.minimum_delta = minimum_delta
-        self.queue = Queue.Queue()
+        self.queue = queue.Queue()
 
     def __enter__(self):
         """
@@ -47,7 +47,7 @@ class TimedLock(object):
                 seconds = self.queue.get(False)
                 headphones.logger.debug('Sleeping %s (queued)', seconds)
                 time.sleep(seconds)
-            except Queue.Empty:
+            except queue.Empty:
                 continue
             self.queue.task_done()
 

headphones/logger.py

@@ -153,7 +153,8 @@ def initLogger(console=False, log_dir=False, verbose=False):
         file_formatter = logging.Formatter(
             '%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
         file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE,
-                                                    backupCount=MAX_FILES)
+                                                    backupCount=MAX_FILES,
+                                                    encoding='utf8')
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(file_formatter)
 

headphones/lyrics.py

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import htmlentitydefs
+import html.entities
 
 import re
 from headphones import logger, request
@@ -25,7 +25,7 @@ def getLyrics(artist, song):
               "fmt": 'xml'
               }
 
-    url = 'http://lyrics.wikia.com/api.php'
+    url = 'https://lyrics.wikia.com/api.php'
     data = request.request_minidom(url, params=params)
 
     if not data:
@@ -53,7 +53,7 @@ def getLyrics(artist, song):
             '''<div class='lyricbox'><span style="padding:1em"><a href="/Category:Instrumental" title="Instrumental">''').search(
             lyricspage)
         if m:
-            return u'(Instrumental)'
+            return '(Instrumental)'
         else:
             logger.warn('Cannot find lyrics on: %s' % lyricsurl)
             return
@@ -72,7 +72,7 @@ def convert_html_entities(s):
             name = hit[2:-1]
             try:
                 entnum = int(name)
-                s = s.replace(hit, unichr(entnum))
+                s = s.replace(hit, chr(entnum))
             except ValueError:
                 pass
 
@@ -83,7 +83,7 @@ def convert_html_entities(s):
         hits.remove(amp)
     for hit in hits:
         name = hit[1:-1]
-        if name in htmlentitydefs.name2codepoint:
+        if name in html.entities.name2codepoint:
-            s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name]))
+            s = s.replace(hit, chr(html.entities.name2codepoint[name]))
     s = s.replace(amp, "&")
     return s