gh-workflow: rename, run on pull request, change python version

Fixes for Python 3.10

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+exclude = .git,data,init-scripts,lib
+ignore = E501

.github/workflows/check.yaml

@@ -0,0 +1,29 @@
+name: check 
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, 3.10]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 .
+      - name: Test with nosetests 
+        run: |
+          nosetests

.travis.yml

@@ -1,26 +0,0 @@
-# Travis CI configuration file
-# http://about.travis-ci.org/docs/
-
-language: python
-
-sudo: false
-
-cache:
-  pip: true
-  directories:
-    - lib
-
-python:
-  - "2.6"
-  - "2.7"
-
-install:
-  - pip install -r requirements-dev.txt
-
-script:
-  - pep8 headphones
-  - pyflakes headphones
-  - nosetests
-
-after_success:
-  - if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then coveralls; fi

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v0.5.20
+Released 15 October 2021
+
+Highlights:
+* Improved: Last.fm & metacrtic api fixes
+* Improved: Various database fixes
+* Added: qBitTorrent
+* Added: Telegram notifications
+* Improved: Many more since the last release. Check the list
+
+Many thanks to @AdeHub for keeping the ship afloat
+and to everyone else who contributed.
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.19...v0.5.20).
+
+## v0.5.19
+Released 27 March 2018
+
+Highlights:
+* Improved: Windows encoding fixes
+* Improved: Performance enhancements
+* Improved: Many more since the last release. Check the list
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.18...v0.5.19).
+
 ## v0.5.18
 Released 01 December 2016
 

Headphones.py

@@ -17,6 +17,10 @@
 import os
 import sys
 
+if sys.version_info <= (3, 5):
+    sys.stdout.write("Headphones requires Python >= 3.5\n")
+    sys.exit(1)
+
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib/'))
 
@@ -54,7 +58,10 @@ def main():
 
     try:
         locale.setlocale(locale.LC_ALL, "")
-        headphones.SYS_ENCODING = locale.getpreferredencoding()
+        if headphones.SYS_PLATFORM == 'win32':
+            headphones.SYS_ENCODING = sys.getdefaultencoding().upper()
+        else:
+            headphones.SYS_ENCODING = locale.getpreferredencoding()
     except (locale.Error, IOError):
         pass
 

README.md

@@ -1,4 +1,4 @@
-##![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
+## ![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
 
 [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=master)](https://travis-ci.org/rembo10/headphones)
 [![Build Status](https://img.shields.io/travis/rembo10/headphones/develop.svg?label=develop%20branch%20build)](https://travis-ci.org/rembo10/headphones)

contrib/sni_test.py

@@ -6,7 +6,7 @@ import sys
 # Ensure that we use the Headphones provided libraries.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib"))
 
-import urlparse
+import urllib.parse
 
 
 def can_import(module):
@@ -89,7 +89,7 @@ def main():
         url = sys.argv[1]
 
     # Check if it is a HTTPS website.
-    parts = urlparse.urlparse(url)
+    parts = urllib.parse.urlparse(url)
 
     if parts.scheme.lower() != "https":
         sys.stderr.write(

data/interfaces/default/album.html

@@ -324,7 +324,7 @@
                 if (++x === 10) {
                     clearInterval(refreshInterval);
                 }
-                var sts = $("#artistname").text();
+                var sts = $("#artistname").text().trim();
                 if (wasLoading == true || sts == "Loading"){
                     location.reload();
                 $('#albumnamespinner').remove()

data/interfaces/default/config.html

@@ -133,6 +133,12 @@
                                     </label>
                                     <input type="text" name="mb_ignore_age" value="${config['mb_ignore_age']}" size="4">days
                                 </div>
+                                <div class="row checkbox">
+                                    <label title="Ignore MusicBrainz album updates missing a release date.">
+                                        Ignore Undated Releases
+                                    </label>
+                                    <input type="checkbox" id="mb_ignore_age_missing" name="mb_ignore_age_missing" value="1" ${config['mb_ignore_age_missing']}
+                                </div>                                
                             </fieldset>
                         </td>
                     </tr>
@@ -312,6 +318,7 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_qbittorrent" value="4" ${config['torrent_downloader_qbittorrent']}> QBitTorrent
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
                                 <div class="row">
@@ -386,6 +393,26 @@
                                     <input type="text" name="utorrent_label" value="${config['utorrent_label']}" size="30">
                                 </div>
                             </fieldset>
+                            <fieldset id="qbittorrent_options">
+                                <small class="heading"><i class="fa fa-info-circle"></i> Note: Works with WebAPI Rev 11 and later (QBitTorrent 3.4.0 and later) </small>
+                                <div class="row">
+                                    <label>QBitTorrent Host</label>
+                                    <input type="text" name="qbittorrent_host" value="${config['qbittorrent_host']}" size="30">
+                                    <small>usually http://localhost:8081</small>
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Username</label>
+                                    <input type="text" name="qbittorrent_username" value="${config['qbittorrent_username']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Password</label>
+                                    <input type="password" name="qbittorrent_password" value="${config['qbittorrent_password']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Label</label>
+                                    <input type="text" name="qbittorrent_label" value="${config['qbittorrent_label']}" size="30">
+				</div>
+                            </fieldset>
                             <fieldset id="deluge_options">
                                 <div class="row">
                                     <label>Deluge WebUI Host and Port</label>
@@ -574,23 +601,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_kat" type="checkbox" class="bigcheck" name="use_kat" value="1" ${config['use_kat']} /><label for="use_kat"><span class="option">Kick Ass Torrents</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Proxy URL</label>
-                                            <input type="text" name="kat_proxy_url" value="${config['kat_proxy_url']}" size="36">
-                                            <small>Optional. Leave empty for default.</small>
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="kat_ratio" value="${config['kat_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                                 <fieldset>
                                     <div class="row checkbox left">
                                         <input id="use_waffles" type="checkbox" class="bigcheck" name="use_waffles" value="1" ${config['use_waffles']} /><label for="use_waffles"><span class="option">Waffles.ch</span></label>
@@ -628,65 +638,57 @@
                                             <label>Seed Ratio</label>
                                             <input type="text" class="override-float" name="rutracker_ratio" value="${config['rutracker_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
+                                        <div class="row">
+                                            <label>Session Cookie (Optional - Advanced)</label>
+                                            <input type="text" class="override-float" name="rutracker_cookie" value="${config['rutracker_cookie']}" size="10" title="bb_session cookie (Advanced)">
+                                        </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_whatcd" type="checkbox" class="bigcheck" name="use_whatcd" value="1" ${config['use_whatcd']} /><label for="use_whatcd"><span class="option">What.cd</span></label>
+                                        <input id="use_orpheus" type="checkbox" class="bigcheck" name="use_orpheus" value="1" ${config['use_orpheus']} /><label for="use_orpheus"><span class="option">Orpheus.network</span></label>
                                     </div>
                                     <div class="config">
                                         <div class="row">
                                             <label>Username</label>
-                                            <input type="text" name="whatcd_username" value="${config['whatcd_username']}" size="36">
+                                            <input type="text" name="orpheus_username" value="${config['orpheus_username']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="whatcd_password" value="${config['whatcd_password'] | h}" size="36">
+                                            <input type="password" name="orpheus_password" value="${config['orpheus_password'] | h}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>URL</label>
-                                            <input type="text" name="whatcd_url" value="${config['whatcd_url']}" size="36">
+                                            <input type="text" name="orpheus_url" value="${config['orpheus_url']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="whatcd_ratio" value="${config['whatcd_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                            <input type="text" class="override-float" name="orpheus_ratio" value="${config['orpheus_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_pth" type="checkbox" class="bigcheck" name="use_pth" value="1" ${config['use_pth']} /><label for="use_pth"><span class="option">PassTheHeadphones.me</span></label>
+                                        <input id="use_redacted" type="checkbox" class="bigcheck" name="use_redacted" value="1" ${config['use_redacted']} /><label for="use_redacted"><span class="option">Redacted</span></label>
                                     </div>
                                     <div class="config">
                                         <div class="row">
                                             <label>Username</label>
-                                            <input type="text" name="pth_username" value="${config['pth_username']}" size="36">
+                                            <input type="text" name="redacted_username" value="${config['redacted_username']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="pth_password" value="${config['pth_password']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>URL</label>
-                                            <input type="text" name="pth_url" value="${config['pth_url']}" size="36">
+                                            <input type="password" name="redacted_password" value="${config['redacted_password'] | h}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="pth_ratio" value="${config['pth_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                            <input type="text" class="override-float" name="redacted_ratio" value="${config['redacted_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
-                                    </div>
-                                </fieldset>
-
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_strike" type="checkbox" class="bigcheck" name="use_strike" value="1" ${config['use_strike']} /><label for="use_strike"><span class="option">Strike Search</span></label>
-                                    </div>
-                                    <div class="config">
                                         <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="strike_ratio" value="${config['strike_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            <label>Use freeleech tokens when available</label>
+                                            <input type="checkbox" name="redacted_use_fltoken" value="1" ${config['redacted_use_fltoken']}>
                                         </div>
                                     </div>
                                 </fieldset>
@@ -699,13 +701,17 @@
                                         <div class="config" id="torznab1">
                                             <div class="row">
                                                 <label>Torznab Host</label>
-                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="30">
-                                                <small>e.g. http://localhost:9117/torznab/iptorrents</small>
+                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="36">
+                                                <small>e.g. http://localhost:9117/api/v2.0/indexers/demonoid/results/torznab/</small>
                                             </div>
                                             <div class="row">
                                                 <label>Torznab API</label>
                                                 <input type="text" name="torznab_apikey" value="${config['torznab_apikey']}" size="36">
                                             </div>
+                                            <div class="row">
+                                                <label>Torznab Seed Ratio</label>
+                                                <input type="text" class="override-float" name="torznab_ratio" value="${config['torznab_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            </div>
                                             <div class="row checkbox">
                                                 <input id="torznab_enabled" type="checkbox" name="torznab_enabled" value="1" ${config['torznab_enabled']} /><label>Enabled</label>
                                             </div>
@@ -715,7 +721,7 @@
                                         %>
                                         %for torznab in config['extra_torznabs']:
                                             <%
-                                                if torznab[2] == '1' or torznab[2] == 1:
+                                                if torznab[3] == '1' or torznab[3] == 1:
                                                     torznab_enabled = "checked"
                                                 else:
                                                     torznab_enabled = ""
@@ -723,12 +729,16 @@
                                             <div class="config" id="torznab${torznab_number}">
                                                 <div class="row">
                                                     <label>Torznab Host</label>
-                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="30">
+                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="36">
                                                 </div>
                                                 <div class="row">
                                                     <label>Torznab API</label>
                                                     <input type="text" name="torznab_api${torznab_number}" value="${torznab[1]}" size="36">
                                                 </div>
+                                                <div class="row">
+                                                    <label>Torznab Seed Ratio</label>
+                                                    <input type="text" class="override-float" name="torznab_ratio${torznab_number}" value="${torznab[2]}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                                </div>
                                                 <div class="row checkbox">
                                                     <input id="torznab_enabled" type="checkbox" name="torznab_enabled${torznab_number}" value="1" ${torznab_enabled} /><label>Enabled</label>
                                                 </div>
@@ -760,34 +770,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                     <div class="row checkbox left">
-                                        <input id="use_mininova" type="checkbox" class="bigcheck" name="use_mininova" value="1" ${config['use_mininova']} /><label for="use_mininova"><span class="option">Mininova</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="mininova_ratio" value="${config['mininova_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-								
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_tquattrecentonze" type="checkbox" class="bigcheck" name="use_tquattrecentonze" value="1" ${config['use_tquattrecentonze']} /><label for="use_tquattrecentonze"><span class="option">t411</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Username</label>
-                                            <input type="text" name="tquattrecentonze_user" value="${config['tquattrecentonze_user']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Password</label>
-                                            <input type="password" name="tquattrecentonze_password" value="${config['tquattrecentonze_password'] | h}" size="36">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                             </fieldset>
                         </td>
                     </tr>
@@ -928,30 +910,42 @@
                                         <input type="checkbox" name="keep_nfo" value="1" ${config['keep_nfo']} />
                                     </label>
                                 </div>
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed lyrics
+                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
+                                    </label>
+                                </div>
+
+
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed album art in each file
+                                        <input type="checkbox" name="embed_album_art" id="embed_album_art" value="1" ${config['embed_album_art']}>
+                                    </label>
+                                </div>
+
                                 <div class="row checkbox left clearfix nopad">
                                     <label>
                                         Add album art jpeg to album folder
                                         <input type="checkbox" name="add_album_art" id="add_album_art" value="1" ${config['add_album_art']}>
                                     </label>
                                 </div>
+
                                 <div id="album_art_options" style="padding-left: 20px">
                                     <div class="row">
                                         as <input type="text" class="override-float" name="album_art_format" value="${config['album_art_format']}" size="10">.jpg
                                     </div>
                                     <small>Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
                                 </div>
-                                <div class="row checkbox left clearfix nopad">
-                                    <label>
-                                        Embed album art in each file
-                                        <input type="checkbox" name="embed_album_art" value="1" ${config['embed_album_art']}>
-                                    </label>
-                                </div>
-                                <div class="row checkbox left clearfix nopad">
-                                    <label>
-                                        Embed lyrics
-                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
-                                    </label>
+
+                                <div id="album_art_size_options" style="padding-left: 20px">
+                                    <div class="row">
+                                        Album art min width <input type="text" class="override-float" name="album_art_min_width" value="${config['album_art_min_width']}" size="4" title="Only images with a pixel width greater or equal are considered as valid album art candidates. Default: 0.">\
+                                        Album art max width <input type="text" class="override-float" name="album_art_max_width" value="${config['album_art_max_width']}" size="4" title="A maximum image width to downscale fetched images if they are too big.">
+                                    </div>
                                 </div>
+
                                 <div class="row">
                                     <label>
                                         Destination Directory:
@@ -1280,7 +1274,7 @@
                                     <input type="checkbox" name="synoindex_enabled" id="synoindex" value="1" ${config['synoindex_enabled']} /><label for="synoindex"><span class="option">Synology NAS</span></label>
                                 </div>
                             </fieldset>
-
+<!--
                             <fieldset>
                                 <div class="row checkbox left">
                                     <input type="checkbox" class="bigcheck" name="twitter_enabled" id="twitter" value="1" ${config['twitter_enabled']} /><label for="twitter"><span class="option">Twitter</span></label>
@@ -1301,6 +1295,26 @@
                                     </div>
                                 </div>
                             </fieldset>
+-->
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="slack_enabled" id="slack" value="1" ${config['slack_enabled']} /><label for="slack"><span class="option">Slack</span></label>
+                                </div>
+                                <div id="slackoptions">
+                                    <div class="row">
+                                        <label>Slack Webhook</label><input type="text" name="slack_url" value="${config['slack_url']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Channel</label><input type="text" name="slack_channel" value="${config['slack_channel']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <label>Emoji</label><input type="text" name="slack_emoji" value="${config['slack_emoji']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="slack_onsnatch" value="1" ${config['slack_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
 
                             <fieldset>
                                 <div class="row checkbox left">
@@ -1319,6 +1333,23 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="join_enabled" id="join" value="1" ${config['join_enabled']} /><label for="join"><span class="option">Join</span></label>
+                                </div>
+                                <div id="joinoptions">
+                                    <div class="row">
+                                        <label>Join API Key</label><input type="text" name="join_apikey" value="${config['join_apikey']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Device ID(s)</label><input type="text" name="join_deviceid" value="${config['join_deviceid']}" size="50"><small>Comma separated list.  Leave blank to send to all devices</small>
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="join_onsnatch" value="1" ${config['join_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
+
                         </td>
                     </tr>
                 </table>
@@ -1614,6 +1645,10 @@
                                     <label>Cache Directory</label>
                                     <input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
                                 </div>
+                                <div class="row">
+                                    <label>Post Processing Temporary Directory</label>
+                                    <input type="text" name="keep_torrent_files_dir" value="${config['keep_torrent_files_dir']}" size="50" title="Enter the directory to preserve files for seeding, default is system temp directory">
+                                </div>
                             </fieldset>
 
                             <fieldset>
@@ -1858,9 +1893,36 @@
                         }
                 });
 
+                if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").show();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").hide();
+                              }
+                        }
+
+                $("#embed_album_art").click(function(){
+                        if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").slideDown();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
+                        }
+                });
+
                 if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").show();
+                            $("#album_art_size_options").show();
                         }
                 else
                         {
@@ -1871,10 +1933,15 @@
                         if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").slideDown();
+                            $("#album_art_size_options").slideDown();
                         }
                         else
                         {
                             $("#album_art_options").slideUp();
+                            if (!$("#embed_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
                         }
                 });
 
@@ -2079,6 +2146,27 @@
                         }
                 });
 
+                if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").show();
+                        }
+                else
+                        {
+                            $("#joinoptions").hide();
+                        }
+
+
+                $("#join").click(function(){
+                        if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#joinoptions").slideUp();
+                        }
+                });
+
                 if ($("#twitter").is(":checked"))
                         {
                             $("#twitteroptions").show();
@@ -2099,6 +2187,27 @@
                         }
                 });
 
+                if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").show();
+                        }
+                else
+                        {
+                            $("#slackoptions").hide();
+                        }
+
+                $("#slack").click(function(){
+                        if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#slackoptions").slideUp();
+                        }
+                });
+
+
                 if ($("#telegram").is(":checked"))
                         {
                             $("#telegramoptions").show();
@@ -2275,26 +2384,30 @@
 
                 if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#torrent_blackhole_options").show();
                         }
                 if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#transmission_options").show();
                         }
                 if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").hide();
                             $("#utorrent_options").show();
                         }
+                if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#qbittorrent_options").show();
+                        }
                 if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#qbittorrent_options").hide();
                             $("#deluge_options").show();
                         }
 
-
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
                         {
@@ -2330,19 +2443,23 @@
                         }
                         if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
                         }
+                        if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#qbittorrent_options").fadeIn() });
+			}
                         if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
                 });
 
@@ -2404,7 +2521,7 @@
 
                 $("#add_torznab").click(function() {
                     var intId = $("#torznab_providers > div").size() + deletedTorznabs + 1;
-                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
+                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row\"><label>Torznab Seed Ratio</label><input type=\"text\" class=\"override-float\" name=\"torznab_ratio" + intId + "\" size=\"10\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
                     var removeTorznabButton = $("<div class=\"row\"><input type=\"button\" class=\"remove_torznab\" value=\"Remove\" /></div>");
                     removeTorznabButton.click(function() {
                         $(this).parent().remove();
@@ -2433,20 +2550,16 @@
                 initConfigCheckbox("#use_nzbsorg");
                 initConfigCheckbox("#use_omgwtfnzbs");
                 initConfigCheckbox("#use_torznab");
-                initConfigCheckbox("#use_kat");
                 initConfigCheckbox("#use_piratebay");
                 initConfigCheckbox("#use_oldpiratebay");
-                initConfigCheckbox("#use_mininova");
                 initConfigCheckbox("#use_waffles");
                 initConfigCheckbox("#use_rutracker");
-                initConfigCheckbox("#use_whatcd");
-                initConfigCheckbox("#use_pth");
-                initConfigCheckbox("#use_strike");
+                initConfigCheckbox("#use_orpheus");
+                initConfigCheckbox("#use_redacted");
                 initConfigCheckbox("#api_enabled");
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");
-                initConfigCheckbox("#use_tquattrecentonze");
-
+                initConfigCheckbox("#mb_ignore_age_missing");
 
                 $('#twitterStep1').click(function () {
                     $.get("/twitterStep1", function (data) {window.open(data); })

data/interfaces/default/history.html

@@ -1,7 +1,7 @@
 <%inherit file="base.html"/>
 <%!
 	from headphones import helpers
-	import cgi
+	from html import escape as html_escape 
 %>
 
 <%def name="headerIncludes()">
@@ -62,11 +62,11 @@
         %>
 			<tr class="grade${grade}">
 				<td id="dateadded">${item['DateAdded']}</td>
-				<td id="filename">${cgi.escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
+				<td id="filename">${html_escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
 				<td id="size">${helpers.bytes_to_mb(item['Size'])}</td>
 				<td title="${folder}" id="status">${item['Status']}</td>
-				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${cgi.escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${cgi.escape(item['Title'], quote=True)}'">new</a>]</td>
-				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${cgi.escape(item['Title'], quote=True)}',$(this),'table')" data-success="${cgi.escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
+				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${html_escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${html_escape(item['Title'], quote=True)}'">new</a>]</td>
+				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${html_escape(item['Title'], quote=True)}',$(this),'table')" data-success="${html_escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
 			</tr>
 		%endfor
 		</tbody>

data/interfaces/default/index.html

@@ -36,7 +36,7 @@
                         "aTargets": [0],
                         "mData":"ArtistID",
                         "mRender": function ( data, type, full ) {
-                            return '<div id="artistImg"><img class="albumArt" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
+                            return '<div id="artistImg"><img class="albumArt" height="50" width="50" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
                         }
                     },
                     {

data/interfaces/default/js/script.js

@@ -312,6 +312,7 @@ function doAjaxCall(url,elem,reload,form) {
 	$.ajax({
 	  url: url,
 	  data: dataString,
+	  type: 'POST',
 	  beforeSend: function(jqXHR, settings) {
 	  	// Start loader etc.
 	  	feedback.prepend(loader);

headphones/__init__.py

@@ -86,8 +86,8 @@ CURRENT_VERSION = None
 LATEST_VERSION = None
 COMMITS_BEHIND = None
 
-LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"]
-LOSSLESS_MEDIA_FORMATS = ["flac", "aiff"]
+LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma", "opus"]
+LOSSLESS_MEDIA_FORMATS = ["flac", "aiff", "aif"]
 MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS
 
 MIRRORLIST = ["musicbrainz.org", "headphones", "custom"]
@@ -218,7 +218,7 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno)
 
     os.setsid()
@@ -232,10 +232,10 @@ def daemonize():
         pid = os.fork()  # @UndefinedVariable - only available in UNIX
         if pid != 0:
             sys.exit(0)
-    except OSError, e:
+    except OSError as e:
         raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno)
 
-    dev_null = file('/dev/null', 'r')
+    dev_null = open('/dev/null', 'r')
     os.dup2(dev_null.fileno(), sys.stdin.fileno())
 
     si = open('/dev/null', "r")
@@ -251,7 +251,7 @@ def daemonize():
 
     if CREATEPID:
         logger.info("Writing PID %d to %s", pid, PIDFILE)
-        with file(PIDFILE, 'w') as fp:
+        with open(PIDFILE, 'w') as fp:
             fp.write("%s\n" % pid)
 
 
@@ -306,9 +306,10 @@ def initialize_scheduler():
                          minutes=minutes)
 
         # Remove Torrent + data if Post Processed and finished Seeding
-        minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
-        schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
-                     minutes=minutes)
+        if headphones.CONFIG.TORRENT_DOWNLOADER != 0:
+            minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
+            schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
+                         minutes=minutes)
 
         # Start scheduler
         if start_jobs and len(SCHED.get_jobs()):
@@ -359,6 +360,8 @@ def sig_handler(signum=None, frame=None):
 
 
 def dbcheck():
+    logger.debug("SQLite Version: %s", sqlite3.sqlite_version)
+    logger.debug("DB-API Version: %s", sqlite3.version)
     conn = sqlite3.connect(DB_FILE)
     c = conn.cursor()
     c.execute(
@@ -376,7 +379,7 @@ def dbcheck():
     c.execute(
         'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)')
     c.execute(
-        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)')
+        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT, TorrentHash TEXT)')
     # Matched is a temporary value used to see if there was a match found in
     # alltracks
     c.execute(
@@ -416,6 +419,33 @@ def dbcheck():
         'CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)')
     c.execute(
         'CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)')
+    c.execute(
+        'CREATE INDEX IF NOT EXISTS tracks_artistid ON tracks(ArtistID ASC)')
+
+    # Speed up album page
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_albumid ON allalbums(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_albumid ON alltracks(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS releases_albumid ON releases(ReleaseGroupID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_albumid ON descriptions(ReleaseGroupID ASC)')
+
+    # Speed up artist deletion
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_artistid ON allalbums(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_artistid ON alltracks(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_artistid ON descriptions(ArtistID ASC)')
+
+    # Speed up Artist refresh hybrid release
+    c.execute('CREATE INDEX IF NOT EXISTS albums_releaseid ON albums(ReleaseID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_releaseid ON tracks(ReleaseID ASC)')
+
+    # Speed up scanning and track matching
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistname ON artists(ArtistName COLLATE NOCASE ASC)')
+
+    # General speed up
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistsortname ON artists(ArtistSortName COLLATE NOCASE ASC)')
+
+    c.execute(
+        """CREATE INDEX IF NOT EXISTS have_matched_artist_album ON have(Matched ASC, ArtistName COLLATE NOCASE ASC, AlbumTitle COLLATE NOCASE ASC)""")
+    c.execute('DROP INDEX IF EXISTS have_matched')
 
     try:
         c.execute('SELECT IncludeExtras from artists')
@@ -613,6 +643,20 @@ def dbcheck():
     except sqlite3.OperationalError:
         c.execute('ALTER TABLE artists ADD COLUMN MetaCritic TEXT DEFAULT NULL')
 
+    try:
+        c.execute('SELECT TorrentHash from snatched')
+    except sqlite3.OperationalError:
+        c.execute('ALTER TABLE snatched ADD COLUMN TorrentHash TEXT')
+        c.execute('UPDATE snatched SET TorrentHash = FolderName WHERE Status LIKE "Seed_%"')
+
+    # One off script to set CleanName to lower case
+    clean_name_mixed = c.execute('SELECT CleanName FROM have ORDER BY Date Desc').fetchone()
+    if clean_name_mixed and clean_name_mixed[0] != clean_name_mixed[0].lower():
+        logger.info("Updating track clean name, this could take some time...")
+        c.execute('UPDATE tracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE alltracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE have SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+
     conn.commit()
     c.close()
 

headphones/albumart.py

@@ -13,16 +13,171 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import request, db, logger
+import struct
+from six.moves.urllib.parse import urlencode
+from io import BytesIO
+
+import headphones
+from headphones import db, request, logger
 
 
 def getAlbumArt(albumid):
-    myDB = db.DBConnection()
-    asin = myDB.action(
-        'SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0]
 
-    if asin:
-        return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
+    artwork_path = None
+    artwork = None
+
+    # CAA
+    logger.info("Searching for artwork at CAA")
+    artwork_path = 'https://coverartarchive.org/release-group/%s/front' % albumid
+    artwork = getartwork(artwork_path)
+    if artwork:
+        logger.info("Artwork found at CAA")
+        return artwork_path, artwork
+
+    # Amazon
+    logger.info("Searching for artwork at Amazon")
+    myDB = db.DBConnection()
+    dbalbum = myDB.action(
+        'SELECT ArtistName, AlbumTitle, ReleaseID, AlbumASIN FROM albums WHERE AlbumID=?',
+        [albumid]).fetchone()
+    if dbalbum['AlbumASIN']:
+        artwork_path = 'https://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % dbalbum['AlbumASIN']
+        artwork = getartwork(artwork_path)
+        if artwork:
+            logger.info("Artwork found at Amazon")
+            return artwork_path, artwork
+
+    # last.fm
+    from headphones import lastfm
+    logger.info("Searching for artwork at last.fm")
+    if dbalbum['ReleaseID'] != albumid:
+        data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'])
+        if not data:
+            data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                         album=dbalbum['AlbumTitle'])
+    else:
+        data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                     album=dbalbum['AlbumTitle'])
+
+    if data:
+        try:
+            images = data['album']['image']
+            for image in images:
+                if image['size'] == 'extralarge':
+                    artwork_path = image['#text']
+                elif image['size'] == 'mega':
+                    artwork_path = image['#text']
+                    break
+        except KeyError:
+            artwork_path = None
+
+        if artwork_path:
+            artwork = getartwork(artwork_path)
+            if artwork:
+                logger.info("Artwork found at last.fm")
+                return artwork_path, artwork
+
+    logger.info("No suitable album art found.")
+    return None, None
+
+
+def jpeg(bites):
+    fhandle = BytesIO(bites)
+    try:
+        fhandle.seek(0)
+        size = 2
+        ftype = 0
+        while not 0xc0 <= ftype <= 0xcf:
+            fhandle.seek(size, 1)
+            byte = fhandle.read(1)
+            while ord(byte) == 0xff:
+                byte = fhandle.read(1)
+            ftype = ord(byte)
+            size = struct.unpack('>H', fhandle.read(2))[0] - 2
+        fhandle.seek(1, 1)
+        height, width = struct.unpack('>HH', fhandle.read(4))
+        return width, height
+    except struct.error:
+        return None, None
+    except TypeError:
+        return None, None
+
+
+def png(bites):
+    try:
+        check = struct.unpack('>i', bites[4:8])[0]
+        if check != 0x0d0a1a0a:
+            return None, None
+        return struct.unpack('>ii', bites[16:24])
+    except struct.error:
+        return None, None
+
+
+def get_image_data(bites):
+    type = None
+    width = None
+    height = None
+    if len(bites) < 24:
+        return None, None, None
+
+    peek = bites[0:2]
+    if peek == b'\xff\xd8':
+        width, height = jpeg(bites)
+        type = 'jpg'
+    elif peek == b'\x89P':
+        width, height = png(bites)
+        type = 'png'
+    return type, width, height
+
+
+def getartwork(artwork_path):
+    artwork = bytes()
+    minwidth = 0
+    maxwidth = 0
+    if headphones.CONFIG.ALBUM_ART_MIN_WIDTH:
+        minwidth = int(headphones.CONFIG.ALBUM_ART_MIN_WIDTH)
+    if headphones.CONFIG.ALBUM_ART_MAX_WIDTH:
+        maxwidth = int(headphones.CONFIG.ALBUM_ART_MAX_WIDTH)
+
+    resp = request.request_response(artwork_path, timeout=20, stream=True, whitelist_status_code=404)
+
+    if resp:
+        img_width = None
+        for chunk in resp.iter_content(chunk_size=1024):
+            artwork += chunk
+            if not img_width and (minwidth or maxwidth):
+                img_type, img_width, img_height = get_image_data(artwork)
+            # Check min/max
+            if img_width and (minwidth or maxwidth):
+                if minwidth and img_width < minwidth:
+                    logger.info("Artwork is too small. Type: %s. Width: %s. Height: %s",
+                                img_type, img_width, img_height)
+                    artwork = None
+                    break
+                elif maxwidth and img_width > maxwidth:
+                    # Downsize using proxy service to max width
+                    artwork = bytes()
+                    url = "https://images.weserv.nl"
+                    params = {
+                        "url": artwork_path,
+                        "w": maxwidth
+                    }
+                    r = request.request_response(
+                        url,
+                        params=params,
+                        timeout=20,
+                        stream=True,
+                        whitelist_status_code=404
+                    )
+                    if r:
+                        for chunk in r.iter_content(chunk_size=1024):
+                            artwork += chunk
+                        r.close()
+                        logger.info("Artwork is greater than the maximum width, downsized using proxy service")
+                    break
+        resp.close()
+
+    return artwork
 
 
 def getCachedArt(albumid):
@@ -34,7 +189,7 @@ def getCachedArt(albumid):
     if not artwork_path:
         return
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith("http"):
         artwork = request.request_content(artwork_path, timeout=20)
 
         if not artwork:

headphones/albumswitcher.py

@@ -85,8 +85,8 @@ def switch(AlbumID, ReleaseID):
 
     # Update have track counts on index
     totaltracks = len(myDB.select(
-        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")',
-        [newalbumdata['ArtistID']]))
+        'SELECT TrackTitle from tracks AS tr INNER JOIN albums AS al ON al.AlbumID = tr.AlbumID WHERE al.ArtistID=? '
+        'AND al.Status != "Ignored"', [newalbumdata['ArtistID']]))
     havetracks = len(myDB.select(
         'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
         [newalbumdata['ArtistID']]))

headphones/api.py

@@ -82,11 +82,11 @@ class Api(object):
     def fetchData(self):
 
         if self.data == 'OK':
-            logger.info('Recieved API command: %s', self.cmd)
+            logger.info('Received API command: %s', self.cmd)
             methodToCall = getattr(self, "_" + self.cmd)
             methodToCall(**self.kwargs)
             if 'callback' not in self.kwargs:
-                if isinstance(self.data, basestring):
+                if isinstance(self.data, str):
                     return self.data
                 else:
                     return json.dumps(self.data)
@@ -106,7 +106,7 @@ class Api(object):
         rows_as_dic = []
 
         for row in rows:
-            row_as_dic = dict(zip(row.keys(), row))
+            row_as_dic = dict(list(zip(list(row.keys()), row)))
             rows_as_dic.append(row_as_dic)
 
         return rows_as_dic
@@ -474,17 +474,21 @@ class Api(object):
         # Handle situations where the torrent url contains arguments that are
         # parsed
         if kwargs:
-            import urllib
-            import urllib2
-            url = urllib2.quote(
-                url, safe=":?/=&") + '&' + urllib.urlencode(kwargs)
+            import urllib.request
+            import urllib.parse
+            import urllib.error
+            import urllib.request
+            import urllib.error
+            import urllib.parse
+            url = urllib.parse.quote(
+                url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
 
         try:
             result = [(title, int(size), url, provider, kind)]
         except ValueError:
             result = [(title, float(size), url, provider, kind)]
 
-        logger.info(u"Making sure we can download the chosen result")
+        logger.info("Making sure we can download the chosen result")
         (data, bestqual) = searcher.preprocess(result)
 
         if data and bestqual:

headphones/cache.py

@@ -14,11 +14,16 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from six.moves.urllib.parse import urlencode
 
 import headphones
-from headphones import db, helpers, logger, lastfm, request
+from headphones import db, helpers, logger, lastfm, request, mb
 
-LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
+LASTFM_API_KEY = "8d983789c771afaeb7412ac358d4bad0"
+
+FANART_URL = 'https://webservice.fanart.tv/v3/music/'
+FANART_PROJECT_KEY = '22b73c9603eba09d0c855f2d2bdba31c'
+FANART_CLIENT_KEY = '919b389a18a3f0b2c916090022ab3c7a'
 
 
 class Cache(object):
@@ -96,6 +101,11 @@ class Cache(object):
     def _is_current(self, filename=None, date=None):
 
         if filename:
+
+            # 2019 last.fm no longer allows access to artist images, for now we'll keep the cached artist image if it exists and get new from fanart.tv
+            if self.id_type == 'artist' and 'fanart' not in filename:
+                return True
+
             base_filename = os.path.basename(filename)
             date = base_filename.split('.')[1]
 
@@ -211,20 +221,34 @@ class Cache(object):
         if ArtistID:
 
             self.id_type = 'artist'
-            data = lastfm.request_lastfm("artist.getinfo", mbid=ArtistID, api_key=LASTFM_API_KEY)
+
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + ArtistID, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
 
             if not data:
                 return
 
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except (KeyError, IndexError):
-                logger.debug('No artist image found')
-                image_url = None
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
 
-            thumb_url = self._get_thumb_url(data)
-            if not thumb_url:
-                logger.debug('No artist thumbnail image found')
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in list(data.get('albums', dict()).items()):
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            if image_url:
+                thumb_url = image_url
+            else:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
 
@@ -283,6 +307,7 @@ class Cache(object):
         """
 
         myDB = db.DBConnection()
+        fanart = False
 
         # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums
         # Exception is when adding albums manually, then we should use release id
@@ -290,6 +315,14 @@ class Cache(object):
 
             data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY)
 
+            # Try with name if not found
+            if not data:
+                dbartist = myDB.action('SELECT ArtistName, Type FROM artists WHERE ArtistID=?', [self.id]).fetchone()
+                if dbartist:
+                    data = lastfm.request_lastfm("artist.getinfo",
+                                                 artist=helpers.clean_musicbrainz_name(dbartist['ArtistName']),
+                                                 api_key=LASTFM_API_KEY)
+
             if not data:
                 return
 
@@ -303,30 +336,70 @@ class Cache(object):
             except KeyError:
                 logger.debug('No artist bio found')
                 self.info_content = None
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except KeyError:
-                logger.debug('No artist image found')
-                image_url = None
 
-            thumb_url = self._get_thumb_url(data)
-            if not thumb_url:
-                logger.debug('No artist thumbnail image found')
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + self.id, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
+
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
+
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in list(data.get('albums', dict()).items()):
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            # finally, use 1st album cover from last.fm
+            if image_url:
+                fanart = True
+                thumb_url = image_url
+            else:
+                dbalbum = myDB.action(
+                    'SELECT ArtworkURL, ThumbURL FROM albums WHERE ArtworkURL IS NOT NULL AND ArtistID=?',
+                    [self.id]).fetchone()
+                if dbalbum:
+                    fanart = True
+                    image_url = dbalbum['ArtworkURL']
+                    thumb_url = dbalbum['ThumbURL']
+
+            if not image_url:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
             dbalbum = myDB.action(
-                'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?',
+                'SELECT ArtistName, AlbumTitle, ReleaseID, Type FROM albums WHERE AlbumID=?',
                 [self.id]).fetchone()
             if dbalbum['ReleaseID'] != self.id:
                 data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'],
                                              api_key=LASTFM_API_KEY)
                 if not data:
-                    data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
-                                                 album=dbalbum['AlbumTitle'],
+                    data = lastfm.request_lastfm("album.getinfo",
+                                                 artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                 album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
                                                  api_key=LASTFM_API_KEY)
             else:
-                data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
-                                             album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
+                if dbalbum['Type'] != "part of":
+                    data = lastfm.request_lastfm("album.getinfo",
+                                                 artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                 album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                 api_key=LASTFM_API_KEY)
+                else:
+
+                    # Series, use actual artist for the release-group
+                    artist = mb.getArtistForReleaseGroup(self.id)
+                    if artist:
+                        data = lastfm.request_lastfm("album.getinfo",
+                                                     artist=helpers.clean_musicbrainz_name(artist),
+                                                     album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                     api_key=LASTFM_API_KEY)
 
             if not data:
                 return
@@ -406,8 +479,12 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                artwork_path = os.path.join(self.path_to_art_cache,
-                                            self.id + '.' + helpers.today() + ext)
+                if fanart:
+                    artwork_path = os.path.join(self.path_to_art_cache,
+                                                self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    artwork_path = os.path.join(self.path_to_art_cache,
+                                                self.id + '.' + helpers.today() + ext)
                 try:
                     with open(artwork_path, 'wb') as f:
                         f.write(artwork)
@@ -422,7 +499,9 @@ class Cache(object):
         # as it's missing/outdated.
         if thumb_url and self.query_type in ['thumb', 'artwork'] and not (
                 self.thumb_files and self._is_current(self.thumb_files[0])):
-            artwork = request.request_content(thumb_url, timeout=20)
+
+            if not (self.query_type == 'artwork' and 'fanart' in thumb_url and artwork):
+                artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
                 # Make sure the artwork dir exists:
@@ -445,11 +524,39 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                thumb_path = os.path.join(self.path_to_art_cache,
-                                          'T_' + self.id + '.' + helpers.today() + ext)
+                if fanart:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '.' + helpers.today() + ext)
                 try:
-                    with open(thumb_path, 'wb') as f:
-                        f.write(artwork)
+                    if self.id_type != 'artist':
+                        with open(thumb_path, 'wb') as f:
+                            f.write(artwork)
+                    else:
+
+                        # 2019 last.fm no longer allows access to artist images, use the fanart.tv image to create a thumb
+                        artwork_thumb = None
+                        if 'fanart' in thumb_url:
+                            # Create thumb using image resizing service
+                            url = "https://images.weserv.nl"
+                            params = {
+                                "url": thumb_url,
+                                "w": 300
+                            }
+                            artwork_thumb = request.request_content(
+                                url,
+                                params=params,
+                                timeout=20,
+                                whitelist_status_code=404
+                            )
+                        if artwork_thumb:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork_thumb)
+                        else:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork)
 
                     os.chmod(thumb_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8))
                 except (OSError, IOError) as e:
@@ -465,7 +572,7 @@ def getArtwork(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         artwork_file = os.path.basename(artwork_path)
@@ -479,7 +586,7 @@ def getThumb(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         thumbnail_file = os.path.basename(artwork_path)

headphones/classes.py

@@ -18,12 +18,14 @@
 #######################################
 
 
-import urllib
+import urllib.request
+import urllib.parse
+import urllib.error
 
-from common import USER_AGENT
+from .common import USER_AGENT
 
 
-class HeadphonesURLopener(urllib.FancyURLopener):
+class HeadphonesURLopener(urllib.request.FancyURLopener):
     version = USER_AGENT
 
 
@@ -44,7 +46,7 @@ class AuthURLOpener(HeadphonesURLopener):
         self.numTries = 0
 
         # call the base class
-        urllib.FancyURLopener.__init__(self)
+        urllib.request.FancyURLopener.__init__(self)
 
     def prompt_user_passwd(self, host, realm):
         """

headphones/common.py

@@ -24,6 +24,7 @@ import operator
 import os
 import re
 from headphones import version
+from functools import reduce
 
 
 # Identify Our Application
@@ -74,9 +75,9 @@ class Quality:
     @staticmethod
     def _getStatusStrings(status):
         toReturn = {}
-        for x in Quality.qualityStrings.keys():
+        for x in list(Quality.qualityStrings.keys()):
             toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + \
-                                                           Quality.qualityStrings[x] + ")"
+                Quality.qualityStrings[x] + ")"
         return toReturn
 
     @staticmethod
@@ -93,7 +94,7 @@ class Quality:
     def splitQuality(quality):
         anyQualities = []
         bestQualities = []
-        for curQual in Quality.qualityStrings.keys():
+        for curQual in list(Quality.qualityStrings.keys()):
             if curQual & quality:
                 anyQualities.append(curQual)
             if curQual << 16 & quality:
@@ -151,7 +152,7 @@ class Quality:
     @staticmethod
     def splitCompositeStatus(status):
         """Returns a tuple containing (status, quality)"""
-        for x in sorted(Quality.qualityStrings.keys(), reverse=True):
+        for x in sorted(list(Quality.qualityStrings.keys()), reverse=True):
             if status > x * 100:
                 return (status - x * 100, x)
 
@@ -169,10 +170,10 @@ class Quality:
     SNATCHED_PROPER = None
 
 
-Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in Quality.qualityStrings.keys()]
-Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in Quality.qualityStrings.keys()]
+Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in list(Quality.qualityStrings.keys())]
+Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in list(Quality.qualityStrings.keys())]
 Quality.SNATCHED_PROPER = [Quality.compositeStatus(SNATCHED_PROPER, x) for x in
-                           Quality.qualityStrings.keys()]
+                           list(Quality.qualityStrings.keys())]
 
 MP3 = Quality.combineQualities([Quality.B192, Quality.B256, Quality.B320, Quality.VBR], [])
 LOSSLESS = Quality.combineQualities([Quality.FLAC], [])

headphones/config.py

@@ -2,15 +2,16 @@ import itertools
 
 import os
 import re
+import ast
+from configparser import ConfigParser
 import headphones.logger
-from configobj import ConfigObj
 
 
 def bool_int(value):
     """
     Casts a config value into a 0 or 1
     """
-    if isinstance(value, basestring):
+    if isinstance(value, str):
         if value.lower() in ('', '0', 'false', 'f', 'no', 'n', 'off'):
             value = 0
     return int(bool(value))
@@ -30,15 +31,23 @@ class path(str):
     def __repr__(self):
         return 'headphones.config.path(%s)' % self
 
+
 _CONFIG_DEFINITIONS = {
     'ADD_ALBUM_ART': (int, 'General', 0),
     'ADVANCEDENCODER': (str, 'General', ''),
     'ALBUM_ART_FORMAT': (str, 'General', 'folder'),
+    'ALBUM_ART_MIN_WIDTH': (str, 'General', ''),
+    'ALBUM_ART_MAX_WIDTH': (str, 'General', ''),
     # This is used in importer.py to determine how complete an album needs to
     # be - to be considered "downloaded". Percentage from 0-100
     'ALBUM_COMPLETION_PCT': (int, 'Advanced', 80),
     'API_ENABLED': (int, 'General', 0),
     'API_KEY': (str, 'General', ''),
+    'ORPHEUS': (int, 'Orpheus.network', 0),
+    'ORPHEUS_PASSWORD': (str, 'Orpheus.network', ''),
+    'ORPHEUS_RATIO': (str, 'Orpheus.network', ''),
+    'ORPHEUS_USERNAME': (str, 'Orpheus.network', ''),
+    'ORPHEUS_URL': (str, 'Orpheus.network', 'https://orpheus.network'),
     'AUTOWANT_ALL': (int, 'General', 0),
     'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1),
     'AUTOWANT_UPCOMING': (int, 'General', 1),
@@ -138,12 +147,14 @@ _CONFIG_DEFINITIONS = {
     'IGNORED_FILES': (list, 'Advanced', []),    # path
     'INCLUDE_EXTRAS': (int, 'General', 0),
     'INTERFACE': (str, 'General', 'default'),
+    'JOIN_APIKEY': (str, 'Join', ''),
+    'JOIN_DEVICEID': (str, 'Join', ''),
+    'JOIN_ENABLED': (int, 'Join', 0),
+    'JOIN_ONSNATCH': (int, 'Join', 0),
     'JOURNAL_MODE': (str, 'Advanced', 'wal'),
-    'KAT': (int, 'Kat', 0),
-    'KAT_PROXY_URL': (str, 'Kat', ''),
-    'KAT_RATIO': (str, 'Kat', ''),
     'KEEP_NFO': (int, 'General', 0),
     'KEEP_TORRENT_FILES': (int, 'General', 0),
+    'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
@@ -155,8 +166,7 @@ _CONFIG_DEFINITIONS = {
     'LOSSLESS_BITRATE_TO': (int, 'General', 0),
     'LOSSLESS_DESTINATION_DIR': (path, 'General', ''),
     'MB_IGNORE_AGE': (int, 'General', 365),
-    'MININOVA': (int, 'Mininova', 0),
-    'MININOVA_RATIO': (str, 'Mininova', ''),
+    'MB_IGNORE_AGE_MISSING': (int, 'General', 0),
     'MIRROR': (str, 'General', 'musicbrainz.org'),
     'MOVE_FILES': (int, 'General', 0),
     'MPC_ENABLED': (bool_int, 'MPC', False),
@@ -226,6 +236,10 @@ _CONFIG_DEFINITIONS = {
     'PUSHOVER_KEYS': (str, 'Pushover', ''),
     'PUSHOVER_ONSNATCH': (int, 'Pushover', 0),
     'PUSHOVER_PRIORITY': (int, 'Pushover', 0),
+    'QBITTORRENT_HOST': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_LABEL': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
@@ -236,6 +250,7 @@ _CONFIG_DEFINITIONS = {
     'RUTRACKER_PASSWORD': (str, 'Rutracker', ''),
     'RUTRACKER_RATIO': (str, 'Rutracker', ''),
     'RUTRACKER_USER': (str, 'Rutracker', ''),
+    'RUTRACKER_COOKIE': (str, 'Rutracker', ''),
     'SAB_APIKEY': (str, 'SABnzbd', ''),
     'SAB_CATEGORY': (str, 'SABnzbd', ''),
     'SAB_HOST': (str, 'SABnzbd', ''),
@@ -243,13 +258,16 @@ _CONFIG_DEFINITIONS = {
     'SAB_USERNAME': (str, 'SABnzbd', ''),
     'SAMPLINGFREQUENCY': (int, 'General', 44100),
     'SEARCH_INTERVAL': (int, 'General', 1440),
+    'SLACK_ENABLED': (int, 'Slack', 0),
+    'SLACK_URL': (str, 'Slack', ''),
+    'SLACK_CHANNEL': (str, 'Slack', ''),
+    'SLACK_EMOJI': (str, 'Slack', ''),
+    'SLACK_ONSNATCH': (int, 'Slack', 0),
     'SOFT_CHROOT': (path, 'General', ''),
     'SONGKICK_APIKEY': (str, 'Songkick', 'nd1We7dFW2RqxPw8'),
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
-    'STRIKE': (int, 'Strike', 0),
-    'STRIKE_RATIO': (str, 'Strike', ''),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
@@ -266,6 +284,7 @@ _CONFIG_DEFINITIONS = {
     'TORZNAB_APIKEY': (str, 'Torznab', ''),
     'TORZNAB_ENABLED': (int, 'Torznab', 1),
     'TORZNAB_HOST': (str, 'Torznab', ''),
+    'TORZNAB_RATIO': (str, 'Torznab', ''),
     'TRANSMISSION_HOST': (str, 'Transmission', ''),
     'TRANSMISSION_PASSWORD': (str, 'Transmission', ''),
     'TRANSMISSION_USERNAME': (str, 'Transmission', ''),
@@ -274,9 +293,6 @@ _CONFIG_DEFINITIONS = {
     'TWITTER_PASSWORD': (str, 'Twitter', ''),
     'TWITTER_PREFIX': (str, 'Twitter', 'Headphones'),
     'TWITTER_USERNAME': (str, 'Twitter', ''),
-    'TQUATTRECENTONZE': (int, 'tquattrecentonze', 0),
-    'TQUATTRECENTONZE_PASSWORD': (str, 'tquattrecentonze', ''),
-    'TQUATTRECENTONZE_USER': (str, 'tquattrecentonze', ''),
     'UPDATE_DB_INTERVAL': (int, 'General', 24),
     'USENET_RETENTION': (int, 'General', '1500'),
     'UTORRENT_HOST': (str, 'uTorrent', ''),
@@ -289,16 +305,11 @@ _CONFIG_DEFINITIONS = {
     'WAFFLES_PASSKEY': (str, 'Waffles', ''),
     'WAFFLES_RATIO': (str, 'Waffles', ''),
     'WAFFLES_UID': (str, 'Waffles', ''),
-    'WHATCD': (int, 'What.cd', 0),
-    'WHATCD_PASSWORD': (str, 'What.cd', ''),
-    'WHATCD_RATIO': (str, 'What.cd', ''),
-    'WHATCD_USERNAME': (str, 'What.cd', ''),
-    'WHATCD_URL': (str, 'What.cd', 'https://what.cd'),
-    'PTH': (int, 'PassTheHeadphones.me', 0),
-    'PTH_PASSWORD': (str, 'PassTheHeadphones.me', ''),
-    'PTH_RATIO': (str, 'PassTheHeadphones.me', ''),
-    'PTH_USERNAME': (str, 'PassTheHeadphones.me', ''),
-    'PTH_URL': (str, 'PassTheHeadphones.me', 'https://passtheheadphones.me'),
+    'REDACTED': (int, 'Redacted', 0),
+    'REDACTED_USERNAME': (str, 'Redacted', ''),
+    'REDACTED_PASSWORD': (str, 'Redacted', ''),
+    'REDACTED_RATIO': (str, 'Redacted', ''),
+    'REDACTED_USE_FLTOKEN': (int, 'Redacted', 0),
     'XBMC_ENABLED': (int, 'XBMC', 0),
     'XBMC_HOST': (str, 'XBMC', ''),
     'XBMC_NOTIFY': (int, 'XBMC', 0),
@@ -317,8 +328,9 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigObj(self._config_file, encoding='utf-8')
-        for key in _CONFIG_DEFINITIONS.keys():
+        self._config = ConfigParser()
+        self._config.read(self._config_file)
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
         self.ENCODER_MULTICORE_COUNT = max(0, self.ENCODER_MULTICORE_COUNT)
         self._upgrade()
@@ -335,7 +347,7 @@ class Config(object):
 
     def check_section(self, section):
         """ Check if INI section exists, if not create it """
-        if section not in self._config:
+        if not self._config.has_section(section):
             self._config[section] = {}
             return True
         else:
@@ -345,28 +357,38 @@ class Config(object):
         """ Cast any value in the config to the right type or use the default """
         key, definition_type, section, ini_key, default = self._define(key)
         self.check_section(section)
+
+        # ConfigParser values are strings, so need to convert to actual list
+        if definition_type == list:
+            definition_type = ast.literal_eval
+
         try:
             my_val = definition_type(self._config[section][ini_key])
+            # ConfigParser interprets empty strings in the config
+            # literally, so we need to sanitize it. It's not really
+            # a config upgrade, since a user can at any time put
+            # some_key = ''
+            if my_val == '""' or my_val == "''":
+                my_val = ''
         except Exception:
-            my_val = definition_type(default)
-            self._config[section][ini_key] = my_val
+            my_val = default
+            self._config[section][ini_key] = str(my_val)
         return my_val
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigObj(encoding="UTF-8")
-        new_config.filename = self._config_file
+        new_config = ConfigParser()
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data
-        for key, subkeys in self._config.items():
+        for key, subkeys in list(self._config.items()):
             if key not in new_config:
                 new_config[key] = {}
-            for subkey, value in subkeys.items():
+            for subkey, value in list(subkeys.items()):
                 new_config[key][subkey] = value
 
         # next make sure that everything we expect to have defined is so
-        for key in _CONFIG_DEFINITIONS.keys():
+        for key in list(_CONFIG_DEFINITIONS.keys()):
             key, definition_type, section, ini_key, default = self._define(key)
             self.check_setting(key)
             if section not in new_config:
@@ -377,15 +399,16 @@ class Config(object):
         headphones.logger.info("Writing configuration to file")
 
         try:
-            new_config.write()
+            with open(self._config_file, 'w') as configfile:
+                new_config.write(configfile)
         except IOError as e:
             headphones.logger.error("Error writing configuration file: %s", e)
 
     def get_extra_newznabs(self):
         """ Return the extra newznab tuples """
         extra_newznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
-                             for i in range(3)])
+            zip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
+                  for i in range(3)])
         )
         return extra_newznabs
 
@@ -403,8 +426,8 @@ class Config(object):
     def get_extra_torznabs(self):
         """ Return the extra torznab tuples """
         extra_torznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
-                             for i in range(3)])
+            zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
+                  for i in range(4)])
         )
         return extra_torznabs
 
@@ -439,20 +462,21 @@ class Config(object):
             return value
         else:
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
             return self._config[section][ini_key]
 
     def process_kwargs(self, kwargs):
         """
         Given a big bunch of key value pairs, apply them to the ini.
         """
-        for name, value in kwargs.items():
+        for name, value in list(kwargs.items()):
             key, definition_type, section, ini_key, default = self._define(name)
-            self._config[section][ini_key] = definition_type(value)
+            self._config[section][ini_key] = str(value)
 
     def _upgrade(self):
         """
-        Bring old configs up to date
+        Bring old configs up to date. Although this is kind of a dumb
+        way to do it because it doesn't handle multi-step upgrades
         """
         if self.CONFIG_VERSION == '2':
             # Update the config to use direct path to the encoder rather than the encoder folder
@@ -475,4 +499,17 @@ class Config(object):
         if self.CONFIG_VERSION == '5':
             if self.OPEN_MAGNET_LINKS:
                 self.MAGNET_LINKS = 2
-            self.CONFIG_VERSION = '5'
+
+            # Add Seed Ratio to Torznabs
+            if self.EXTRA_TORZNABS:
+                extra_torznabs = list(
+                    zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+                          for i in range(3)])
+                )
+                new_torznabs = []
+                for torznab in extra_torznabs:
+                    new_torznabs.extend([torznab[0], torznab[1], '', torznab[2]])
+                if new_torznabs:
+                    self.EXTRA_TORZNABS = new_torznabs
+
+            self.CONFIG_VERSION = '6'

headphones/config_test.py

@@ -2,8 +2,8 @@ import mock
 from mock import MagicMock
 import headphones.config
 import re
-import unittestcompat
-from unittestcompat import TestCase, TestArgs
+from . import unittestcompat
+from .unittestcompat import TestCase, TestArgs
 
 
 class ConfigApiTest(TestCase):
@@ -101,7 +101,7 @@ class ConfigApiTest(TestCase):
         # call methods
         c = headphones.config.Config(path)
         # assertions:
-        with self.assertRaisesRegexp(KeyError, exc_regex):
+        with self.assertRaisesRegex(KeyError, exc_regex):
             c.check_setting(setting_name)
         pass
 
@@ -355,86 +355,89 @@ class ConfigApiTest(TestCase):
     #   TORZNABS
     # TODO : here is copypaste from of NEZNABS tests. Make tests better, plz refactor them
     #
-    @TestArgs(
-        ('', []),
-        ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
-        (['ABC', 'DEF'], []),
-        ([1], []),
-        ([1, 2], []),
-        ([1, 2, 3], [(1, 2, 3)]),
-
-        ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
-        ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-    )
-    def test_get_extra_torznabs(self, conf_value, expected):
-        """ Config: get_extra_torznabs """
-        path = '/tmp/notexist'
-
-        # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
-        # set up mocks:
-        # 'EXTRA_TORZNABS': (list, '', ''),
-        self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
-
-        # call methods
-        c = headphones.config.Config(path)
-        res = c.get_extra_torznabs()
-
-        # assertions:
-        self.assertEqual(res, expected)
-
-    def test_clear_extra_torznabs(self):
-        """ Config: clear_extra_torznabs """
-        path = '/tmp/notexist'
-
-        random_value = -1292721
-        self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
-        self.config_mock["Torznab"] = {"do_not_touch": random_value}
-
-        # call methods
-        c = headphones.config.Config(path)
-        res = c.clear_extra_torznabs()
-
-        # assertions:
-        self.assertIsNone(res)
-        self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
-        self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
-
-    @TestArgs(
-        ([], [''], ['']),
-        ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
-
-        ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
-    )
-    def test_add_extra_torznab(self, initial, added, expected):
-        """ Config: add_extra_torznab """
-        path = '/tmp/notexist'
-
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
-
-        # call methods
-        c = headphones.config.Config(path)
-        c.add_extra_torznab(added)
-        act = self.config_mock["Torznab"]["extra_torznabs"]
-
-        # assertions:
-        self.assertEqual(act, expected)
-
-    @TestArgs(
-        (None),
-        ([]),
-        ([1, 2, 3]),
-        ([True]),
-    )
-    def test_add_extra_torznab_raise_on_none(self, initial):
-        """ Config: add_extra_torznab should raise on None adding"""
-        path = '/tmp/notexist'
-
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
-
-        # call methods
-        c = headphones.config.Config(path)
-        with self.assertRaises(TypeError):
-            c.add_extra_torznab(None)
-        pass
+    # TODO: Fix tests for following:
+    #   CONFIG_VERSION == '5' each entry = 'host, api, enabled'
+    #   CONFIG_VERSION > '5' each entry = 'host, api, seed ratio, enabled'
+    # @TestArgs(
+    #     ('', []),
+    #     ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
+    #     (['ABC', 'DEF'], []),
+    #     ([1], []),
+    #     ([1, 2], []),
+    #     ([1, 2, 3], [(1, 2, 3)]),
+    #
+    #     ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
+    #     ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    # )
+    # def test_get_extra_torznabs(self, conf_value, expected):
+    #     """ Config: get_extra_torznabs """
+    #     path = '/tmp/notexist'
+    #
+    #     # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
+    #     # set up mocks:
+    #     # 'EXTRA_TORZNABS': (list, '', ''),
+    #     self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
+    #
+    #     # call methods
+    #     c = headphones.config.Config(path)
+    #     res = c.get_extra_torznabs()
+    #
+    #     # assertions:
+    #     self.assertEqual(res, expected)
+    #
+    # def test_clear_extra_torznabs(self):
+    #     """ Config: clear_extra_torznabs """
+    #     path = '/tmp/notexist'
+    #
+    #     random_value = -1292721
+    #     self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
+    #     self.config_mock["Torznab"] = {"do_not_touch": random_value}
+    #
+    #     # call methods
+    #     c = headphones.config.Config(path)
+    #     res = c.clear_extra_torznabs()
+    #
+    #     # assertions:
+    #     self.assertIsNone(res)
+    #     self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
+    #     self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
+    #
+    # @TestArgs(
+    #     ([], [''], ['']),
+    #     ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
+    #
+    #     ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
+    # )
+    # def test_add_extra_torznab(self, initial, added, expected):
+    #     """ Config: add_extra_torznab """
+    #     path = '/tmp/notexist'
+    #
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #
+    #     # call methods
+    #     c = headphones.config.Config(path)
+    #     c.add_extra_torznab(added)
+    #     act = self.config_mock["Torznab"]["extra_torznabs"]
+    #
+    #     # assertions:
+    #     self.assertEqual(act, expected)
+    #
+    # @TestArgs(
+    #     (None),
+    #     ([]),
+    #     ([1, 2, 3]),
+    #     ([True]),
+    # )
+    # def test_add_extra_torznab_raise_on_none(self, initial):
+    #     """ Config: add_extra_torznab should raise on None adding"""
+    #     path = '/tmp/notexist'
+    #
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #
+    #     # call methods
+    #     c = headphones.config.Config(path)
+    #     with self.assertRaises(TypeError):
+    #         c.add_extra_torznab(None)
+    #     pass

headphones/crier.py

@@ -21,7 +21,7 @@ def cry():
             main_thread = t
 
     # Loop over each thread's current frame, writing info about it
-    for tid, frame in sys._current_frames().iteritems():
+    for tid, frame in sys._current_frames().items():
         thread = tmap.get(tid, main_thread)
 
         lines = []

headphones/cuesplit.py

@@ -87,7 +87,7 @@ def check_splitter(command):
 
 def split_baby(split_file, split_cmd):
     '''Let's split baby'''
-    logger.info('Splitting %s...', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Splitting {split_file}...")
     logger.debug(subprocess.list2cmdline(split_cmd))
 
     # Prevent Windows from opening a terminal window
@@ -108,16 +108,16 @@ def split_baby(split_file, split_cmd):
 
     process = subprocess.Popen(split_cmd, startupinfo=startupinfo,
                                stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE, env=env)
+                               stderr=subprocess.PIPE, env=env, text=True)
     stdout, stderr = process.communicate()
 
     if process.returncode:
-        logger.error('Split failed for %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
-        out = stdout if stdout else stderr
-        logger.error('Error details: %s', out.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.error(f"Split failed for {split_file}")
+        out = stdout or stderr
+        logger.error(f"Error details: {out}")
         return False
     else:
-        logger.info('Split success %s', split_file.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.info(f"Split succeeded for {split_file}")
         return True
 
 
@@ -232,7 +232,7 @@ class Directory:
         for i in list_dir:
             if not check_match(i):
                 # music file
-                if os.path.splitext(i)[-1] in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+                if os.path.splitext(i)[-1] in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
                     track_nr = identify_track_number(i)
                     if track_nr:
                         self.content.append(WaveFile(self.path + os.sep + i, track_nr=track_nr))
@@ -378,7 +378,7 @@ class CueFile(File):
             except:
                 raise ValueError('Cant encode CUE Sheet.')
 
-        if self.content[0] == u'\ufeff':
+        if self.content[0] == '\ufeff':
             self.content = self.content[1:]
 
         header = header_parser()
@@ -581,7 +581,7 @@ def split(albumpath):
 
     # use xld profile to split cue
     if headphones.CONFIG.ENCODER == 'xld' and headphones.CONFIG.MUSIC_ENCODER and headphones.CONFIG.XLDPROFILE:
-        import getXldProfile
+        from . import getXldProfile
         xldprofile, xldformat, _ = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE)
         if not xldformat:
             raise ValueError(
@@ -601,7 +601,7 @@ def split(albumpath):
         raise ValueError('Command not found, ensure shntool or xld installed')
 
     # Determine if file can be split
-    if wave.name_ext not in WAVE_FILE_TYPE_BY_EXTENSION.keys():
+    if wave.name_ext not in list(WAVE_FILE_TYPE_BY_EXTENSION.keys()):
         raise ValueError('Cannot split, audio file has unsupported extension')
 
     # Split with xld

headphones/db.py

@@ -17,7 +17,8 @@
 # Stolen from Sick-Beard's db.py  #
 ###################################
 
-from __future__ import with_statement
+
+import time
 
 import sqlite3
 
@@ -45,36 +46,97 @@ class DBConnection:
         self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
         # don't wait for the disk to finish writing
         self.connection.execute("PRAGMA synchronous = OFF")
-        # journal disabled since we never do rollbacks
+        # default set to Write-Ahead Logging WAL
         self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE)
         # 64mb of cache memory,probably need to make it user configurable
         self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024))
         self.connection.row_factory = sqlite3.Row
 
-    def action(self, query, args=None):
+    def action(self, query, args=None, upsert_insert_qry=None):
 
         if query is None:
             return
 
         sqlResult = None
+        attempts = 0
+        dberror = None
 
-        try:
-            with self.connection as c:
-                if args is None:
-                    sqlResult = c.execute(query)
+        while attempts < 10:
+            try:
+                with self.connection as c:
+
+                    # log that previous attempt was locked and we're trying again
+                    if dberror:
+                        if args is None:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s', attempts + 1, query)
+                        else:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s. Args: %s', attempts + 1, query, args)
+
+                    # debugging
+                    # try:
+                    #     explain_query = 'EXPLAIN QUERY PLAN ' + query
+                    #     if not args:
+                    #         sql_results = c.execute(explain_query)
+                    #     else:
+                    #         sql_results = c.execute(explain_query, args)
+                    #     if not args:
+                    #         print(explain_query)
+                    #     else:
+                    #         print(explain_query + ' ' + str(args))
+                    #     explain_results = sql_results
+                    #     for row in explain_results:
+                    #         print row
+                    # except Exception as e:
+                    #     print(e)
+
+                    # Execute query
+
+                    # time0 = time.time()
+
+                    if args is None:
+                        sqlResult = c.execute(query)
+                        # logger.debug('SQL: ' + query)
+                    else:
+                        sqlResult = c.execute(query, args)
+                        # logger.debug('SQL: %s. Args: %s', query, args)
+
+                        # INSERT part of upsert query
+                        if upsert_insert_qry:
+                            sqlResult = c.execute(upsert_insert_qry, args)
+                            # logger.debug('SQL: %s. Args: %s', upsert_insert_qry, args)
+
+                    # debugging: loose test to log queries taking longer than 5 seconds
+                    # seconds = time.time() - time0
+                    # if seconds > 5:
+                    #     if args is None:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s", seconds, query)
+                    #     else:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s with args %s", seconds, query, args)
+
+                    break
+
+            except sqlite3.OperationalError as e:
+                if "unable to open database file" in e.message or "database is locked" in e.message:
+                    dberror = e
+                    if args is None:
+                        logger.debug('Database error: %s. Query: %s', e, query)
+                    else:
+                        logger.debug('Database error: %s. Query: %s. Args: %s', e, query, args)
+                    attempts += 1
+                    time.sleep(1)
                 else:
-                    sqlResult = c.execute(query, args)
-
-        except sqlite3.OperationalError, e:
-            if "unable to open database file" in e.message or "database is locked" in e.message:
-                logger.warn('Database Error: %s', e)
-            else:
-                logger.error('Database error: %s', e)
+                    logger.error('Database error: %s', e)
+                    raise
+            except sqlite3.DatabaseError as e:
+                logger.error('Fatal Error executing %s :: %s', query, e)
                 raise
 
-        except sqlite3.DatabaseError, e:
-            logger.error('Fatal Error executing %s :: %s', query, e)
-            raise
+        # log if no results returned due to lock
+        if not sqlResult and attempts:
+            if args is None:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s', dberror, query)
+            else:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s. Args: %s', dberror, query, args)
 
         return sqlResult
 
@@ -88,24 +150,19 @@ class DBConnection:
         return sqlResults
 
     def upsert(self, tableName, valueDict, keyDict):
-
+        """
+        Transactions an Update or Insert to a table based on key.
+        If the table is not updated then the 'WHERE changes' will be 0 and the table inserted
+        """
         def genParams(myDict):
-            return [x + " = ?" for x in myDict.keys()]
+            return [x + " = ?" for x in list(myDict.keys())]
 
-        changesBefore = self.connection.total_changes
+        update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
 
-        update_query = "UPDATE " + tableName + " SET " + ", ".join(
-            genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
+        insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(list(valueDict.keys()) + list(keyDict.keys())) + ")" + " SELECT " + ", ".join(
+            ["?"] * len(list(valueDict.keys()) + list(keyDict.keys()))) + " WHERE changes()=0")
 
-        self.action(update_query, valueDict.values() + keyDict.values())
-
-        if self.connection.total_changes == changesBefore:
-            insert_query = (
-                "INSERT INTO " + tableName + " (" + ", ".join(
-                    valueDict.keys() + keyDict.keys()) + ")" +
-                " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
-            )
-            try:
-                self.action(insert_query, valueDict.values() + keyDict.values())
-            except sqlite3.IntegrityError:
-                logger.info('Queries failed: %s and %s', update_query, insert_query)
+        try:
+            self.action(update_query, list(valueDict.values()) + list(keyDict.values()), upsert_insert_qry=insert_query)
+        except sqlite3.IntegrityError:
+            logger.info('Queries failed: %s and %s', update_query, insert_query)

headphones/deluge.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 #  This file is part of Headphones.
 #
 #  Headphones is free software: you can redistribute it and/or modify
@@ -32,7 +34,6 @@
 # You should have received a copy of the GNU General Public License
 # along with SickRage. If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import unicode_literals
 
 from headphones import logger
 
@@ -49,6 +50,7 @@ delugeweb_auth = {}
 delugeweb_url = ''
 deluge_verify_cert = False
 scrub_logs = True
+headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 
 
 def _scrubber(text):
@@ -80,13 +82,13 @@ def addTorrent(link, data=None, name=None):
 
         result = {}
         retid = False
-        url_what = ['https://what.cd/', 'http://what.cd/']
+        url_orpheus = ['https://orpheus.network/', 'http://orpheus.network/']
         url_waffles = ['https://waffles.ch/', 'http://waffles.ch/']
 
         if link.lower().startswith('magnet:'):
             logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link))
             result = {'type': 'magnet',
-                        'url': link}
+                      'url': link}
             retid = _add_torrent_magnet(result)
 
         elif link.lower().startswith('http://') or link.lower().startswith('https://'):
@@ -94,7 +96,7 @@ def addTorrent(link, data=None, name=None):
             if link.lower().startswith(tuple(url_waffles)):
                 if 'rss=' not in link:
                     link = link + '&rss=1'
-            if link.lower().startswith(tuple(url_what)):
+            if link.lower().startswith(tuple(url_orpheus)):
                 logger.debug('Deluge: Using different User-Agent for this site')
                 user_agent = 'Headphones'
                 # This method will make Deluge download the file
@@ -104,11 +106,11 @@ def addTorrent(link, data=None, name=None):
                 # return addTorrent(local_torrent_path)
             else:
                 user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
-            headers = {'User-Agent': user_agent}
+            get_headers = {'User-Agent': user_agent}
             torrentfile = ''
             logger.debug('Deluge: Trying to download (GET)')
             try:
-                r = requests.get(link, headers=headers)
+                r = requests.get(link, headers=get_headers)
                 if r.status_code == 200:
                     logger.debug('Deluge: 200 OK')
                     # .text will ruin the encoding for some torrents
@@ -135,10 +137,13 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
-            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            try:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except:
+                logger.debug('Deluge: Sending Deluge torrent with problematic name and some content')
             result = {'type': 'torrent',
-                        'name': name,
-                        'content': torrentfile}
+                      'name': name,
+                      'content': torrentfile}
             retid = _add_torrent_file(result)
 
         # elif link.endswith('.torrent') or data:
@@ -164,10 +169,13 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
-            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            try:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except UnicodeDecodeError:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name.decode('utf-8'), str(torrentfile)[:40]))
             result = {'type': 'torrent',
-                        'name': name,
-                        'content': torrentfile}
+                      'name': name,
+                      'content': torrentfile}
             retid = _add_torrent_file(result)
 
         else:
@@ -199,7 +207,7 @@ def getTorrentFolder(result):
                                 ],
                                 "id": 21})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         result['total_done'] = json.loads(response.text)['result']['total_done']
 
         tries = 0
@@ -207,7 +215,7 @@ def getTorrentFolder(result):
             tries += 1
             time.sleep(5)
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
             result['total_done'] = json.loads(response.text)['result']['total_done']
 
         post_data = json.dumps({"method": "web.get_torrent_status",
@@ -226,7 +234,7 @@ def getTorrentFolder(result):
                                 "id": 23})
 
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
 
         result['save_path'] = json.loads(response.text)['result']['save_path']
         result['name'] = json.loads(response.text)['result']['name']
@@ -242,18 +250,45 @@ def removeTorrent(torrentid, remove_data=False):
         _get_auth()
 
     try:
-        result = False
-        post_data = json.dumps({"method": "core.remove_torrent",
+        logger.debug('Deluge: Checking if torrent %s finished seeding' % str(torrentid))
+        post_data = json.dumps({"method": "web.get_torrent_status",
                                 "params": [
                                     torrentid,
-                                    remove_data
-                                    ],
-                                "id": 25})
-        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
-        result = json.loads(response.text)['result']
+                                    [
+                                        "name",
+                                        "ratio",
+                                        "state"
+                                    ]
+                                ],
+                                "id": 26})
 
-        return result
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+                                 verify=deluge_verify_cert, headers=headers)
+
+        try:
+            state = json.loads(response.text)['result']['state']
+        except KeyError as e:
+            logger.debug('Deluge: "state" KeyError when trying to remove torrent %s' % str(torrentid))
+            return False
+
+        not_finished = ["queued", "seeding", "downloading", "checking", "error"]
+        result = False
+        if state.lower() in not_finished:
+            logger.debug('Deluge: Torrent %s is either queued or seeding, not removing yet' % str(torrentid))
+            return False
+        else:
+            logger.debug('Deluge: Removing torrent %s' % str(torrentid))
+            post_data = json.dumps({"method": "core.remove_torrent",
+                                    "params": [
+                                        torrentid,
+                                        remove_data
+                                    ],
+                                    "id": 25})
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+                                     verify=deluge_verify_cert, headers=headers)
+            result = json.loads(response.text)['result']
+
+            return result
     except Exception as e:
         logger.error('Deluge: Removing torrent failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -293,12 +328,12 @@ def _get_auth():
                             "id": 1})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
     except requests.ConnectionError:
         try:
             logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case')
             response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
             # If the previous line didn't fail, change delugeweb_url for the rest of this session
             logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!')
             delugeweb_url = delugeweb_url.replace('http:', 'https:')
@@ -323,7 +358,7 @@ def _get_auth():
                             "id": 10})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
     except Exception as e:
         logger.error('Deluge: Authentication failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -340,7 +375,7 @@ def _get_auth():
                                 "id": 11})
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -348,7 +383,8 @@ def _get_auth():
             return None
 
         delugeweb_hosts = json.loads(response.text)['result']
-        if len(delugeweb_hosts) == 0:
+        # Check if delugeweb_hosts is None before checking its length
+        if not delugeweb_hosts or len(delugeweb_hosts) == 0:
             logger.error('Deluge: WebUI does not contain daemons')
             return None
 
@@ -358,7 +394,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -371,7 +407,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -396,7 +432,7 @@ def _add_torrent_magnet(result):
                                 "params": [result['url'], {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -416,7 +452,7 @@ def _add_torrent_url(result):
                                 "params": [result['url'], {}],
                                 "id": 32})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         result['location'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -434,30 +470,14 @@ def _add_torrent_file(result):
     try:
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
-                                "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}],
+                                "params": [result['name'] + '.torrent',
+                                           b64encode(result['content']).decode(), {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
-    except UnicodeDecodeError:
-        try:
-            # content is torrent file contents that needs to be encoded to base64
-            # this time let's try leaving the encoding as is
-            post_data = json.dumps({"method": "core.add_torrent_file",
-                                    "params": [result['name'] + '.torrent', b64encode(result['content']), {}],
-                                    "id": 22})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
-            result['hash'] = json.loads(response.text)['result']
-            logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
-            return json.loads(response.text)['result']
-        except Exception as e:
-            logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e))
-            formatted_lines = traceback.format_exc().splitlines()
-            logger.error('; '.join(formatted_lines))
-            return False
     except Exception as e:
         logger.error('Deluge: Adding torrent file failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -481,7 +501,7 @@ def setTorrentLabel(result):
                                 "params": [],
                                 "id": 3})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                                 verify=deluge_verify_cert, headers=headers)
         labels = json.loads(response.text)['result']
 
         if labels is not None:
@@ -492,7 +512,7 @@ def setTorrentLabel(result):
                                             "params": [label],
                                             "id": 4})
                     response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                        verify=deluge_verify_cert)
+                                             verify=deluge_verify_cert, headers=headers)
                     logger.debug('Deluge: %s label added to Deluge' % label)
                 except Exception as e:
                     logger.error('Deluge: Setting label failed: %s' % str(e))
@@ -504,7 +524,7 @@ def setTorrentLabel(result):
                                     "params": [result['hash'], label],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
             logger.debug('Deluge: %s label added to torrent' % label)
         else:
             logger.debug('Deluge: Label plugin not detected')
@@ -528,12 +548,12 @@ def setSeedRatio(result):
                                     "params": [result['hash'], True],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
             post_data = json.dumps({"method": "core.set_torrent_stop_ratio",
                                     "params": [result['hash'], float(ratio)],
                                     "id": 6})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -556,7 +576,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], True],
                                     "id": 7})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
 
             if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
                 move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -570,7 +590,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], move_to],
                                     "id": 8})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -593,7 +613,7 @@ def setTorrentPause(result):
                                     "params": [[result['hash']]],
                                     "id": 9})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                                     verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 

headphones/getXldProfile.py

@@ -15,7 +15,7 @@ def getXldProfile(xldProfile):
     # Get xld preferences plist
     try:
         preferences = biplist.readPlist(expanded)
-    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException), e:
+    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException) as e:
         logger.error("Error reading xld preferences plist: %s", e)
         return (xldProfileNotFound, None, None)
 

headphones/helpers.py

@@ -20,11 +20,19 @@ import datetime
 import shutil
 import time
 import sys
+import tempfile
+import glob
+
+from beets import logging as beetslogging
+import six
+from contextlib import contextmanager
 
 import fnmatch
+import functools
 import re
 import os
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from unidecode import unidecode
 import headphones
 
 
@@ -35,6 +43,19 @@ RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)?", re.I)
 RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
 
 
+def cmp(x, y):
+    """
+    Replacement for built-in function cmp that was removed in Python 3
+
+    Compare the two objects x and y and return an integer according to
+    the outcome. The return value is negative if x < y, zero if x == y
+    and strictly positive if x > y.
+
+    https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function
+    """
+    return (x > y) - (x < y)
+
+
 def multikeysort(items, columns):
     comparers = [
         ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1))
@@ -48,7 +69,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, cmp=comparer)
+    return sorted(items, key=functools.cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -191,14 +212,20 @@ def piratesize(size):
     return size
 
 
-def replace_all(text, dic, normalize=False):
+def pattern_substitute(pattern, dic, normalize=False):
+    """
+    Execute path rendering/substitution based on replacement dictionary
+    e.g. pattern = $Artist/$Album
+         dic = {Artist: 'My artist', Album: 'My album'}
+         returns My artist/My album
+    """
     from headphones import pathrender
-    if not text:
+    if not pattern:
         return ''
 
     if normalize:
         new_dic = {}
-        for i, j in dic.iteritems():
+        for i, j in dic.items():
             if j is not None:
                 try:
                     if sys.platform == 'darwin':
@@ -207,23 +234,31 @@ def replace_all(text, dic, normalize=False):
                         j = unicodedata.normalize('NFC', j)
                 except TypeError:
                     j = unicodedata.normalize('NFC',
-                        j.decode(headphones.SYS_ENCODING, 'replace'))
+                                              j.decode(headphones.SYS_ENCODING, 'replace'))
             new_dic[i] = j
         dic = new_dic
-    return pathrender.render(text, dic)[0]
+    return pathrender.render(pattern, dic)[0]
+
+
+def replace_all(text, dic):
+    if not text:
+        return ''
+
+    for i, j in dic.items():
+        text = text.replace(i, j)
+    return text
 
 
 def replace_illegal_chars(string, type="file"):
     if type == "file":
         string = re.sub('[\?"*:|<>/]', '_', string)
     if type == "folder":
-        string = re.sub('[:\?<>"|]', '_', string)
-
+        string = re.sub('[:\?<>"|*]', '_', string)
     return string
 
 
-_CN_RE1 = re.compile(ur'[^\w]+', re.UNICODE)
-_CN_RE2 = re.compile(ur'[\s_]+', re.UNICODE)
+_CN_RE1 = re.compile(r'[^\w]+', re.UNICODE)
+_CN_RE2 = re.compile(r'[\s_]+', re.UNICODE)
 
 
 _XLATE_GRAPHICAL_AND_DIACRITICAL = {
@@ -233,26 +268,33 @@ _XLATE_GRAPHICAL_AND_DIACRITICAL = {
     # ©ª«®²³¹»¼½¾ÆÐØÞßæðøþĐđĦħıIJijĸĿŀŁłŒœŦŧDŽDždžLJLjljNJNjnjǤǥDZDzdzȤȥ. This
     # includes also some graphical symbols which can be easily replaced and
     # usually are written by people who don't have appropriate keyboard layout.
-    u'©': '(C)', u'ª': 'a.', u'«': '<<', u'®': '(R)', u'²': '2', u'³': '3',
-    u'¹': '1', u'»': '>>', u'¼': ' 1/4 ', u'½': ' 1/2 ', u'¾': ' 3/4 ',
-    u'Æ': 'AE', u'Ð': 'D', u'Ø': 'O', u'Þ': 'Th', u'ß': 'ss', u'æ': 'ae',
-    u'ð': 'd', u'ø': 'o', u'þ': 'th', u'Đ': 'D', u'đ': 'd', u'Ħ': 'H',
-    u'ħ': 'h', u'ı': 'i', u'IJ': 'IJ', u'ij': 'ij', u'ĸ': 'q', u'Ŀ': 'L',
-    u'ŀ': 'l', u'Ł': 'L', u'ł': 'l', u'Œ': 'OE', u'œ': 'oe', u'Ŧ': 'T',
-    u'ŧ': 't', u'DŽ': 'DZ', u'Dž': 'Dz', u'LJ': 'LJ', u'Lj': 'Lj',
-    u'lj': 'lj', u'NJ': 'NJ', u'Nj': 'Nj', u'nj': 'nj',
-    u'Ǥ': 'G', u'ǥ': 'g', u'DZ': 'DZ', u'Dz': 'Dz', u'dz': 'dz',
-    u'Ȥ': 'Z', u'ȥ': 'z', u'№': 'No.',
-    u'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
-                       # this is 'masculine ordering indicator', not degree
+    '©': '(C)', 'ª': 'a.', '«': '<<', '®': '(R)', '²': '2', '³': '3',
+    '¹': '1', '»': '>>', '¼': ' 1/4 ', '½': ' 1/2 ', '¾': ' 3/4 ',
+    'Æ': 'AE', 'Ð': 'D', 'Ø': 'O', 'Þ': 'Th', 'ß': 'ss', 'æ': 'ae',
+    'ð': 'd', 'ø': 'o', 'þ': 'th', 'Đ': 'D', 'đ': 'd', 'Ħ': 'H',
+    'ħ': 'h', 'ı': 'i', 'IJ': 'IJ', 'ij': 'ij', 'ĸ': 'q', 'Ŀ': 'L',
+    'ŀ': 'l', 'Ł': 'L', 'ł': 'l', 'Œ': 'OE', 'œ': 'oe', 'Ŧ': 'T',
+    'ŧ': 't', 'DŽ': 'DZ', 'Dž': 'Dz', 'LJ': 'LJ', 'Lj': 'Lj',
+    'lj': 'lj', 'NJ': 'NJ', 'Nj': 'Nj', 'nj': 'nj',
+    'Ǥ': 'G', 'ǥ': 'g', 'DZ': 'DZ', 'Dz': 'Dz', 'dz': 'dz',
+    'Ȥ': 'Z', 'ȥ': 'z', '№': 'No.',
+    'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
+    # this is 'masculine ordering indicator', not degree
 }
 
 _XLATE_SPECIAL = {
     # Translation table.
     # Cover additional special characters processing normalization.
-    u"'": '',         # replace apostrophe with nothing
-    u"’": '',         # replace musicbrainz style apostrophe with nothing
-    u'&': ' and ',     # expand & to ' and '
+    "'": '',         # replace apostrophe with nothing
+    "’": '',         # replace musicbrainz style apostrophe with nothing
+    '&': ' and ',     # expand & to ' and '
+}
+
+_XLATE_MUSICBRAINZ = {
+    # Translation table for Musicbrainz.
+    "…": '...',     # HORIZONTAL ELLIPSIS (U+2026)
+    "’": "'",       # APOSTROPHE (U+0027)
+    "‐": "-",       # EN DASH (U+2013)
 }
 
 
@@ -287,10 +329,10 @@ def _transliterate(u, xlate):
     Perform transliteration using the specified dictionary
     """
     u = unicodedata.normalize('NFD', u)
-    u = u''.join([u'' if _is_unicode_combining(x) else x for x in u])
+    u = ''.join(['' if _is_unicode_combining(x) else x for x in u])
     u = _translate(u, xlate)
     # at this point output is either unicode, or plain ascii
-    return unicode(u)
+    return str(u)
 
 
 def clean_name(s):
@@ -300,10 +342,10 @@ def clean_name(s):
     :param s: string to clean up, possibly unicode one.
     :return: cleaned-up version of input string.
     """
-    if not isinstance(s, unicode):
+    if not isinstance(s, str):
         # ignore extended chars if someone was dumb enough to pass non-ascii
         # narrow string here, use only unicode for meaningful texts
-        u = unicode(s, 'ascii', 'replace')
+        u = str(s, 'ascii', 'replace')
     else:
         u = s
     # 1. don't bother doing normalization NFKC, rather transliterate
@@ -314,15 +356,33 @@ def clean_name(s):
     # 3. translate spacials
     u = _translate(u, _XLATE_SPECIAL)
     # 4. replace any non-alphanumeric character sequences by spaces
-    u = _CN_RE1.sub(u' ', u)
+    u = _CN_RE1.sub(' ', u)
     # 5. coalesce interleaved space/underscore sequences
-    u = _CN_RE2.sub(u' ', u)
+    u = _CN_RE2.sub(' ', u)
     # 6. trim
     u = u.strip()
     # 7. lowercase
+    u = u.lower()
     return u
 
 
+def clean_musicbrainz_name(s, return_as_string=True):
+    # type: (basestring)->unicode
+    """Substitute special Musicbrainz characters.
+    :param s: string to clean up, probably unicode.
+    :return: cleaned-up version of input string.
+    """
+    if not isinstance(s, str):
+        u = str(s, 'ascii', 'replace')
+    else:
+        u = s
+    u = _translate(u, _XLATE_MUSICBRAINZ)
+    if return_as_string:
+        return u.encode('utf-8')
+    else:
+        return u
+
+
 def cleanTitle(title):
     title = re.sub('[\.\-\/\_]', ' ', title).lower()
 
@@ -407,8 +467,7 @@ def expand_subfolders(f):
 
     if difference > 0:
         logger.info(
-            "Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.",
-            len(media_folders), difference)
+            f"Found {len(media_folders)} media folders, but depth difference between lowest and deepest media folder is {difference} (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.")
 
         # While already failed, advice the user what he could try. We assume the
         # directory may contain separate CD's and maybe some extra's. The
@@ -420,8 +479,7 @@ def expand_subfolders(f):
             set([os.path.join(*media_folder) for media_folder in extra_media_folders]))
 
         logger.info(
-            "Please look at the following folder(s), since they cause the depth difference: %s",
-            extra_media_folders)
+            f"Please look at the following folder(s), since they cause the depth difference: {extra_media_folders}")
         return
 
     # Convert back to paths and remove duplicates, which may be there after
@@ -435,7 +493,7 @@ def expand_subfolders(f):
         logger.debug("Did not expand subfolder, as it resulted in one folder.")
         return
 
-    logger.debug("Expanded subfolders in folder: %s", media_folders)
+    logger.debug(f"Expanded subfolders in folder: {media_folders}")
     return media_folders
 
 
@@ -453,7 +511,7 @@ def path_match_patterns(path, patterns):
     return False
 
 
-def path_filter_patterns(paths, patterns, root=None):
+def path_filter_patterns(paths, patterns, root=''):
     """
     Scan for ignored paths based on glob patterns. Note that the whole path
     will be matched, therefore paths should only contain the relative paths.
@@ -467,8 +525,7 @@ def path_filter_patterns(paths, patterns, root=None):
 
     for path in paths[:]:
         if path_match_patterns(path, patterns):
-            logger.debug("Path ignored by pattern: %s",
-                         os.path.join(root or "", path))
+            logger.debug(f"Path ignored by pattern: {os.path.join(root, path)}")
 
             ignored += 1
             paths.remove(path)
@@ -550,7 +607,7 @@ def extract_metadata(f):
     count_ratio = 0.75
 
     if count < (count_ratio * len(results)):
-        logger.info("Counted %d media files, but only %d have tags, ignoring.", count, len(results))
+        logger.info(f"Counted {count} media files, but only {len(results)} have tags, ignoring.")
         return (None, None, None)
 
     # Count distinct values
@@ -568,8 +625,7 @@ def extract_metadata(f):
                 old_album = new_albums[index]
                 new_albums[index] = RE_CD_ALBUM.sub("", album).strip()
 
-                logger.debug("Stripped albumd number identifier: %s -> %s", old_album,
-                             new_albums[index])
+                logger.debug(f"Stripped album number identifier: {old_album} -> {new_albums[index]}")
 
         # Remove duplicates
         new_albums = list(set(new_albums))
@@ -587,7 +643,7 @@ def extract_metadata(f):
     if len(artists) > 1 and len(albums) == 1:
         split_artists = [RE_FEATURING.split(x) for x in artists]
         featurings = [len(split_artist) - 1 for split_artist in split_artists]
-        logger.info("Album seem to feature %d different artists", sum(featurings))
+        logger.info("Album seem to feature {sum(featurings)} different artists")
 
         if sum(featurings) > 0:
             # Find the artist of which the least splits have been generated.
@@ -599,9 +655,11 @@ def extract_metadata(f):
             return (artist, albums[0], years[0])
 
     # Not sure what to do here.
-    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists),
-                len(albums), len(years))
-    logger.debug("Artists: %s, Albums: %s, Years: %s", artists, albums, years)
+    logger.info(
+        f"Found {len(artists)} artists, {len(albums)} albums and "
+        f"{len(years)} years in metadata, so ignoring"
+    )
+    logger.debug("Artists: {artists}, Albums: {albums}, Years: {years}")
 
     return (None, None, None)
 
@@ -621,26 +679,71 @@ def get_downloaded_track_list(albumpath):
     return downloaded_track_list
 
 
-def preserve_torrent_directory(albumpath):
+def preserve_torrent_directory(albumpath, forced=False, single=False):
     """
-    Copy torrent directory to headphones-modified to keep files for seeding.
+    Copy torrent directory to temp headphones_ directory to keep files for seeding.
     """
     from headphones import logger
-    new_folder = os.path.join(albumpath,
-                              'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
-    logger.info(
-        "Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
+
+    # Create temp dir
+    if headphones.CONFIG.KEEP_TORRENT_FILES_DIR:
+        tempdir = headphones.CONFIG.KEEP_TORRENT_FILES_DIR
+    else:
+        tempdir = tempfile.gettempdir()
+
+    logger.info(f"Preparing to copy to a temporary directory for post processing: {albumpath}")
+
     try:
-        shutil.copytree(albumpath, new_folder)
-        return new_folder
+        file_name = os.path.basename(os.path.normpath(albumpath))
+        if not single:
+            prefix = "headphones_" + file_name + "_@hp@_"
+        else:
+            prefix = "headphones_" + os.path.splitext(file_name)[0] + "_@hp@_"
+        new_folder = tempfile.mkdtemp(prefix=prefix, dir=tempdir)
     except Exception as e:
-        logger.warn("Cannot copy/move files to temp folder: " +
-                    new_folder.decode(headphones.SYS_ENCODING, 'replace') +
-                    ". Not continuing. Error: " + str(e))
+        logger.error(f"Cannot create temp directory: {tempdir}. Error: {e}")
+        return None
+
+    # Attempt to stop multiple temp dirs being created for the same albumpath
+    if not forced:
+        try:
+            workdir = os.path.join(tempdir, prefix)
+            workdir = re.sub(r'\[', '[[]', workdir)
+            workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
+            if len(glob.glob(workdir + '*/')) >= 3:
+                logger.error(
+                    "Looks like a temp directory has previously been created "
+                    "for this albumpath, not continuing "
+                )
+                shutil.rmtree(new_folder)
+                return None
+        except Exception as e:
+            logger.warn(
+                "Cannot determine if already copied/processed, will copy anyway. "
+                f"Warning: {e}"
+            )
+
+    # Copy to temp dir
+    try:
+        subdir = os.path.join(new_folder, "headphones")
+        logger.info(f"Copying files to {subdir}")
+        if not single:
+            shutil.copytree(albumpath, subdir)
+        else:
+            os.makedirs(subdir)
+            shutil.copy(albumpath, subdir)
+        # Update the album path with the new location
+        return subdir
+    except Exception as e:
+        logger.warn(
+            f"Cannot copy/move files to temp directory: {new_folder}. "
+            f"Not continuing. Error: {e}"
+        )
+        shutil.rmtree(new_folder)
         return None
 
 
-def cue_split(albumpath):
+def cue_split(albumpath, keep_original_folder=False):
     """
      Attempts to check and split audio files by a cue for the given directory.
      """
@@ -662,6 +765,15 @@ def cue_split(albumpath):
     # Split cue
     if cue_count and cue_count >= count and cue_dirs:
 
+        # Copy to temp directory
+        if keep_original_folder:
+            temppath = preserve_torrent_directory(albumpath)
+            if temppath:
+                cue_dirs = [cue_dir.replace(albumpath, temppath) for cue_dir in cue_dirs]
+                albumpath = temppath
+            else:
+                return None
+
         from headphones import logger, cuesplit
         logger.info("Attempting to split audio files by cue")
 
@@ -671,13 +783,13 @@ def cue_split(albumpath):
                 cuesplit.split(cue_dir)
             except Exception as e:
                 os.chdir(cwd)
-                logger.warn("Cue not split: " + str(e))
-                return False
+                logger.warn(f"Cue not split. Error: {e}")
+                return None
 
         os.chdir(cwd)
-        return True
+        return albumpath
 
-    return False
+    return None
 
 
 def extract_logline(s):
@@ -709,7 +821,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid default format", s)
+        logger.info(f"Couldn't parse {s} into a valid default format")
 
     # newzbin default format
     pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s\((?P<year>\d+?\))', re.VERBOSE)
@@ -720,7 +832,7 @@ def extract_song_data(s):
         year = match.group("year")
         return (name, album, year)
     else:
-        logger.info("Couldn't parse %s into a valid Newbin format", s)
+        logger.info(f"Couldn't parse {s} into a valid Newbin format")
         return (name, album, year)
 
 
@@ -729,9 +841,11 @@ def smartMove(src, dest, delete=True):
 
     source_dir = os.path.dirname(src)
     filename = os.path.basename(src)
+    source_path = os.path.join(source_dir, filename)
+    dest_path = os.path.join(dest, filename)
 
-    if os.path.isfile(os.path.join(dest, filename)):
-        logger.info('Destination file exists: %s', os.path.join(dest, filename))
+    if os.path.isfile(dest_path):
+        logger.info(f"Destination file exists: {dest_path}")
         title = os.path.splitext(filename)[0]
         ext = os.path.splitext(filename)[1]
         i = 1
@@ -740,24 +854,37 @@ def smartMove(src, dest, delete=True):
             if os.path.isfile(os.path.join(dest, newfile)):
                 i += 1
             else:
-                logger.info('Renaming to %s', newfile)
+                logger.info(f"Renaming to {newfile}")
                 try:
                     os.rename(src, os.path.join(source_dir, newfile))
                     filename = newfile
                 except Exception as e:
-                    logger.warn('Error renaming %s: %s',
-                                src.decode(headphones.SYS_ENCODING, 'replace'), e)
+                    logger.warn(f"Error renaming {src}: {e}")
                 break
 
-    try:
-        if delete:
-            shutil.move(os.path.join(source_dir, filename), os.path.join(dest, filename))
-        else:
-            shutil.copy(os.path.join(source_dir, filename), os.path.join(dest, filename))
+    if delete:
+        try:
+            logger.info('Moving "%s" to "%s"', source_path, dest_path)
+            shutil.move(source_path, dest_path)
+        except Exception as e:
+            exists = os.path.exists(dest_path)
+            if exists and os.path.getsize(source_path) == os.path.getsize(dest_path):
+                logger.warn(
+                    f"Successfully moved {filename}, but something went wrong: {e}"
+                )
+                os.unlink(source_path)
+            else:
+                # remove faultly copied file
+                if exists:
+                    os.unlink(dest_path)
+                raise
+    else:
+        try:
+            logger.info(f"Copying {source_path} to {dest_path}")
+            shutil.copy(source_path, dest_path)
             return True
-    except Exception as e:
-        logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
-                    e)
+        except Exception as e:
+            logger.warn(f"Error copying {filename}: {e}")
 
 
 def walk_directory(basedir, followlinks=True):
@@ -766,7 +893,7 @@ def walk_directory(basedir, followlinks=True):
     with care. In case a folder is already processed, don't traverse it again.
     """
 
-    import logger
+    from . import logger
 
     # Add the base path, because symlinks poiting to the basedir should not be
     # traversed again.
@@ -780,8 +907,10 @@ def walk_directory(basedir, followlinks=True):
                 real_path = os.path.abspath(os.readlink(path))
 
                 if real_path in traversed:
-                    logger.debug("Skipping '%s' since it is a symlink to "
-                                 "'%s', which is already visited.", path, real_path)
+                    logger.debug(
+                        f"Skipping {path} since it is a symlink to "
+                        f"{real_path}, which is already visited."
+                    )
                 else:
                     traversed.append(real_path)
 
@@ -823,22 +952,15 @@ def sab_sanitize_foldername(name):
     FL_ILLEGAL = CH_ILLEGAL + ':\x92"'
     FL_LEGAL = CH_LEGAL + "-''"
 
-    uFL_ILLEGAL = FL_ILLEGAL.decode('latin-1')
-    uFL_LEGAL = FL_LEGAL.decode('latin-1')
-
     if not name:
-        return name
-    if isinstance(name, unicode):
-        illegal = uFL_ILLEGAL
-        legal = uFL_LEGAL
-    else:
-        illegal = FL_ILLEGAL
-        legal = FL_LEGAL
+        return
+
+    name = unidecode(name)
 
     lst = []
     for ch in name.strip():
-        if ch in illegal:
-            ch = legal[illegal.find(ch)]
+        if ch in FL_ILLEGAL:
+            ch = FL_LEGAL[FL_ILLEGAL.find(ch)]
             lst.append(ch)
         else:
             lst.append(ch)
@@ -894,7 +1016,28 @@ def create_https_certificates(ssl_cert, ssl_key):
         with open(ssl_cert, "w") as fp:
             fp.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
     except IOError as e:
-        logger.error("Error creating SSL key and certificate: %s", e)
+        logger.error(f"Error creating SSL key and certificate: e")
         return False
 
     return True
+
+
+class BeetsLogCapture(beetslogging.Handler):
+
+    def __init__(self):
+        beetslogging.Handler.__init__(self)
+        self.messages = []
+
+    def emit(self, record):
+        self.messages.append(six.text_type(record.msg))
+
+
+@contextmanager
+def capture_beets_log(logger='beets'):
+    capture = BeetsLogCapture()
+    log = beetslogging.getLogger(logger)
+    log.addHandler(capture)
+    try:
+        yield capture.messages
+    finally:
+        log.removeHandler(capture)

headphones/helpers_test.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from unittestcompat import TestCase
+from .unittestcompat import TestCase
 from headphones.helpers import clean_name
 
 
@@ -8,28 +8,28 @@ class HelpersTest(TestCase):
     def test_clean_name(self):
         """helpers: check correctness of clean_name() function"""
         cases = {
-            u' Weiße & rose ': 'Weisse and rose',
-            u'Multiple / spaces': 'Multiple spaces',
-            u'Kevin\'s m²': 'Kevins m2',
-            u'Symphonęy Nº9': 'Symphoney No.9',
-            u'ÆæßðÞIJij': u'AeaessdThIJıj',
-            u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
-                                                    'apoplexy remix',
-            u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
-                                                    'zbojow',
-            u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
-            u'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
-            u'L\'Arc~en~Ciel': 'larc en ciel',
-            u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
-                u'Orquesta de la Luz オルケスタ デ ラ ルス'
+            ' Weiße & rose ': 'Weisse and rose',
+            'Multiple / spaces': 'Multiple spaces',
+            'Kevin\'s m²': 'Kevins m2',
+            'Symphonęy Nº9': 'Symphoney No.9',
+            'ÆæßðÞIJij': 'AeaessdThIJıj',
+            'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
+            'apoplexy remix',
+            'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
+            'zbojow',
+            'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
+            'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
+            'L\'Arc~en~Ciel': 'larc en ciel',
+            'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)':
+                'Orquesta de la Luz オルケスタ デ ラ ルス'
 
         }
-        for first, second in cases.iteritems():
+        for first, second in cases.items():
             nf = clean_name(first).lower()
             ns = clean_name(second).lower()
             self.assertEqual(
-                nf, ns, u"check cleaning of case (%s,"
-                        u"%s)" % (nf, ns)
+                nf, ns, "check cleaning of case (%s,"
+                        "%s)" % (nf, ns)
             )
 
     def test_clean_name_nonunicode(self):

headphones/importer.py

@@ -16,7 +16,7 @@
 import time
 
 from headphones import logger, helpers, db, mb, lastfm, metacritic
-from beets.mediafile import MediaFile
+from mediafile import MediaFile
 import headphones
 
 blacklisted_special_artist_names = ['[anonymous]', '[data]', '[no artist]',
@@ -39,7 +39,7 @@ def is_exists(artistid):
 
     if any(artistid in x for x in artistlist):
         logger.info(artistlist[0][
-                        1] + u" is already in the database. Updating 'have tracks', but not artist information")
+            1] + " is already in the database. Updating 'have tracks', but not artist information")
         return True
     else:
         return False
@@ -53,7 +53,7 @@ def artistlist_to_mbids(artistlist, forced=False):
 
         # If adding artists through Manage New Artists, they're coming through as non-unicode (utf-8?)
         # and screwing everything up
-        if not isinstance(artist, unicode):
+        if not isinstance(artist, str):
             try:
                 artist = artist.decode('utf-8', 'replace')
             except Exception:
@@ -89,11 +89,13 @@ def artistlist_to_mbids(artistlist, forced=False):
             addArtisttoDB(artistid)
 
         # Just update the tracks if it does
-        else:
-            havetracks = len(
-                myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
-                myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
+
+        # not sure this is correct and we're updating during scanning in librarysync
+        # else:
+        #     havetracks = len(
+        #         myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+        #         myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+        #     myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
 
         # Delete it from the New Artists if the request came from there
         if forced:
@@ -105,7 +107,7 @@ def artistlist_to_mbids(artistlist, forced=False):
     try:
         lastfm.getSimilar()
     except Exception as e:
-        logger.warn('Failed to update arist information from Last.fm: %s' % e)
+        logger.warn('Failed to update artist information from Last.fm: %s' % e)
 
 
 def addArtistIDListToDB(artistidlist):
@@ -182,7 +184,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     else:
         sortname = artist['artist_name']
 
-    logger.info(u"Now adding/updating: " + artist['artist_name'])
+    logger.info("Now adding/updating: " + artist['artist_name'])
     controlValueDict = {"ArtistID": artistid}
     newValueDict = {"ArtistName": artist['artist_name'],
                     "ArtistSortName": sortname,
@@ -261,9 +263,13 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 new_releases = mb.get_new_releases(rgid, includeExtras)
 
             else:
-                if check_release_date is None or check_release_date == u"None":
-                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
-                    new_releases = mb.get_new_releases(rgid, includeExtras, True)
+                if check_release_date is None or check_release_date == "None":
+                    if not headphones.CONFIG.MB_IGNORE_AGE_MISSING:
+                        logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                        new_releases = mb.get_new_releases(rgid, includeExtras, True)
+                    else:
+                        logger.info("[%s] Skipping update of: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                        new_releases = 0
                 else:
                     if len(check_release_date) == 10:
                         release_date = check_release_date
@@ -305,7 +311,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
             # This will be used later to build a hybrid release
             fullreleaselist = []
             # Search for releases within a release group
-            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
+            find_hybrid_releases = myDB.select("SELECT * from allalbums WHERE AlbumID=?",
                                                [rg['id']])
 
             # Build the dictionary for the fullreleaselist
@@ -511,8 +517,11 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     marked_as_downloaded = True
 
             logger.info(
-                u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
-            cache.getThumb(AlbumID=rg['id'])
+                "[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+            try:
+                cache.getThumb(AlbumID=rg['id'])
+            except Exception as e:
+                logger.error("Error getting album art: %s", e)
 
             # Start a search for the album if it's new, hasn't been marked as
             # downloaded and autowant_all is selected. This search is deferred,
@@ -521,17 +530,23 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 album_searches.append(rg['id'])
         else:
             if skip_log == 0:
-                logger.info(u"[%s] No new releases, so no changes made to %s" % (
+                logger.info("[%s] No new releases, so no changes made to %s" % (
                             artist['artist_name'], rg['title']))
 
     time.sleep(3)
     finalize_update(artistid, artist['artist_name'], errors)
 
-    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
-    cache.getThumb(ArtistID=artistid)
+    logger.info("Seeing if we need album art for: %s" % artist['artist_name'])
+    try:
+        cache.getThumb(ArtistID=artistid)
+    except Exception as e:
+        logger.error("Error getting album art: %s", e)
 
-    logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
-    metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
+    logger.info("Fetching Metacritic reviews for: %s" % artist['artist_name'])
+    try:
+        metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
+    except Exception as e:
+        logger.error("Error getting Metacritic reviews: %s", e)
 
     if errors:
         logger.info(
@@ -539,7 +554,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 artist['artist_name'], artist['artist_name']))
     else:
         myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
-        logger.info(u"Updating complete for: %s" % artist['artist_name'])
+        logger.info("Updating complete for: %s" % artist['artist_name'])
 
     # Start searching for newly added albums
     if album_searches:
@@ -648,7 +663,7 @@ def addReleaseById(rid, rgid=None):
             sortname = release_dict['artist_name']
 
         logger.info(
-            u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
+            "Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
         controlValueDict = {"ArtistID": release_dict['artist_id']}
         newValueDict = {"ArtistName": release_dict['artist_name'],
                         "ArtistSortName": sortname,
@@ -681,7 +696,7 @@ def addReleaseById(rid, rgid=None):
 
     if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
         # but if it is this will fail
-        logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
+        logger.info("Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
         controlValueDict = {"AlbumID": rgid}
         if status != 'Loading':
             status = 'Wanted'
@@ -757,7 +772,7 @@ def addReleaseById(rid, rgid=None):
 
         # Start a search for the album
         if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED:
-            import searcher
+            from . import searcher
             searcher.searchforalbum(rgid, False)
 
     elif not rg_exists and not release_dict:

headphones/lastfm.py

@@ -22,7 +22,7 @@ from headphones import db, logger, request
 
 TIMEOUT = 60.0  # seconds
 REQUEST_LIMIT = 1.0 / 5  # seconds
-ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
+ENTRY_POINT = "https://ws.audioscrobbler.com/2.0/"
 API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
 # Required for API request limit
@@ -55,7 +55,7 @@ def request_lastfm(method, **kwargs):
         return
 
     if "error" in data:
-        logger.error("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.FM returned an error: %s", data["message"])
         return
 
     return data
@@ -91,7 +91,7 @@ def getSimilar():
     for artist, mbid in artistlist:
         count[artist, mbid] += 1
 
-    items = count.items()
+    items = list(count.items())
     top_list = sorted(items, key=lambda x: x[1], reverse=True)[:25]
 
     random.shuffle(top_list)
@@ -149,7 +149,10 @@ def getTagTopArtists(tag, limit=50):
         logger.debug("Fetched %d artists from Last.FM", len(artists))
 
         for artist in artists:
-            artist_mbid = artist["mbid"]
+            try:
+                artist_mbid = artist["mbid"]
+            except KeyError:
+                continue
 
             if not any(artist_mbid in x for x in results):
                 artistlist.append(artist_mbid)

headphones/librarysync.py

@@ -17,7 +17,7 @@ import os
 import math
 
 import headphones
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from headphones import db, logger, helpers, importer, lastfm
 
 
@@ -30,56 +30,64 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     if not dir:
         if not headphones.CONFIG.MUSIC_DIR:
+            logger.info(
+                "No music directory configured. Add it under "
+                "Manage -> Scan Music Library"
+            )
             return
         else:
             dir = headphones.CONFIG.MUSIC_DIR
 
-    # If we're appending a dir, it's coming from the post processor which is
-    # already bytestring
-    if not append or artistScan:
-        dir = dir.encode(headphones.SYS_ENCODING)
-
     if not os.path.isdir(dir):
-        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING,
-                                                                           'replace'))
+        logger.warn(f"Cannot find music directory: {dir}")
         return
 
     myDB = db.DBConnection()
     new_artists = []
 
-    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(f"Scanning music directory: {dir}")
 
     if not append:
-        # Clean up bad filepaths
-        tracks = myDB.select(
-            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
 
-        for track in tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
-            if not os.path.isfile(encoded_track_string):
+        # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
+        if ArtistID:
+            dbtracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
+                'IS NOT NULL',
+                [ArtistID, ArtistID])
+        else:
+            dbtracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL')
+
+        for track in dbtracks:
+            track_location = track['Location']
+            if not os.path.isfile(track_location):
                 myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, track_location])
                 myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, track_location])
 
-        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
+        if ArtistName:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName])
+        else:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have')
 
         for track in del_have_tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
-            if not os.path.isfile(encoded_track_string):
+            if not os.path.isfile(track['Location']):
                 if track['ArtistName']:
                     # Make sure deleted files get accounted for when updating artist track counts
                     new_artists.append(track['ArtistName'])
-                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
+                myDB.action('DELETE FROM have WHERE Location=?', [Track['Location']])
                 logger.info(
-                    'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
-                        headphones.SYS_ENCODING, 'replace'))
+                    f"{Track['Location']} removed from Headphones, as it "
+                    f"is no longer on disk"
+                )
 
     bitrates = []
-    song_list = []
+    track_list = []
     latest_subdirectory = []
 
-    new_song_count = 0
+    new_track_count = 0
     file_count = 0
 
     for r, d, f in helpers.walk_directory(dir):
@@ -94,32 +102,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 subdirectory = r.replace(dir, '')
                 latest_subdirectory.append(subdirectory)
 
-                if file_count == 0 and r.replace(dir, '') != '':
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-                elif latest_subdirectory[file_count] != latest_subdirectory[
-                            file_count - 1] and file_count != 0:
-                    logger.info("[%s] Now scanning subdirectory %s" % (
-                        dir.decode(headphones.SYS_ENCODING, 'replace'),
-                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-
-                song = os.path.join(r, files)
-
-                # We need the unicode path to use for logging, inserting into database
-                unicode_song_path = song.decode(headphones.SYS_ENCODING, 'replace')
+                track_path = os.path.join(r, files)
 
                 # Try to read the metadata
                 try:
-                    f = MediaFile(song)
+                    f = MediaFile(track_path)
                 except (FileTypeError, UnreadableFileError):
-                    logger.warning(
-                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
-                        unicode_song_path)
+                    logger.warning(f"Cannot read `{track_path}`. It may be corrupted or not a media file.")
                     continue
                 except IOError:
-                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?",
-                                   unicode_song_path)
+                    logger.warning(f"Cannnot read `{track_path}`. Does the file exists?")
                     continue
 
                 # Grab the bitrates for the auto detect bit rate option
@@ -134,15 +126,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 else:
                     f_artist = None
 
-                # Add the song to our song list -
-                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
+                # Add the track to our track list -
+                # TODO: skip adding tracks without the minimum requisite information (just a matter of putting together the right if statements)
 
                 if f_artist and f.album and f.title:
                     CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title)
                 else:
                     CleanName = None
 
-                controlValueDict = {'Location': unicode_song_path}
+                controlValueDict = {'Location': track_path}
 
                 newValueDict = {'TrackID': f.mb_trackid,
                                 # 'ReleaseID' : f.mb_albumid,
@@ -158,24 +150,24 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                                 'CleanName': CleanName
                                 }
 
-                # song_list.append(song_dict)
-                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?",
-                                               [unicode_song_path]).fetchone()
-                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
-                if not check_exist_song:
+                # track_list.append(track_dict)
+                check_exist_track = myDB.action("SELECT * FROM have WHERE Location=?",
+                                                [track_path]).fetchone()
+                # Only attempt to match tracks that are new, haven't yet been matched, or metadata has changed.
+                if not check_exist_track:
                     # This is a new track
                     if f_artist:
                         new_artists.append(f_artist)
                     myDB.upsert("have", newValueDict, controlValueDict)
-                    new_song_count += 1
+                    new_track_count += 1
                 else:
-                    if check_exist_song['ArtistName'] != f_artist or check_exist_song[
-                            'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                    if check_exist_track['ArtistName'] != f_artist or check_exist_track[
+                            'AlbumTitle'] != f.album or check_exist_track['TrackTitle'] != f.title:
                         # Important track metadata has been modified, need to run matcher again
-                        if f_artist and f_artist != check_exist_song['ArtistName']:
+                        if f_artist and f_artist != check_exist_track['ArtistName']:
                             new_artists.append(f_artist)
-                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
-                                        check_exist_song['Matched'] != "Ignored":
+                        elif f_artist and f_artist == check_exist_track['ArtistName'] and \
+                                check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
                         else:
                             continue
@@ -184,49 +176,56 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                         myDB.upsert("have", newValueDict, controlValueDict)
                         myDB.action(
                             'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
+                            [None, None, None, track_path])
                         myDB.action(
                             'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, unicode_song_path])
-                        new_song_count += 1
+                            [None, None, None, track_path])
+                        new_track_count += 1
                     else:
                         # This track information hasn't changed
-                        if f_artist and check_exist_song['Matched'] != "Ignored":
+                        if f_artist and check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
 
                 file_count += 1
 
     # Now we start track matching
-    logger.info("%s new/modified songs found and added to the database" % new_song_count)
-    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-                            [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
-    total_number_of_songs = \
-        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
-    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(
-        headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+    logger.info(f"{new_track_count} new/modified tracks found and added to the database")
+    dbtracks = myDB.action(
+        "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+        [f"{dir}%"]
+    )
+    dbtracks_count = myDB.action(
+        "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+        [f"{dir}%"]
+    ).fetchone()[0]
+    logger.info(f"Found {dbtracks_count} new/modified tracks in `{dir}`")
+    logger.info("Matching tracks to the appropriate releases....")
 
-    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
-    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
+    # Sort the track_list by most vague (e.g. no trackid or releaseid)
+    # to most specific (both trackid & releaseid)
+    # When we insert into the database, the tracks with the most
+    # specific information will overwrite the more general matches
 
-    # song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
-    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
+    sorted_dbtracks = helpers.multikeysort(dbtracks, ['ArtistName', 'AlbumTitle'])
 
-    # We'll use this to give a % completion, just because the track matching might take a while
-    song_count = 0
-    latest_artist = []
+    # We'll use this to give a % completion, just because the
+    # track matching might take a while
+    tracks_completed = 0
+    latest_artist = None
     last_completion_percentage = 0
+    prev_artist_name = None
+    artistid = None
 
-    for song in song_list:
+    for track in sorted_dbtracks:
 
-        latest_artist.append(song['ArtistName'])
-        if song_count == 0:
-            logger.info("Now matching songs by %s" % song['ArtistName'])
-        elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0:
-            logger.info("Now matching songs by %s" % song['ArtistName'])
+        if latest_artist != track['ArtistName']:
+            logger.info(f"Now matching tracks by {track['ArtistName']}")
+            latest_artist = track['ArtistName']
 
-        song_count += 1
-        completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10
+        tracks_completed += 1
+        completion_percentage = math.floor(
+            float(tracks_completed) / dbtracks_count * 1000
+        ) / 10
 
         if completion_percentage >= (last_completion_percentage + 10):
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
@@ -237,139 +236,184 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
         # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+        albumid = None
 
-            track = myDB.action(
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-            have_updated = False
-            if track:
-                controlValueDict = {'ArtistName': track['ArtistName'],
-                                    'AlbumTitle': track['AlbumTitle'],
-                                    'TrackTitle': track['TrackTitle']}
-                newValueDict = {'Location': song['Location'],
-                                'BitRate': song['BitRate'],
-                                'Format': song['Format']}
-                myDB.upsert("tracks", newValueDict, controlValueDict)
+        if track['ArtistName'] and track['CleanName']:
+            artist_name = track['ArtistName']
+            clean_name = track['CleanName']
 
-                controlValueDict2 = {'Location': song['Location']}
-                newValueDict2 = {'Matched': track['AlbumID']}
-                myDB.upsert("have", newValueDict2, controlValueDict2)
-                have_updated = True
-            else:
-                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
-                                    [song['CleanName']]).fetchone()
-                if track:
-                    controlValueDict = {'CleanName': track['CleanName']}
-                    newValueDict = {'Location': song['Location'],
-                                    'BitRate': song['BitRate'],
-                                    'Format': song['Format']}
-                    myDB.upsert("tracks", newValueDict, controlValueDict)
+            # Only update if artist is in the db
+            if artist_name != prev_artist_name:
+                prev_artist_name = artist_name
+                artistid = None
 
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': track['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
-                else:
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': "Failed"}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
+                artist_lookup = "\"" + artist_name.replace("\"", "\"\"") + "\""
 
-            alltrack = myDB.action(
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-            if alltrack:
-                controlValueDict = {'ArtistName': alltrack['ArtistName'],
-                                    'AlbumTitle': alltrack['AlbumTitle'],
-                                    'TrackTitle': alltrack['TrackTitle']}
-                newValueDict = {'Location': song['Location'],
-                                'BitRate': song['BitRate'],
-                                'Format': song['Format']}
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                try:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
+                except:
+                    dbartist = None
+                if not dbartist:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name])
+                    if not dbartist:
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name])
+                        if not dbartist:
+                            clean_artist = helpers.clean_name(artist_name)
+                            if clean_artist:
+                                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                                       [clean_artist, clean_artist + '{'])
+                                if not dbartist:
+                                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                                           [clean_artist, clean_artist + '{'])
 
-                controlValueDict2 = {'Location': song['Location']}
-                newValueDict2 = {'Matched': alltrack['AlbumID']}
-                myDB.upsert("have", newValueDict2, controlValueDict2)
-            else:
-                alltrack = myDB.action(
-                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
-                    [song['CleanName']]).fetchone()
+                if dbartist:
+                    artistid = dbartist[0][0]
+
+            if artistid:
+
+                # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index
+                # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like)
+                # Now selects/updates using CleanName index (may have to revert if not working)
+
+                # matching on CleanName should be enough, ensure it's the same artist just in case
+
+                # Update tracks
+                dbtrack = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
+                if dbtrack:
+                    albumid = dbtrack['AlbumID']
+                    myDB.action(
+                        'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
+
+                # Update alltracks
+                alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
                 if alltrack:
-                    controlValueDict = {'CleanName': alltrack['CleanName']}
-                    newValueDict = {'Location': song['Location'],
-                                    'BitRate': song['BitRate'],
-                                    'Format': song['Format']}
-                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': alltrack['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                else:
-                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
-                    if not have_updated:
-                        controlValueDict2 = {'Location': song['Location']}
-                        newValueDict2 = {'Matched': "Failed"}
-                        myDB.upsert("have", newValueDict2, controlValueDict2)
+                    albumid = alltrack['AlbumID']
+                    myDB.action(
+                        'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
+                        [track['Location'], track['BitRate'], track['Format'], clean_name, artistid])
 
+        # Update have
+        controlValueDict2 = {'Location': track['Location']}
+        if albumid:
+            newValueDict2 = {'Matched': albumid}
         else:
-            controlValueDict2 = {'Location': song['Location']}
             newValueDict2 = {'Matched': "Failed"}
-            myDB.upsert("have", newValueDict2, controlValueDict2)
+        myDB.upsert("have", newValueDict2, controlValueDict2)
 
-            # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+        # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [track['ArtistName'], track['AlbumTitle'], track['TrackNumber'], track['TrackTitle'], track['TrackLength'], track['BitRate'], track['Genre'], track['Date'], track['TrackID'], track['Location'], CleanName, track['Format']])
 
-    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
-                                                                            'replace'))
+    logger.info(f"Completed matching tracks from `{dir}`")
 
     if not append or artistScan:
         logger.info('Updating scanned artist track counts')
 
         # Clean up the new artist list
-        unique_artists = {}.fromkeys(new_artists).keys()
-        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
+        unique_artists = list({}.fromkeys(new_artists).keys())
 
-        # There was a bug where artists with special characters (-,') would show up in new artists.
-        artist_list = [
-            x for x in unique_artists
-            if helpers.clean_name(x).lower() not in [
-                helpers.clean_name(y[0]).lower()
-                for y in current_artists
-                ]
-            ]
-        artists_checked = [
-            x for x in unique_artists
-            if helpers.clean_name(x).lower() in [
-                helpers.clean_name(y[0]).lower()
-                for y in current_artists
-                ]
-            ]
+        # # Don't think we need to do this, check the db instead below
+        #
+        # # artist scan
+        # if ArtistName:
+        #     current_artists = [[ArtistName]]
+        # # directory scan
+        # else:
+        #     current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL')
+        #
+        # # There was a bug where artists with special characters (-,') would show up in new artists.
+        #
+        # # artist_list = scanned artists not in the db
+        # artist_list = [
+        #     x for x in unique_artists
+        #     if helpers.clean_name(x).lower() not in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
+        #
+        # # artists_checked = scanned artists that exist in the db
+        # artists_checked = [
+        #     x for x in unique_artists
+        #     if helpers.clean_name(x).lower() in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
 
-        # Update track counts
-        for artist in artists_checked:
-            # Have tracks are selected from tracks table and not all tracks because of duplicates
-            # We update the track count upon an album switch to compliment this
-            havetracks = (
-                len(myDB.select(
-                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
-                    [artist])) + len(myDB.select(
-                        'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
-                        [artist]))
-            )
-            # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
-            # (can fix by getting rid of second len statement)
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
+        new_artist_list = []
 
-        logger.info('Found %i new artists' % len(artist_list))
+        for artist in unique_artists:
 
-        if artist_list:
+            if not artist:
+                continue
+
+            logger.info('Processing artist: %s' % artist)
+
+            # check if artist is already in the db
+            artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\""
+
+            try:
+                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
+            except:
+                dbartist = None
+            if not dbartist:
+                clean_artist = helpers.clean_name(artist)
+                if clean_artist:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                           [clean_artist, clean_artist + '{'])
+                    if not dbartist:
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                               [clean_artist, clean_artist + '{'])
+
+            # new artist not in db, add to list
+            if not dbartist:
+                new_artist_list.append(artist)
+            else:
+
+                # artist in db, update have track counts
+                artistid = dbartist[0][0]
+
+                # Have tracks are selected from tracks table and not all tracks because of duplicates
+                # We update the track count upon an album switch to compliment this
+
+                # havetracks = (
+                #     len(myDB.select(
+                #         'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
+                #         [artist])) + len(myDB.select(
+                #             'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                #             [artist]))
+                # )
+
+                try:
+                    havetracks = (
+                        len(myDB.select(
+                            'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
+                            [artistid])) + len(myDB.select(
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+                    )
+                except Exception as e:
+                    logger.warn('Error updating counts for artist: %s: %s' % (artist, e))
+
+                # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
+                # (can fix by getting rid of second len statement)
+
+                if havetracks:
+                    myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
+
+                    # Update albums to downloaded
+                    update_album_status(ArtistID=artistid)
+
+        logger.info('Found %i new artists' % len(new_artist_list))
+
+        # Add scanned artists not in the db
+        if new_artist_list:
             if headphones.CONFIG.AUTO_ADD_ARTISTS:
-                logger.info('Importing %i new artists' % len(artist_list))
-                importer.artistlist_to_mbids(artist_list)
+                logger.info('Importing %i new artists' % len(new_artist_list))
+                importer.artistlist_to_mbids(new_artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
                 # myDB.action('DELETE from newartists')
-                for artist in artist_list:
+                for artist in new_artist_list:
                     myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
 
         if headphones.CONFIG.DETECT_BITRATE and bitrates:
@@ -379,50 +423,78 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
 
-        havetracks = len(
-            myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
-                        [ArtistID])) + len(myDB.select(
-                            'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
-                            [ArtistName]))
-        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
+        artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\""
+        try:
+            havetracks = len(
+                myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
+                            [ArtistID])) + len(myDB.select(
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+        except Exception as e:
+            logger.warn('Error updating counts for artist: %s: %s' % (ArtistName, e))
 
-    if not append:
-        update_album_status()
+        if havetracks:
+            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
+
+    # Moved above to call for each artist
+    # if not append:
+    #     update_album_status()
 
     if not append and not artistScan:
         lastfm.getSimilar()
 
-    logger.info('Library scan complete')
+    if ArtistName:
+        logger.info('Scanning complete for artist: %s', ArtistName)
+    else:
+        logger.info('Library scan complete')
 
 
 # ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
 
+# Think the above comment relates to calling from Manage Unmatched
 
-def update_album_status(AlbumID=None):
+# This used to select and update all albums and would clobber the db, changed to run by ArtistID.
+
+def update_album_status(AlbumID=None, ArtistID=None):
     myDB = db.DBConnection()
-    logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+    # logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+
     if AlbumID:
         album_status_updater = myDB.action(
-            'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+            'SELECT'
+            ' a.AlbumID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
+            'FROM'
+            ' albums AS a '
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
+            'WHERE'
+            ' a.AlbumID = ? AND a.Status != "Downloaded" '
+            'GROUP BY'
+            ' a.AlbumID '
+            'HAVING'
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
+            [AlbumID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
+        )
     else:
-        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
-    for album in album_status_updater:
-        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?',
-                                    [album['AlbumID']])
-        total_tracks = 0
-        have_tracks = 0
-        for track in track_counter:
-            total_tracks += 1
-            if track['Location']:
-                have_tracks += 1
-        if total_tracks != 0:
-            album_completion = float(float(have_tracks) / float(total_tracks)) * 100
-        else:
-            album_completion = 0
-            logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
+        album_status_updater = myDB.action(
+            'SELECT'
+            ' a.AlbumID, a.ArtistID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
+            'FROM'
+            ' albums AS a '
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
+            'WHERE'
+            ' a.ArtistID = ? AND a.Status != "Downloaded" '
+            'GROUP BY'
+            ' a.AlbumID '
+            'HAVING'
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
+            [ArtistID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
+        )
 
-        if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT:
-            new_album_status = "Downloaded"
+    new_album_status = "Downloaded"
+
+    albums = []
+    for album in album_status_updater:
+        albums.append([album['AlbumID'], album['ArtistName'], album['AlbumTitle']])
+    for album in albums:
 
         # I don't think we want to change Downloaded->Skipped.....
         # I think we can only automatically change Skipped->Downloaded when updating
@@ -433,10 +505,13 @@ def update_album_status(AlbumID=None):
         #        new_album_status = "Skipped"
         #    else:
         #        new_album_status = album['Status']
-        else:
-            new_album_status = album['Status']
+        #     else:
+        #         new_album_status = album['Status']
+        #
+        #     myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
+        #     if new_album_status != album['Status']:
+        #         logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
+        # logger.info('Album status update complete')
 
-        myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
-        if new_album_status != album['Status']:
-            logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
-    logger.info('Album status update complete')
+        myDB.action('UPDATE albums SET Status = ? WHERE AlbumID = ?', [new_album_status, album[0]])
+        logger.info('Album: %s - %s. Status updated to %s' % (album[1], album[2], new_album_status))

headphones/lock.py

@@ -4,7 +4,7 @@ Locking-related classes
 
 import time
 import threading
-import Queue
+import queue
 
 import headphones.logger
 
@@ -29,7 +29,7 @@ class TimedLock(object):
         self.lock = threading.Lock()
         self.last_used = 0
         self.minimum_delta = minimum_delta
-        self.queue = Queue.Queue()
+        self.queue = queue.Queue()
 
     def __enter__(self):
         """
@@ -47,7 +47,7 @@ class TimedLock(object):
                 seconds = self.queue.get(False)
                 headphones.logger.debug('Sleeping %s (queued)', seconds)
                 time.sleep(seconds)
-            except Queue.Empty:
+            except queue.Empty:
                 continue
             self.queue.task_done()
 

headphones/logger.py

@@ -153,7 +153,8 @@ def initLogger(console=False, log_dir=False, verbose=False):
         file_formatter = logging.Formatter(
             '%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
         file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE,
-                                                    backupCount=MAX_FILES)
+                                                    backupCount=MAX_FILES,
+                                                    encoding='utf8')
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(file_formatter)