v0.5.20

last.fm api

.travis.yml

@@ -11,7 +11,6 @@ cache:
     - lib
 
 python:
-  - "2.6"
   - "2.7"
 
 install:

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v0.5.20
+Released 15 October 2021
+
+Highlights:
+* Improved: Last.fm & metacrtic api fixes
+* Improved: Various database fixes
+* Added: qBitTorrent
+* Added: Telegram notifications
+* Improved: Many more since the last release. Check the list
+
+Many thanks to @AdeHub for keeping the ship afloat
+and to everyone else who contributed.
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.19...v0.5.20).
+
+## v0.5.19
+Released 27 March 2018
+
+Highlights:
+* Improved: Windows encoding fixes
+* Improved: Performance enhancements
+* Improved: Many more since the last release. Check the list
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.18...v0.5.19).
+
 ## v0.5.18
 Released 01 December 2016
 

Headphones.py

@@ -54,6 +54,9 @@ def main():
 
     try:
         locale.setlocale(locale.LC_ALL, "")
+        if headphones.SYS_PLATFORM == 'win32':
+            headphones.SYS_ENCODING = sys.getdefaultencoding().upper()
+        else:
             headphones.SYS_ENCODING = locale.getpreferredencoding()
     except (locale.Error, IOError):
         pass

data/interfaces/default/album.html

@@ -324,7 +324,7 @@
                 if (++x === 10) {
                     clearInterval(refreshInterval);
                 }
-                var sts = $("#artistname").text();
+                var sts = $("#artistname").text().trim();
                 if (wasLoading == true || sts == "Loading"){
                     location.reload();
                 $('#albumnamespinner').remove()

data/interfaces/default/config.html

@@ -133,6 +133,12 @@
                                     </label>
                                     <input type="text" name="mb_ignore_age" value="${config['mb_ignore_age']}" size="4">days
                                 </div>
+                                <div class="row checkbox">
+                                    <label title="Ignore MusicBrainz album updates missing a release date.">
+                                        Ignore Undated Releases
+                                    </label>
+                                    <input type="checkbox" id="mb_ignore_age_missing" name="mb_ignore_age_missing" value="1" ${config['mb_ignore_age_missing']}
+                                </div>                                
                             </fieldset>
                         </td>
                     </tr>
@@ -312,6 +318,7 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_qbittorrent" value="4" ${config['torrent_downloader_qbittorrent']}> QBitTorrent
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
                                 <div class="row">
@@ -386,6 +393,26 @@
                                     <input type="text" name="utorrent_label" value="${config['utorrent_label']}" size="30">
                                 </div>
                             </fieldset>
+                            <fieldset id="qbittorrent_options">
+                                <small class="heading"><i class="fa fa-info-circle"></i> Note: Works with WebAPI Rev 11 and later (QBitTorrent 3.4.0 and later) </small>
+                                <div class="row">
+                                    <label>QBitTorrent Host</label>
+                                    <input type="text" name="qbittorrent_host" value="${config['qbittorrent_host']}" size="30">
+                                    <small>usually http://localhost:8081</small>
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Username</label>
+                                    <input type="text" name="qbittorrent_username" value="${config['qbittorrent_username']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Password</label>
+                                    <input type="password" name="qbittorrent_password" value="${config['qbittorrent_password']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <label>QBitTorrent Label</label>
+                                    <input type="text" name="qbittorrent_label" value="${config['qbittorrent_label']}" size="30">
+				</div>
+                            </fieldset>
                             <fieldset id="deluge_options">
                                 <div class="row">
                                     <label>Deluge WebUI Host and Port</label>
@@ -574,23 +601,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_kat" type="checkbox" class="bigcheck" name="use_kat" value="1" ${config['use_kat']} /><label for="use_kat"><span class="option">Kick Ass Torrents</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Proxy URL</label>
-                                            <input type="text" name="kat_proxy_url" value="${config['kat_proxy_url']}" size="36">
-                                            <small>Optional. Leave empty for default.</small>
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="kat_ratio" value="${config['kat_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                                 <fieldset>
                                     <div class="row checkbox left">
                                         <input id="use_waffles" type="checkbox" class="bigcheck" name="use_waffles" value="1" ${config['use_waffles']} /><label for="use_waffles"><span class="option">Waffles.ch</span></label>
@@ -628,65 +638,57 @@
                                             <label>Seed Ratio</label>
                                             <input type="text" class="override-float" name="rutracker_ratio" value="${config['rutracker_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
+                                        <div class="row">
+                                            <label>Session Cookie (Optional - Advanced)</label>
+                                            <input type="text" class="override-float" name="rutracker_cookie" value="${config['rutracker_cookie']}" size="10" title="bb_session cookie (Advanced)">
+                                        </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_whatcd" type="checkbox" class="bigcheck" name="use_whatcd" value="1" ${config['use_whatcd']} /><label for="use_whatcd"><span class="option">What.cd</span></label>
+                                        <input id="use_orpheus" type="checkbox" class="bigcheck" name="use_orpheus" value="1" ${config['use_orpheus']} /><label for="use_orpheus"><span class="option">Orpheus.network</span></label>
                                     </div>
                                     <div class="config">
                                         <div class="row">
                                             <label>Username</label>
-                                            <input type="text" name="whatcd_username" value="${config['whatcd_username']}" size="36">
+                                            <input type="text" name="orpheus_username" value="${config['orpheus_username']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="whatcd_password" value="${config['whatcd_password'] | h}" size="36">
+                                            <input type="password" name="orpheus_password" value="${config['orpheus_password'] | h}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>URL</label>
-                                            <input type="text" name="whatcd_url" value="${config['whatcd_url']}" size="36">
+                                            <input type="text" name="orpheus_url" value="${config['orpheus_url']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="whatcd_ratio" value="${config['whatcd_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                            <input type="text" class="override-float" name="orpheus_ratio" value="${config['orpheus_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
                                     </div>
                                 </fieldset>
 
                                 <fieldset>
                                     <div class="row checkbox left">
-                                        <input id="use_pth" type="checkbox" class="bigcheck" name="use_pth" value="1" ${config['use_pth']} /><label for="use_pth"><span class="option">PassTheHeadphones.me</span></label>
+                                        <input id="use_redacted" type="checkbox" class="bigcheck" name="use_redacted" value="1" ${config['use_redacted']} /><label for="use_redacted"><span class="option">Redacted</span></label>
                                     </div>
                                     <div class="config">
                                         <div class="row">
                                             <label>Username</label>
-                                            <input type="text" name="pth_username" value="${config['pth_username']}" size="36">
+                                            <input type="text" name="redacted_username" value="${config['redacted_username']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Password</label>
-                                            <input type="password" name="pth_password" value="${config['pth_password']}" size="36">
+                                            <input type="password" name="redacted_password" value="${config['redacted_password'] | h}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>URL</label>
-                                            <input type="text" name="pth_url" value="${config['pth_url']}" size="36">
                                         </div>
                                         <div class="row">
                                             <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="pth_ratio" value="${config['pth_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
+                                            <input type="text" class="override-float" name="redacted_ratio" value="${config['redacted_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
                                         </div>
-                                    </div>
-                                </fieldset>
-
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_strike" type="checkbox" class="bigcheck" name="use_strike" value="1" ${config['use_strike']} /><label for="use_strike"><span class="option">Strike Search</span></label>
-                                    </div>
-                                    <div class="config">
                                         <div class="row">
-                                            <label>Seed Ratio</label>
+                                            <label>Use freeleech tokens when available</label>
-                                            <input type="text" class="override-float" name="strike_ratio" value="${config['strike_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            <input type="checkbox" name="redacted_use_fltoken" value="1" ${config['redacted_use_fltoken']}>
                                         </div>
                                     </div>
                                 </fieldset>
@@ -699,13 +701,17 @@
                                         <div class="config" id="torznab1">
                                             <div class="row">
                                                 <label>Torznab Host</label>
-                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="30">
+                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="36">
-                                                <small>e.g. http://localhost:9117/torznab/iptorrents</small>
+                                                <small>e.g. http://localhost:9117/api/v2.0/indexers/demonoid/results/torznab/</small>
                                             </div>
                                             <div class="row">
                                                 <label>Torznab API</label>
                                                 <input type="text" name="torznab_apikey" value="${config['torznab_apikey']}" size="36">
                                             </div>
+                                            <div class="row">
+                                                <label>Torznab Seed Ratio</label>
+                                                <input type="text" class="override-float" name="torznab_ratio" value="${config['torznab_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                            </div>
                                             <div class="row checkbox">
                                                 <input id="torznab_enabled" type="checkbox" name="torznab_enabled" value="1" ${config['torznab_enabled']} /><label>Enabled</label>
                                             </div>
@@ -715,7 +721,7 @@
                                         %>
                                         %for torznab in config['extra_torznabs']:
                                             <%
-                                                if torznab[2] == '1' or torznab[2] == 1:
+                                                if torznab[3] == '1' or torznab[3] == 1:
                                                     torznab_enabled = "checked"
                                                 else:
                                                     torznab_enabled = ""
@@ -723,12 +729,16 @@
                                             <div class="config" id="torznab${torznab_number}">
                                                 <div class="row">
                                                     <label>Torznab Host</label>
-                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="30">
+                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="36">
                                                 </div>
                                                 <div class="row">
                                                     <label>Torznab API</label>
                                                     <input type="text" name="torznab_api${torznab_number}" value="${torznab[1]}" size="36">
                                                 </div>
+                                                <div class="row">
+                                                    <label>Torznab Seed Ratio</label>
+                                                    <input type="text" class="override-float" name="torznab_ratio${torznab_number}" value="${torznab[2]}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                                </div>
                                                 <div class="row checkbox">
                                                     <input id="torznab_enabled" type="checkbox" name="torznab_enabled${torznab_number}" value="1" ${torznab_enabled} /><label>Enabled</label>
                                                 </div>
@@ -760,34 +770,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                     <div class="row checkbox left">
-                                        <input id="use_mininova" type="checkbox" class="bigcheck" name="use_mininova" value="1" ${config['use_mininova']} /><label for="use_mininova"><span class="option">Mininova</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="mininova_ratio" value="${config['mininova_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding">
-                                        </div>
-                                    </div>
-                                </fieldset>
-								
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_tquattrecentonze" type="checkbox" class="bigcheck" name="use_tquattrecentonze" value="1" ${config['use_tquattrecentonze']} /><label for="use_tquattrecentonze"><span class="option">t411</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>Username</label>
-                                            <input type="text" name="tquattrecentonze_user" value="${config['tquattrecentonze_user']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Password</label>
-                                            <input type="password" name="tquattrecentonze_password" value="${config['tquattrecentonze_password'] | h}" size="36">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                             </fieldset>
                         </td>
                     </tr>
@@ -928,30 +910,42 @@
                                         <input type="checkbox" name="keep_nfo" value="1" ${config['keep_nfo']} />
                                     </label>
                                 </div>
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed lyrics
+                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
+                                    </label>
+                                </div>
+
+
+                                <div class="row checkbox left clearfix nopad">
+                                    <label>
+                                        Embed album art in each file
+                                        <input type="checkbox" name="embed_album_art" id="embed_album_art" value="1" ${config['embed_album_art']}>
+                                    </label>
+                                </div>
+
                                 <div class="row checkbox left clearfix nopad">
                                     <label>
                                         Add album art jpeg to album folder
                                         <input type="checkbox" name="add_album_art" id="add_album_art" value="1" ${config['add_album_art']}>
                                     </label>
                                 </div>
+
                                 <div id="album_art_options" style="padding-left: 20px">
                                     <div class="row">
                                         as <input type="text" class="override-float" name="album_art_format" value="${config['album_art_format']}" size="10">.jpg
                                     </div>
                                     <small>Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
                                 </div>
-                                <div class="row checkbox left clearfix nopad">
+
-                                    <label>
+                                <div id="album_art_size_options" style="padding-left: 20px">
-                                        Embed album art in each file
+                                    <div class="row">
-                                        <input type="checkbox" name="embed_album_art" value="1" ${config['embed_album_art']}>
+                                        Album art min width <input type="text" class="override-float" name="album_art_min_width" value="${config['album_art_min_width']}" size="4" title="Only images with a pixel width greater or equal are considered as valid album art candidates. Default: 0.">\
-                                    </label>
+                                        Album art max width <input type="text" class="override-float" name="album_art_max_width" value="${config['album_art_max_width']}" size="4" title="A maximum image width to downscale fetched images if they are too big.">
                                     </div>
-                                <div class="row checkbox left clearfix nopad">
-                                    <label>
-                                        Embed lyrics
-                                        <input type="checkbox" name="embed_lyrics" value="1" ${config['embed_lyrics']}>
-                                    </label>
                                 </div>
+
                                 <div class="row">
                                     <label>
                                         Destination Directory:
@@ -1302,6 +1296,26 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="slack_enabled" id="slack" value="1" ${config['slack_enabled']} /><label for="slack"><span class="option">Slack</span></label>
+                                </div>
+                                <div id="slackoptions">
+                                    <div class="row">
+                                        <label>Slack Webhook</label><input type="text" name="slack_url" value="${config['slack_url']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Channel</label><input type="text" name="slack_channel" value="${config['slack_channel']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <label>Emoji</label><input type="text" name="slack_emoji" value="${config['slack_emoji']}" size="50">
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="slack_onsnatch" value="1" ${config['slack_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
+
                             <fieldset>
                                 <div class="row checkbox left">
                                     <input type="checkbox" class="bigcheck" name="telegram_enabled" id="telegram" value="1" ${config['telegram_enabled']} /><label for="telegram"><span class="option">Telegram</span></label>
@@ -1319,6 +1333,23 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="join_enabled" id="join" value="1" ${config['join_enabled']} /><label for="join"><span class="option">Join</span></label>
+                                </div>
+                                <div id="joinoptions">
+                                    <div class="row">
+                                        <label>Join API Key</label><input type="text" name="join_apikey" value="${config['join_apikey']}" size="50">
+                                    </div>
+                                    <div class="row">
+                                        <label>Device ID(s)</label><input type="text" name="join_deviceid" value="${config['join_deviceid']}" size="50"><small>Comma separated list.  Leave blank to send to all devices</small>
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="join_onsnatch" value="1" ${config['join_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
+
                         </td>
                     </tr>
                 </table>
@@ -1614,6 +1645,10 @@
                                     <label>Cache Directory</label>
                                     <input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
                                 </div>
+                                <div class="row">
+                                    <label>Post Processing Temporary Directory</label>
+                                    <input type="text" name="keep_torrent_files_dir" value="${config['keep_torrent_files_dir']}" size="50" title="Enter the directory to preserve files for seeding, default is system temp directory">
+                                </div>
                             </fieldset>
 
                             <fieldset>
@@ -1858,9 +1893,36 @@
                         }
                 });
 
+                if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").show();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").hide();
+                              }
+                        }
+
+                $("#embed_album_art").click(function(){
+                        if ($("#embed_album_art").is(":checked"))
+                        {
+                            $("#album_art_size_options").slideDown();
+                        }
+                        else
+                        {
+                            if (!$("#add_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
+                        }
+                });
+
                 if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").show();
+                            $("#album_art_size_options").show();
                         }
                 else
                         {
@@ -1871,10 +1933,15 @@
                         if ($("#add_album_art").is(":checked"))
                         {
                             $("#album_art_options").slideDown();
+                            $("#album_art_size_options").slideDown();
                         }
                         else
                         {
                             $("#album_art_options").slideUp();
+                            if (!$("#embed_album_art").is(":checked"))
+                            {
+                                $("#album_art_size_options").slideUp();
+                            }
                         }
                 });
 
@@ -2079,6 +2146,27 @@
                         }
                 });
 
+                if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").show();
+                        }
+                else
+                        {
+                            $("#joinoptions").hide();
+                        }
+
+
+                $("#join").click(function(){
+                        if ($("#join").is(":checked"))
+                        {
+                            $("#joinoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#joinoptions").slideUp();
+                        }
+                });
+
                 if ($("#twitter").is(":checked"))
                         {
                             $("#twitteroptions").show();
@@ -2099,6 +2187,27 @@
                         }
                 });
 
+                if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").show();
+                        }
+                else
+                        {
+                            $("#slackoptions").hide();
+                        }
+
+                $("#slack").click(function(){
+                        if ($("#slack").is(":checked"))
+                        {
+                            $("#slackoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#slackoptions").slideUp();
+                        }
+                });
+
+
                 if ($("#telegram").is(":checked"))
                         {
                             $("#telegramoptions").show();
@@ -2275,26 +2384,30 @@
 
                 if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#torrent_blackhole_options").show();
                         }
                 if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").hide();
                             $("#transmission_options").show();
                         }
                 if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").hide();
                             $("#utorrent_options").show();
                         }
+                if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").hide();
+                            $("#qbittorrent_options").show();
+                        }
                 if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#qbittorrent_options").hide();
                             $("#deluge_options").show();
                         }
 
-
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
                         {
@@ -2330,19 +2443,23 @@
                         }
                         if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
+                            $("#transmission_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options,#qbittorrent_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                        }
+                        if ($("#torrent_downloader_qbittorrent").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#qbittorrent_options").fadeIn() });
 			}
                         if ($("#torrent_downloader_deluge").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
                 });
 
@@ -2404,7 +2521,7 @@
 
                 $("#add_torznab").click(function() {
                     var intId = $("#torznab_providers > div").size() + deletedTorznabs + 1;
-                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
+                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row\"><label>Torznab Seed Ratio</label><input type=\"text\" class=\"override-float\" name=\"torznab_ratio" + intId + "\" size=\"10\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
                     var removeTorznabButton = $("<div class=\"row\"><input type=\"button\" class=\"remove_torznab\" value=\"Remove\" /></div>");
                     removeTorznabButton.click(function() {
                         $(this).parent().remove();
@@ -2433,20 +2550,16 @@
                 initConfigCheckbox("#use_nzbsorg");
                 initConfigCheckbox("#use_omgwtfnzbs");
                 initConfigCheckbox("#use_torznab");
-                initConfigCheckbox("#use_kat");
                 initConfigCheckbox("#use_piratebay");
                 initConfigCheckbox("#use_oldpiratebay");
-                initConfigCheckbox("#use_mininova");
                 initConfigCheckbox("#use_waffles");
                 initConfigCheckbox("#use_rutracker");
-                initConfigCheckbox("#use_whatcd");
+                initConfigCheckbox("#use_orpheus");
-                initConfigCheckbox("#use_pth");
+                initConfigCheckbox("#use_redacted");
-                initConfigCheckbox("#use_strike");
                 initConfigCheckbox("#api_enabled");
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");
-                initConfigCheckbox("#use_tquattrecentonze");
+                initConfigCheckbox("#mb_ignore_age_missing");
-
 
                 $('#twitterStep1').click(function () {
                     $.get("/twitterStep1", function (data) {window.open(data); })

data/interfaces/default/index.html

@@ -36,7 +36,7 @@
                         "aTargets": [0],
                         "mData":"ArtistID",
                         "mRender": function ( data, type, full ) {
-                            return '<div id="artistImg"><img class="albumArt" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
+                            return '<div id="artistImg"><img class="albumArt" height="50" width="50" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
                         }
                     },
                     {

data/interfaces/default/js/script.js

@@ -312,6 +312,7 @@ function doAjaxCall(url,elem,reload,form) {
 	$.ajax({
 	  url: url,
 	  data: dataString,
+	  type: 'POST',
 	  beforeSend: function(jqXHR, settings) {
 	  	// Start loader etc.
 	  	feedback.prepend(loader);

headphones/__init__.py

@@ -86,8 +86,8 @@ CURRENT_VERSION = None
 LATEST_VERSION = None
 COMMITS_BEHIND = None
 
-LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"]
+LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma", "opus"]
-LOSSLESS_MEDIA_FORMATS = ["flac", "aiff"]
+LOSSLESS_MEDIA_FORMATS = ["flac", "aiff", "aif"]
 MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS
 
 MIRRORLIST = ["musicbrainz.org", "headphones", "custom"]
@@ -306,6 +306,7 @@ def initialize_scheduler():
                          minutes=minutes)
 
         # Remove Torrent + data if Post Processed and finished Seeding
+        if headphones.CONFIG.TORRENT_DOWNLOADER != 0:
             minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
             schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
                          minutes=minutes)
@@ -359,6 +360,8 @@ def sig_handler(signum=None, frame=None):
 
 
 def dbcheck():
+    logger.debug("SQLite Version: %s", sqlite3.sqlite_version)
+    logger.debug("DB-API Version: %s", sqlite3.version)
     conn = sqlite3.connect(DB_FILE)
     c = conn.cursor()
     c.execute(
@@ -376,7 +379,7 @@ def dbcheck():
     c.execute(
         'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)')
     c.execute(
-        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)')
+        'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT, TorrentHash TEXT)')
     # Matched is a temporary value used to see if there was a match found in
     # alltracks
     c.execute(
@@ -416,6 +419,33 @@ def dbcheck():
         'CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)')
     c.execute(
         'CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)')
+    c.execute(
+        'CREATE INDEX IF NOT EXISTS tracks_artistid ON tracks(ArtistID ASC)')
+
+    # Speed up album page
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_albumid ON allalbums(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_albumid ON alltracks(AlbumID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS releases_albumid ON releases(ReleaseGroupID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_albumid ON descriptions(ReleaseGroupID ASC)')
+
+    # Speed up artist deletion
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_artistid ON allalbums(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_artistid ON alltracks(ArtistID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS descriptions_artistid ON descriptions(ArtistID ASC)')
+
+    # Speed up Artist refresh hybrid release
+    c.execute('CREATE INDEX IF NOT EXISTS albums_releaseid ON albums(ReleaseID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_releaseid ON tracks(ReleaseID ASC)')
+
+    # Speed up scanning and track matching
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistname ON artists(ArtistName COLLATE NOCASE ASC)')
+
+    # General speed up
+    c.execute('CREATE INDEX IF NOT EXISTS artist_artistsortname ON artists(ArtistSortName COLLATE NOCASE ASC)')
+
+    c.execute(
+        """CREATE INDEX IF NOT EXISTS have_matched_artist_album ON have(Matched ASC, ArtistName COLLATE NOCASE ASC, AlbumTitle COLLATE NOCASE ASC)""")
+    c.execute('DROP INDEX IF EXISTS have_matched')
 
     try:
         c.execute('SELECT IncludeExtras from artists')
@@ -613,6 +643,20 @@ def dbcheck():
     except sqlite3.OperationalError:
         c.execute('ALTER TABLE artists ADD COLUMN MetaCritic TEXT DEFAULT NULL')
 
+    try:
+        c.execute('SELECT TorrentHash from snatched')
+    except sqlite3.OperationalError:
+        c.execute('ALTER TABLE snatched ADD COLUMN TorrentHash TEXT')
+        c.execute('UPDATE snatched SET TorrentHash = FolderName WHERE Status LIKE "Seed_%"')
+
+    # One off script to set CleanName to lower case
+    clean_name_mixed = c.execute('SELECT CleanName FROM have ORDER BY Date Desc').fetchone()
+    if clean_name_mixed and clean_name_mixed[0] != clean_name_mixed[0].lower():
+        logger.info("Updating track clean name, this could take some time...")
+        c.execute('UPDATE tracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE alltracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+        c.execute('UPDATE have SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName')
+
     conn.commit()
     c.close()
 

headphones/albumart.py

@@ -13,16 +13,164 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import request, db, logger
+import struct
+from six.moves.urllib.parse import urlencode
+from io import BytesIO
+
+import headphones
+from headphones import db, request, logger
 
 
 def getAlbumArt(albumid):
-    myDB = db.DBConnection()
-    asin = myDB.action(
-        'SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0]
 
-    if asin:
+    artwork_path = None
-        return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
+    artwork = None
+
+    # CAA
+    logger.info("Searching for artwork at CAA")
+    artwork_path = 'http://coverartarchive.org/release-group/%s/front' % albumid
+    artwork = getartwork(artwork_path)
+    if artwork:
+        logger.info("Artwork found at CAA")
+        return artwork_path, artwork
+
+    # Amazon
+    logger.info("Searching for artwork at Amazon")
+    myDB = db.DBConnection()
+    dbalbum = myDB.action(
+        'SELECT ArtistName, AlbumTitle, ReleaseID, AlbumASIN FROM albums WHERE AlbumID=?',
+        [albumid]).fetchone()
+    if dbalbum['AlbumASIN']:
+        artwork_path = 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % dbalbum['AlbumASIN']
+        artwork = getartwork(artwork_path)
+        if artwork:
+            logger.info("Artwork found at Amazon")
+            return artwork_path, artwork
+
+    # last.fm
+    from headphones import lastfm
+    logger.info("Searching for artwork at last.fm")
+    if dbalbum['ReleaseID'] != albumid:
+        data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'])
+        if not data:
+            data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                         album=dbalbum['AlbumTitle'])
+    else:
+        data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                     album=dbalbum['AlbumTitle'])
+
+    if data:
+        try:
+            images = data['album']['image']
+            for image in images:
+                if image['size'] == 'extralarge':
+                    artwork_path = image['#text']
+                elif image['size'] == 'mega':
+                    artwork_path = image['#text']
+                    break
+        except KeyError:
+            artwork_path = None
+
+        if artwork_path:
+            artwork = getartwork(artwork_path)
+            if artwork:
+                logger.info("Artwork found at last.fm")
+                return artwork_path, artwork
+
+    logger.info("No suitable album art found.")
+    return None, None
+
+
+def jpeg(bites):
+    fhandle = BytesIO(bites)
+    try:
+        fhandle.seek(0)
+        size = 2
+        ftype = 0
+        while not 0xc0 <= ftype <= 0xcf:
+            fhandle.seek(size, 1)
+            byte = fhandle.read(1)
+            while ord(byte) == 0xff:
+                byte = fhandle.read(1)
+            ftype = ord(byte)
+            size = struct.unpack('>H', fhandle.read(2))[0] - 2
+        fhandle.seek(1, 1)
+        height, width = struct.unpack('>HH', fhandle.read(4))
+        return width, height
+    except struct.error:
+        return None, None
+    except TypeError:
+        return None, None
+
+
+def png(bites):
+    try:
+        check = struct.unpack('>i', bites[4:8])[0]
+        if check != 0x0d0a1a0a:
+            return None, None
+        return struct.unpack('>ii', bites[16:24])
+    except struct.error:
+        return None, None
+
+
+def get_image_data(bites):
+    type = None
+    width = None
+    height = None
+    if len(bites) < 24:
+        return None, None, None
+
+    peek = bites[0:2]
+    if peek == b'\xff\xd8':
+        width, height = jpeg(bites)
+        type = 'jpg'
+    elif peek == b'\x89P':
+        width, height = png(bites)
+        type = 'png'
+    return type, width, height
+
+
+def getartwork(artwork_path):
+    artwork = bytes()
+    minwidth = 0
+    maxwidth = 0
+    if headphones.CONFIG.ALBUM_ART_MIN_WIDTH:
+        minwidth = int(headphones.CONFIG.ALBUM_ART_MIN_WIDTH)
+    if headphones.CONFIG.ALBUM_ART_MAX_WIDTH:
+        maxwidth = int(headphones.CONFIG.ALBUM_ART_MAX_WIDTH)
+
+    resp = request.request_response(artwork_path, timeout=20, stream=True, whitelist_status_code=404)
+
+    if resp:
+        img_width = None
+        for chunk in resp.iter_content(chunk_size=1024):
+            artwork += chunk
+            if not img_width and (minwidth or maxwidth):
+                img_type, img_width, img_height = get_image_data(artwork)
+            # Check min/max
+            if img_width and (minwidth or maxwidth):
+                if minwidth and img_width < minwidth:
+                    logger.info("Artwork is too small. Type: %s. Width: %s. Height: %s",
+                                img_type, img_width, img_height)
+                    artwork = None
+                    break
+                elif maxwidth and img_width > maxwidth:
+                    # Downsize using proxy service to max width
+                    artwork_path = '{0}?{1}'.format('http://images.weserv.nl/', urlencode({
+                        'url': artwork_path.replace('http://', ''),
+                        'w': maxwidth,
+                    }))
+                    artwork = bytes()
+                    r = request.request_response(artwork_path, timeout=20, stream=True, whitelist_status_code=404)
+                    if r:
+                        for chunk in r.iter_content(chunk_size=1024):
+                            artwork += chunk
+                        r.close()
+                        logger.info("Artwork is greater than the maximum width, downsized using proxy service")
+                    break
+        resp.close()
+
+    return artwork
 
 
 def getCachedArt(albumid):

headphones/albumswitcher.py

@@ -85,8 +85,8 @@ def switch(AlbumID, ReleaseID):
 
     # Update have track counts on index
     totaltracks = len(myDB.select(
-        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")',
+        'SELECT TrackTitle from tracks AS tr INNER JOIN albums AS al ON al.AlbumID = tr.AlbumID WHERE al.ArtistID=? '
-        [newalbumdata['ArtistID']]))
+        'AND al.Status != "Ignored"', [newalbumdata['ArtistID']]))
     havetracks = len(myDB.select(
         'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
         [newalbumdata['ArtistID']]))

headphones/api.py

@@ -82,7 +82,7 @@ class Api(object):
     def fetchData(self):
 
         if self.data == 'OK':
-            logger.info('Recieved API command: %s', self.cmd)
+            logger.info('Received API command: %s', self.cmd)
             methodToCall = getattr(self, "_" + self.cmd)
             methodToCall(**self.kwargs)
             if 'callback' not in self.kwargs:

headphones/cache.py

@@ -14,11 +14,16 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from six.moves.urllib.parse import urlencode
 
 import headphones
-from headphones import db, helpers, logger, lastfm, request
+from headphones import db, helpers, logger, lastfm, request, mb
 
-LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
+LASTFM_API_KEY = "8d983789c771afaeb7412ac358d4bad0"
+
+FANART_URL = 'https://webservice.fanart.tv/v3/music/'
+FANART_PROJECT_KEY = '22b73c9603eba09d0c855f2d2bdba31c'
+FANART_CLIENT_KEY = '919b389a18a3f0b2c916090022ab3c7a'
 
 
 class Cache(object):
@@ -96,6 +101,11 @@ class Cache(object):
     def _is_current(self, filename=None, date=None):
 
         if filename:
+
+            # 2019 last.fm no longer allows access to artist images, for now we'll keep the cached artist image if it exists and get new from fanart.tv
+            if self.id_type == 'artist' and 'fanart' not in filename:
+                return True
+
             base_filename = os.path.basename(filename)
             date = base_filename.split('.')[1]
 
@@ -211,20 +221,34 @@ class Cache(object):
         if ArtistID:
 
             self.id_type = 'artist'
-            data = lastfm.request_lastfm("artist.getinfo", mbid=ArtistID, api_key=LASTFM_API_KEY)
+
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + ArtistID, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
 
             if not data:
                 return
 
-            try:
+            if data.get('artistthumb'):
-                image_url = data['artist']['image'][-1]['#text']
+                image_url = data['artistthumb'][0]['url']
-            except (KeyError, IndexError):
+            elif data.get('artistbackground'):
-                logger.debug('No artist image found')
+                image_url = data['artistbackground'][0]['url']
-                image_url = None
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
 
-            thumb_url = self._get_thumb_url(data)
+            # fallback to 1st album cover if none of the above
-            if not thumb_url:
+            elif 'albums' in data:
-                logger.debug('No artist thumbnail image found')
+                for mbid, art in data.get('albums', dict()).items():
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            if image_url:
+                thumb_url = image_url
+            else:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
 
@@ -283,6 +307,7 @@ class Cache(object):
         """
 
         myDB = db.DBConnection()
+        fanart = False
 
         # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums
         # Exception is when adding albums manually, then we should use release id
@@ -290,6 +315,14 @@ class Cache(object):
 
             data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY)
 
+            # Try with name if not found
+            if not data:
+                dbartist = myDB.action('SELECT ArtistName, Type FROM artists WHERE ArtistID=?', [self.id]).fetchone()
+                if dbartist:
+                    data = lastfm.request_lastfm("artist.getinfo",
+                                                 artist=helpers.clean_musicbrainz_name(dbartist['ArtistName']),
+                                                 api_key=LASTFM_API_KEY)
+
             if not data:
                 return
 
@@ -303,30 +336,70 @@ class Cache(object):
             except KeyError:
                 logger.debug('No artist bio found')
                 self.info_content = None
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except KeyError:
-                logger.debug('No artist image found')
-                image_url = None
 
-            thumb_url = self._get_thumb_url(data)
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
-            if not thumb_url:
+            image_url = None
-                logger.debug('No artist thumbnail image found')
+            thumb_url = None
+            data = request.request_json(FANART_URL + self.id, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
+
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
+
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in data.get('albums', dict()).items():
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            # finally, use 1st album cover from last.fm
+            if image_url:
+                fanart = True
+                thumb_url = image_url
+            else:
+                dbalbum = myDB.action(
+                    'SELECT ArtworkURL, ThumbURL FROM albums WHERE ArtworkURL IS NOT NULL AND ArtistID=?',
+                    [self.id]).fetchone()
+                if dbalbum:
+                    fanart = True
+                    image_url = dbalbum['ArtworkURL']
+                    thumb_url = dbalbum['ThumbURL']
+
+            if not image_url:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
             dbalbum = myDB.action(
-                'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?',
+                'SELECT ArtistName, AlbumTitle, ReleaseID, Type FROM albums WHERE AlbumID=?',
                 [self.id]).fetchone()
             if dbalbum['ReleaseID'] != self.id:
                 data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'],
                                              api_key=LASTFM_API_KEY)
                 if not data:
-                    data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                    data = lastfm.request_lastfm("album.getinfo",
-                                                 album=dbalbum['AlbumTitle'],
+                                                 artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                 album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
                                                  api_key=LASTFM_API_KEY)
             else:
-                data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                if dbalbum['Type'] != "part of":
-                                             album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
+                    data = lastfm.request_lastfm("album.getinfo",
+                                                artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                api_key=LASTFM_API_KEY)
+                else:
+
+                    # Series, use actual artist for the release-group
+                    artist = mb.getArtistForReleaseGroup(self.id)
+                    if artist:
+                        data = lastfm.request_lastfm("album.getinfo",
+                                                     artist=helpers.clean_musicbrainz_name(artist),
+                                                     album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                     api_key=LASTFM_API_KEY)
 
             if not data:
                 return
@@ -406,6 +479,10 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
+                if fanart:
+                    artwork_path = os.path.join(self.path_to_art_cache,
+                                                self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
                     artwork_path = os.path.join(self.path_to_art_cache,
                                             self.id + '.' + helpers.today() + ext)
                 try:
@@ -422,6 +499,8 @@ class Cache(object):
         # as it's missing/outdated.
         if thumb_url and self.query_type in ['thumb', 'artwork'] and not (
                 self.thumb_files and self._is_current(self.thumb_files[0])):
+
+            if not (self.query_type == 'artwork' and 'fanart' in thumb_url and artwork):
                 artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
@@ -445,9 +524,32 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
+                if fanart:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
                     thumb_path = os.path.join(self.path_to_art_cache,
                                               'T_' + self.id + '.' + helpers.today() + ext)
                 try:
+                    if self.id_type != 'artist':
+                        with open(thumb_path, 'wb') as f:
+                            f.write(artwork)
+                    else:
+
+                        # 2019 last.fm no longer allows access to artist images, use the fanart.tv image to create a thumb
+                        artwork_thumb = None
+                        if 'fanart' in thumb_url:
+                            # Create thumb using image resizing service
+                            artwork_path = '{0}?{1}'.format('http://images.weserv.nl/', urlencode({
+                                'url': thumb_url.replace('http://', ''),
+                                'w': 300,
+                            }))
+                            artwork_thumb = request.request_content(artwork_path, timeout=20, whitelist_status_code=404)
+
+                        if artwork_thumb:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork_thumb)
+                        else:
                             with open(thumb_path, 'wb') as f:
                                 f.write(artwork)
 
@@ -465,7 +567,7 @@ def getArtwork(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         artwork_file = os.path.basename(artwork_path)
@@ -479,7 +581,7 @@ def getThumb(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         thumbnail_file = os.path.basename(artwork_path)

headphones/config.py

@@ -34,11 +34,18 @@ _CONFIG_DEFINITIONS = {
     'ADD_ALBUM_ART': (int, 'General', 0),
     'ADVANCEDENCODER': (str, 'General', ''),
     'ALBUM_ART_FORMAT': (str, 'General', 'folder'),
+    'ALBUM_ART_MIN_WIDTH': (str, 'General', ''),
+    'ALBUM_ART_MAX_WIDTH': (str, 'General', ''),
     # This is used in importer.py to determine how complete an album needs to
     # be - to be considered "downloaded". Percentage from 0-100
     'ALBUM_COMPLETION_PCT': (int, 'Advanced', 80),
     'API_ENABLED': (int, 'General', 0),
     'API_KEY': (str, 'General', ''),
+    'ORPHEUS': (int, 'Orpheus.network', 0),
+    'ORPHEUS_PASSWORD': (str, 'Orpheus.network', ''),
+    'ORPHEUS_RATIO': (str, 'Orpheus.network', ''),
+    'ORPHEUS_USERNAME': (str, 'Orpheus.network', ''),
+    'ORPHEUS_URL': (str, 'Orpheus.network', 'https://orpheus.network'),
     'AUTOWANT_ALL': (int, 'General', 0),
     'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1),
     'AUTOWANT_UPCOMING': (int, 'General', 1),
@@ -138,12 +145,14 @@ _CONFIG_DEFINITIONS = {
     'IGNORED_FILES': (list, 'Advanced', []),    # path
     'INCLUDE_EXTRAS': (int, 'General', 0),
     'INTERFACE': (str, 'General', 'default'),
+    'JOIN_APIKEY': (str, 'Join', ''),
+    'JOIN_DEVICEID': (str, 'Join', ''),
+    'JOIN_ENABLED': (int, 'Join', 0),
+    'JOIN_ONSNATCH': (int, 'Join', 0),
     'JOURNAL_MODE': (str, 'Advanced', 'wal'),
-    'KAT': (int, 'Kat', 0),
-    'KAT_PROXY_URL': (str, 'Kat', ''),
-    'KAT_RATIO': (str, 'Kat', ''),
     'KEEP_NFO': (int, 'General', 0),
     'KEEP_TORRENT_FILES': (int, 'General', 0),
+    'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
@@ -155,8 +164,7 @@ _CONFIG_DEFINITIONS = {
     'LOSSLESS_BITRATE_TO': (int, 'General', 0),
     'LOSSLESS_DESTINATION_DIR': (path, 'General', ''),
     'MB_IGNORE_AGE': (int, 'General', 365),
-    'MININOVA': (int, 'Mininova', 0),
+    'MB_IGNORE_AGE_MISSING': (int, 'General', 0),
-    'MININOVA_RATIO': (str, 'Mininova', ''),
     'MIRROR': (str, 'General', 'musicbrainz.org'),
     'MOVE_FILES': (int, 'General', 0),
     'MPC_ENABLED': (bool_int, 'MPC', False),
@@ -226,6 +234,10 @@ _CONFIG_DEFINITIONS = {
     'PUSHOVER_KEYS': (str, 'Pushover', ''),
     'PUSHOVER_ONSNATCH': (int, 'Pushover', 0),
     'PUSHOVER_PRIORITY': (int, 'Pushover', 0),
+    'QBITTORRENT_HOST': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_LABEL': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
+    'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
@@ -236,6 +248,7 @@ _CONFIG_DEFINITIONS = {
     'RUTRACKER_PASSWORD': (str, 'Rutracker', ''),
     'RUTRACKER_RATIO': (str, 'Rutracker', ''),
     'RUTRACKER_USER': (str, 'Rutracker', ''),
+    'RUTRACKER_COOKIE': (str, 'Rutracker', ''),
     'SAB_APIKEY': (str, 'SABnzbd', ''),
     'SAB_CATEGORY': (str, 'SABnzbd', ''),
     'SAB_HOST': (str, 'SABnzbd', ''),
@@ -243,13 +256,16 @@ _CONFIG_DEFINITIONS = {
     'SAB_USERNAME': (str, 'SABnzbd', ''),
     'SAMPLINGFREQUENCY': (int, 'General', 44100),
     'SEARCH_INTERVAL': (int, 'General', 1440),
+    'SLACK_ENABLED': (int, 'Slack', 0),
+    'SLACK_URL': (str, 'Slack', ''),
+    'SLACK_CHANNEL': (str, 'Slack', ''),
+    'SLACK_EMOJI': (str, 'Slack', ''),
+    'SLACK_ONSNATCH': (int, 'Slack', 0),
     'SOFT_CHROOT': (path, 'General', ''),
     'SONGKICK_APIKEY': (str, 'Songkick', 'nd1We7dFW2RqxPw8'),
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
-    'STRIKE': (int, 'Strike', 0),
-    'STRIKE_RATIO': (str, 'Strike', ''),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
@@ -266,6 +282,7 @@ _CONFIG_DEFINITIONS = {
     'TORZNAB_APIKEY': (str, 'Torznab', ''),
     'TORZNAB_ENABLED': (int, 'Torznab', 1),
     'TORZNAB_HOST': (str, 'Torznab', ''),
+    'TORZNAB_RATIO': (str, 'Torznab', ''),
     'TRANSMISSION_HOST': (str, 'Transmission', ''),
     'TRANSMISSION_PASSWORD': (str, 'Transmission', ''),
     'TRANSMISSION_USERNAME': (str, 'Transmission', ''),
@@ -274,9 +291,6 @@ _CONFIG_DEFINITIONS = {
     'TWITTER_PASSWORD': (str, 'Twitter', ''),
     'TWITTER_PREFIX': (str, 'Twitter', 'Headphones'),
     'TWITTER_USERNAME': (str, 'Twitter', ''),
-    'TQUATTRECENTONZE': (int, 'tquattrecentonze', 0),
-    'TQUATTRECENTONZE_PASSWORD': (str, 'tquattrecentonze', ''),
-    'TQUATTRECENTONZE_USER': (str, 'tquattrecentonze', ''),
     'UPDATE_DB_INTERVAL': (int, 'General', 24),
     'USENET_RETENTION': (int, 'General', '1500'),
     'UTORRENT_HOST': (str, 'uTorrent', ''),
@@ -289,16 +303,11 @@ _CONFIG_DEFINITIONS = {
     'WAFFLES_PASSKEY': (str, 'Waffles', ''),
     'WAFFLES_RATIO': (str, 'Waffles', ''),
     'WAFFLES_UID': (str, 'Waffles', ''),
-    'WHATCD': (int, 'What.cd', 0),
+    'REDACTED': (int, 'Redacted', 0),
-    'WHATCD_PASSWORD': (str, 'What.cd', ''),
+    'REDACTED_USERNAME': (str, 'Redacted', ''),
-    'WHATCD_RATIO': (str, 'What.cd', ''),
+    'REDACTED_PASSWORD': (str, 'Redacted', ''),
-    'WHATCD_USERNAME': (str, 'What.cd', ''),
+    'REDACTED_RATIO': (str, 'Redacted', ''),
-    'WHATCD_URL': (str, 'What.cd', 'https://what.cd'),
+    'REDACTED_USE_FLTOKEN': (int, 'Redacted', 0),
-    'PTH': (int, 'PassTheHeadphones.me', 0),
-    'PTH_PASSWORD': (str, 'PassTheHeadphones.me', ''),
-    'PTH_RATIO': (str, 'PassTheHeadphones.me', ''),
-    'PTH_USERNAME': (str, 'PassTheHeadphones.me', ''),
-    'PTH_URL': (str, 'PassTheHeadphones.me', 'https://passtheheadphones.me'),
     'XBMC_ENABLED': (int, 'XBMC', 0),
     'XBMC_HOST': (str, 'XBMC', ''),
     'XBMC_NOTIFY': (int, 'XBMC', 0),
@@ -403,8 +412,8 @@ class Config(object):
     def get_extra_torznabs(self):
         """ Return the extra torznab tuples """
         extra_torznabs = list(
-            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
-                             for i in range(3)])
+                             for i in range(4)])
         )
         return extra_torznabs
 
@@ -475,4 +484,17 @@ class Config(object):
         if self.CONFIG_VERSION == '5':
             if self.OPEN_MAGNET_LINKS:
                 self.MAGNET_LINKS = 2
-            self.CONFIG_VERSION = '5'
+
+            # Add Seed Ratio to Torznabs
+            if self.EXTRA_TORZNABS:
+                extra_torznabs = list(
+                    itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+                                     for i in range(3)])
+                )
+                new_torznabs = []
+                for torznab in extra_torznabs:
+                    new_torznabs.extend([torznab[0], torznab[1], u'', torznab[2]])
+                if new_torznabs:
+                    self.EXTRA_TORZNABS = new_torznabs
+
+            self.CONFIG_VERSION = '6'

headphones/config_test.py

@@ -355,86 +355,89 @@ class ConfigApiTest(TestCase):
     #   TORZNABS
     # TODO : here is copypaste from of NEZNABS tests. Make tests better, plz refactor them
     #
-    @TestArgs(
+    # TODO: Fix tests for following:
-        ('', []),
+    #   CONFIG_VERSION == '5' each entry = 'host, api, enabled'
-        ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
+    #   CONFIG_VERSION > '5' each entry = 'host, api, seed ratio, enabled'
-        (['ABC', 'DEF'], []),
+    # @TestArgs(
-        ([1], []),
+    #     ('', []),
-        ([1, 2], []),
+    #     ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
-        ([1, 2, 3], [(1, 2, 3)]),
+    #     (['ABC', 'DEF'], []),
-
+    #     ([1], []),
-        ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
+    #     ([1, 2], []),
-        ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
+    #     ([1, 2, 3], [(1, 2, 3)]),
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    #
-        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    #     ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
-    )
+    #     ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
-    def test_get_extra_torznabs(self, conf_value, expected):
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-        """ Config: get_extra_torznabs """
+    #     ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_get_extra_torznabs(self, conf_value, expected):
-        # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
+    #     """ Config: get_extra_torznabs """
-        # set up mocks:
+    #     path = '/tmp/notexist'
-        # 'EXTRA_TORZNABS': (list, '', ''),
+    #
-        self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
+    #     # itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
-
+    #     # set up mocks:
-        # call methods
+    #     # 'EXTRA_TORZNABS': (list, '', ''),
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
-        res = c.get_extra_torznabs()
+    #
-
+    #     # call methods
-        # assertions:
+    #     c = headphones.config.Config(path)
-        self.assertEqual(res, expected)
+    #     res = c.get_extra_torznabs()
-
+    #
-    def test_clear_extra_torznabs(self):
+    #     # assertions:
-        """ Config: clear_extra_torznabs """
+    #     self.assertEqual(res, expected)
-        path = '/tmp/notexist'
+    #
-
+    # def test_clear_extra_torznabs(self):
-        random_value = -1292721
+    #     """ Config: clear_extra_torznabs """
-        self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
+    #     path = '/tmp/notexist'
-        self.config_mock["Torznab"] = {"do_not_touch": random_value}
+    #
-
+    #     random_value = -1292721
-        # call methods
+    #     self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"do_not_touch": random_value}
-        res = c.clear_extra_torznabs()
+    #
-
+    #     # call methods
-        # assertions:
+    #     c = headphones.config.Config(path)
-        self.assertIsNone(res)
+    #     res = c.clear_extra_torznabs()
-        self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
+    #
-        self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
+    #     # assertions:
-
+    #     self.assertIsNone(res)
-    @TestArgs(
+    #     self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
-        ([], [''], ['']),
+    #     self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
-        ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
+    #
-
+    # @TestArgs(
-        ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
+    #     ([], [''], ['']),
-    )
+    #     ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
-    def test_add_extra_torznab(self, initial, added, expected):
+    #
-        """ Config: add_extra_torznab """
+    #     ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_add_extra_torznab(self, initial, added, expected):
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #     """ Config: add_extra_torznab """
-
+    #     path = '/tmp/notexist'
-        # call methods
+    #
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
-        c.add_extra_torznab(added)
+    #
-        act = self.config_mock["Torznab"]["extra_torznabs"]
+    #     # call methods
-
+    #     c = headphones.config.Config(path)
-        # assertions:
+    #     c.add_extra_torznab(added)
-        self.assertEqual(act, expected)
+    #     act = self.config_mock["Torznab"]["extra_torznabs"]
-
+    #
-    @TestArgs(
+    #     # assertions:
-        (None),
+    #     self.assertEqual(act, expected)
-        ([]),
+    #
-        ([1, 2, 3]),
+    # @TestArgs(
-        ([True]),
+    #     (None),
-    )
+    #     ([]),
-    def test_add_extra_torznab_raise_on_none(self, initial):
+    #     ([1, 2, 3]),
-        """ Config: add_extra_torznab should raise on None adding"""
+    #     ([True]),
-        path = '/tmp/notexist'
+    # )
-
+    # def test_add_extra_torznab_raise_on_none(self, initial):
-        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+    #     """ Config: add_extra_torznab should raise on None adding"""
-
+    #     path = '/tmp/notexist'
-        # call methods
+    #
-        c = headphones.config.Config(path)
+    #     self.config_mock["Torznab"] = {"extra_torznabs": initial}
-        with self.assertRaises(TypeError):
+    #
-            c.add_extra_torznab(None)
+    #     # call methods
-        pass
+    #     c = headphones.config.Config(path)
+    #     with self.assertRaises(TypeError):
+    #         c.add_extra_torznab(None)
+    #     pass

headphones/db.py

@@ -19,6 +19,8 @@
 
 from __future__ import with_statement
 
+import time
+
 import sqlite3
 
 import os
@@ -45,37 +47,98 @@ class DBConnection:
         self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
         # don't wait for the disk to finish writing
         self.connection.execute("PRAGMA synchronous = OFF")
-        # journal disabled since we never do rollbacks
+        # default set to Write-Ahead Logging WAL
         self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE)
         # 64mb of cache memory,probably need to make it user configurable
         self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024))
         self.connection.row_factory = sqlite3.Row
 
-    def action(self, query, args=None):
+    def action(self, query, args=None, upsert_insert_qry=None):
 
         if query is None:
             return
 
         sqlResult = None
+        attempts = 0
+        dberror = None
 
+        while attempts < 10:
             try:
                 with self.connection as c:
+
+                    # log that previous attempt was locked and we're trying again
+                    if dberror:
+                        if args is None:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s', attempts + 1, query)
+                        else:
+                            logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s. Args: %s', attempts + 1, query, args)
+
+                    # debugging
+                    # try:
+                    #     explain_query = 'EXPLAIN QUERY PLAN ' + query
+                    #     if not args:
+                    #         sql_results = c.execute(explain_query)
+                    #     else:
+                    #         sql_results = c.execute(explain_query, args)
+                    #     if not args:
+                    #         print(explain_query)
+                    #     else:
+                    #         print(explain_query + ' ' + str(args))
+                    #     explain_results = sql_results
+                    #     for row in explain_results:
+                    #         print row
+                    # except Exception as e:
+                    #     print(e)
+
+                    # Execute query
+
+                    # time0 = time.time()
+
                     if args is None:
                         sqlResult = c.execute(query)
+                        # logger.debug('SQL: ' + query)
                     else:
                         sqlResult = c.execute(query, args)
+                        # logger.debug('SQL: %s. Args: %s', query, args)
+
+                        # INSERT part of upsert query
+                        if upsert_insert_qry:
+                            sqlResult = c.execute(upsert_insert_qry, args)
+                            # logger.debug('SQL: %s. Args: %s', upsert_insert_qry, args)
+
+                    # debugging: loose test to log queries taking longer than 5 seconds
+                    # seconds = time.time() - time0
+                    # if seconds > 5:
+                    #     if args is None:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s", seconds, query)
+                    #     else:
+                    #         logger.debug("SQL: Query ran for %s seconds: %s with args %s", seconds, query, args)
+
+                    break
 
             except sqlite3.OperationalError, e:
                 if "unable to open database file" in e.message or "database is locked" in e.message:
-                logger.warn('Database Error: %s', e)
+                    dberror = e
+                    if args is None:
+                        logger.debug('Database error: %s. Query: %s', e, query)
+                    else:
+                        logger.debug('Database error: %s. Query: %s. Args: %s', e, query, args)
+                    attempts += 1
+                    time.sleep(1)
                 else:
                     logger.error('Database error: %s', e)
                     raise
-
             except sqlite3.DatabaseError, e:
                 logger.error('Fatal Error executing %s :: %s', query, e)
                 raise
 
+        # log if no results returned due to lock
+        if not sqlResult and attempts:
+            if args is None:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s', dberror, query)
+            else:
+                logger.warn('SQL: Query failed due to database error: %s. Query: %s. Args: %s', dberror, query, args)
+
         return sqlResult
 
     def select(self, query, args=None):
@@ -88,24 +151,19 @@ class DBConnection:
         return sqlResults
 
     def upsert(self, tableName, valueDict, keyDict):
-
+        """
+        Transactions an Update or Insert to a table based on key.
+        If the table is not updated then the 'WHERE changes' will be 0 and the table inserted
+        """
         def genParams(myDict):
             return [x + " = ?" for x in myDict.keys()]
 
-        changesBefore = self.connection.total_changes
+        update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
 
-        update_query = "UPDATE " + tableName + " SET " + ", ".join(
+        insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + " SELECT " + ", ".join(
-            genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
+            ["?"] * len(valueDict.keys() + keyDict.keys())) + " WHERE changes()=0")
 
-        self.action(update_query, valueDict.values() + keyDict.values())
-
-        if self.connection.total_changes == changesBefore:
-            insert_query = (
-                "INSERT INTO " + tableName + " (" + ", ".join(
-                    valueDict.keys() + keyDict.keys()) + ")" +
-                " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
-            )
         try:
-                self.action(insert_query, valueDict.values() + keyDict.values())
+            self.action(update_query, valueDict.values() + keyDict.values(), upsert_insert_qry=insert_query)
         except sqlite3.IntegrityError:
             logger.info('Queries failed: %s and %s', update_query, insert_query)

headphones/deluge.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 #  This file is part of Headphones.
 #
 #  Headphones is free software: you can redistribute it and/or modify
@@ -49,6 +51,7 @@ delugeweb_auth = {}
 delugeweb_url = ''
 deluge_verify_cert = False
 scrub_logs = True
+headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 
 
 def _scrubber(text):
@@ -80,7 +83,7 @@ def addTorrent(link, data=None, name=None):
 
         result = {}
         retid = False
-        url_what = ['https://what.cd/', 'http://what.cd/']
+        url_orpheus = ['https://orpheus.network/', 'http://orpheus.network/']
         url_waffles = ['https://waffles.ch/', 'http://waffles.ch/']
 
         if link.lower().startswith('magnet:'):
@@ -94,7 +97,7 @@ def addTorrent(link, data=None, name=None):
             if link.lower().startswith(tuple(url_waffles)):
                 if 'rss=' not in link:
                     link = link + '&rss=1'
-            if link.lower().startswith(tuple(url_what)):
+            if link.lower().startswith(tuple(url_orpheus)):
                 logger.debug('Deluge: Using different User-Agent for this site')
                 user_agent = 'Headphones'
                 # This method will make Deluge download the file
@@ -104,11 +107,11 @@ def addTorrent(link, data=None, name=None):
                 # return addTorrent(local_torrent_path)
             else:
                 user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
-            headers = {'User-Agent': user_agent}
+            get_headers = {'User-Agent': user_agent}
             torrentfile = ''
             logger.debug('Deluge: Trying to download (GET)')
             try:
-                r = requests.get(link, headers=headers)
+                r = requests.get(link, headers=get_headers)
                 if r.status_code == 200:
                     logger.debug('Deluge: 200 OK')
                     # .text will ruin the encoding for some torrents
@@ -135,7 +138,10 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
+            try:
                 logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except:
+                logger.debug('Deluge: Sending Deluge torrent with problematic name and some content')
             result = {'type': 'torrent',
                         'name': name,
                         'content': torrentfile}
@@ -164,7 +170,10 @@ def addTorrent(link, data=None, name=None):
                     # remove '.torrent' suffix
                     if name[-len('.torrent'):] == '.torrent':
                         name = name[:-len('.torrent')]
+            try:
                 logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
+            except UnicodeDecodeError:
+                logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name.decode('utf-8'), str(torrentfile)[:40]))
             result = {'type': 'torrent',
                         'name': name,
                         'content': torrentfile}
@@ -199,7 +208,7 @@ def getTorrentFolder(result):
                                 ],
                                 "id": 21})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['total_done'] = json.loads(response.text)['result']['total_done']
 
         tries = 0
@@ -207,7 +216,7 @@ def getTorrentFolder(result):
             tries += 1
             time.sleep(5)
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             result['total_done'] = json.loads(response.text)['result']['total_done']
 
         post_data = json.dumps({"method": "web.get_torrent_status",
@@ -226,7 +235,7 @@ def getTorrentFolder(result):
                                 "id": 23})
 
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
 
         result['save_path'] = json.loads(response.text)['result']['save_path']
         result['name'] = json.loads(response.text)['result']['name']
@@ -242,7 +251,34 @@ def removeTorrent(torrentid, remove_data=False):
         _get_auth()
 
     try:
+        logger.debug('Deluge: Checking if torrent %s finished seeding' % str(torrentid))
+        post_data = json.dumps({"method": "web.get_torrent_status",
+                                "params": [
+                                    torrentid,
+                                    [
+                                        "name",
+                                        "ratio",
+                                        "state"
+                                    ]
+                                ],
+                                "id": 26})
+
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+            verify=deluge_verify_cert, headers=headers)
+
+        try:
+            state = json.loads(response.text)['result']['state']
+        except KeyError as e:
+            logger.debug('Deluge: "state" KeyError when trying to remove torrent %s' % str(torrentid))
+            return False
+
+        not_finished = ["queued", "seeding", "downloading", "checking", "error"]
         result = False
+        if state.lower() in not_finished:
+            logger.debug('Deluge: Torrent %s is either queued or seeding, not removing yet' % str(torrentid))
+            return False
+        else:
+            logger.debug('Deluge: Removing torrent %s' % str(torrentid))
             post_data = json.dumps({"method": "core.remove_torrent",
                                     "params": [
                                         torrentid,
@@ -250,7 +286,7 @@ def removeTorrent(torrentid, remove_data=False):
                                         ],
                                     "id": 25})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             result = json.loads(response.text)['result']
 
             return result
@@ -293,12 +329,12 @@ def _get_auth():
                             "id": 1})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
     except requests.ConnectionError:
         try:
             logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case')
             response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             # If the previous line didn't fail, change delugeweb_url for the rest of this session
             logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!')
             delugeweb_url = delugeweb_url.replace('http:', 'https:')
@@ -323,7 +359,7 @@ def _get_auth():
                             "id": 10})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
     except Exception as e:
         logger.error('Deluge: Authentication failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -340,7 +376,7 @@ def _get_auth():
                                 "id": 11})
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -348,7 +384,8 @@ def _get_auth():
             return None
 
         delugeweb_hosts = json.loads(response.text)['result']
-        if len(delugeweb_hosts) == 0:
+        # Check if delugeweb_hosts is None before checking its length
+        if not delugeweb_hosts or len(delugeweb_hosts) == 0:
             logger.error('Deluge: WebUI does not contain daemons')
             return None
 
@@ -358,7 +395,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -371,7 +408,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -396,7 +433,7 @@ def _add_torrent_magnet(result):
                                 "params": [result['url'], {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -416,7 +453,7 @@ def _add_torrent_url(result):
                                 "params": [result['url'], {}],
                                 "id": 32})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['location'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -434,10 +471,11 @@ def _add_torrent_file(result):
     try:
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
-                                "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}],
+                                "params": [result['name'] + '.torrent',
+                                b64encode(result['content'].encode('utf8')), {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -445,11 +483,13 @@ def _add_torrent_file(result):
         try:
             # content is torrent file contents that needs to be encoded to base64
             # this time let's try leaving the encoding as is
+            logger.debug('Deluge: There was a decoding issue, let\'s try again')
             post_data = json.dumps({"method": "core.add_torrent_file",
-                                    "params": [result['name'] + '.torrent', b64encode(result['content']), {}],
+                                    "params": [result['name'].decode('utf8') + '.torrent',
+                                    b64encode(result['content']), {}],
                                     "id": 22})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             result['hash'] = json.loads(response.text)['result']
             logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
             return json.loads(response.text)['result']
@@ -481,7 +521,7 @@ def setTorrentLabel(result):
                                 "params": [],
                                 "id": 3})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-            verify=deluge_verify_cert)
+            verify=deluge_verify_cert, headers=headers)
         labels = json.loads(response.text)['result']
 
         if labels is not None:
@@ -492,7 +532,7 @@ def setTorrentLabel(result):
                                             "params": [label],
                                             "id": 4})
                     response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                        verify=deluge_verify_cert)
+                        verify=deluge_verify_cert, headers=headers)
                     logger.debug('Deluge: %s label added to Deluge' % label)
                 except Exception as e:
                     logger.error('Deluge: Setting label failed: %s' % str(e))
@@ -504,7 +544,7 @@ def setTorrentLabel(result):
                                     "params": [result['hash'], label],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             logger.debug('Deluge: %s label added to torrent' % label)
         else:
             logger.debug('Deluge: Label plugin not detected')
@@ -528,12 +568,12 @@ def setSeedRatio(result):
                                     "params": [result['hash'], True],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
             post_data = json.dumps({"method": "core.set_torrent_stop_ratio",
                                     "params": [result['hash'], float(ratio)],
                                     "id": 6})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -556,7 +596,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], True],
                                     "id": 7})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
 
             if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
                 move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -570,7 +610,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], move_to],
                                     "id": 8})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -593,7 +633,7 @@ def setTorrentPause(result):
                                     "params": [[result['hash']]],
                                     "id": 9})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 

headphones/helpers.py

@@ -20,6 +20,12 @@ import datetime
 import shutil
 import time
 import sys
+import tempfile
+import glob
+
+from beets import logging as beetslogging
+import six
+from contextlib import contextmanager
 
 import fnmatch
 import re
@@ -191,9 +197,15 @@ def piratesize(size):
     return size
 
 
-def replace_all(text, dic, normalize=False):
+def pattern_substitute(pattern, dic, normalize=False):
+    """
+    Execute path rendering/substitution based on replacement dictionary
+    e.g. pattern = $Artist/$Album
+         dic = {Artist: 'My artist', Album: 'My album'}
+         returns My artist/My album
+    """
     from headphones import pathrender
-    if not text:
+    if not pattern:
         return ''
 
     if normalize:
@@ -210,15 +222,23 @@ def replace_all(text, dic, normalize=False):
                         j.decode(headphones.SYS_ENCODING, 'replace'))
             new_dic[i] = j
         dic = new_dic
-    return pathrender.render(text, dic)[0]
+    return pathrender.render(pattern, dic)[0]
+
+
+def replace_all(text, dic):
+    if not text:
+        return ''
+
+    for i, j in dic.iteritems():
+        text = text.replace(i, j)
+    return text
 
 
 def replace_illegal_chars(string, type="file"):
     if type == "file":
         string = re.sub('[\?"*:|<>/]', '_', string)
     if type == "folder":
-        string = re.sub('[:\?<>"|]', '_', string)
+        string = re.sub('[:\?<>"|*]', '_', string)
-
     return string
 
 
@@ -255,6 +275,13 @@ _XLATE_SPECIAL = {
     u'&': ' and ',     # expand & to ' and '
 }
 
+_XLATE_MUSICBRAINZ = {
+    # Translation table for Musicbrainz.
+    u"…": '...',     # HORIZONTAL ELLIPSIS (U+2026)
+    u"’": "'",       # APOSTROPHE (U+0027)
+    u"‐": "-",       # EN DASH (U+2013)
+}
+
 
 def _translate(s, dictionary):
     # type: (basestring,Mapping[basestring,basestring])->basestring
@@ -320,6 +347,24 @@ def clean_name(s):
     # 6. trim
     u = u.strip()
     # 7. lowercase
+    u = u.lower()
+    return u
+
+
+def clean_musicbrainz_name(s, return_as_string=True):
+    # type: (basestring)->unicode
+    """Substitute special Musicbrainz characters.
+    :param s: string to clean up, probably unicode.
+    :return: cleaned-up version of input string.
+    """
+    if not isinstance(s, unicode):
+        u = unicode(s, 'ascii', 'replace')
+    else:
+        u = s
+    u = _translate(u, _XLATE_MUSICBRAINZ)
+    if return_as_string:
+        return u.encode('utf-8')
+    else:
         return u
 
 
@@ -621,26 +666,68 @@ def get_downloaded_track_list(albumpath):
     return downloaded_track_list
 
 
-def preserve_torrent_directory(albumpath):
+def preserve_torrent_directory(albumpath, forced=False, single=False):
     """
-    Copy torrent directory to headphones-modified to keep files for seeding.
+    Copy torrent directory to temp headphones_ directory to keep files for seeding.
     """
     from headphones import logger
-    new_folder = os.path.join(albumpath,
+
-                              'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
+    # Create temp dir
-    logger.info(
+    if headphones.CONFIG.KEEP_TORRENT_FILES_DIR:
-        "Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
+        tempdir = headphones.CONFIG.KEEP_TORRENT_FILES_DIR
+    else:
+        tempdir = tempfile.gettempdir()
+
+    logger.info("Preparing to copy to a temporary directory for post processing: " + albumpath.decode(
+            headphones.SYS_ENCODING, 'replace'))
+
     try:
-        shutil.copytree(albumpath, new_folder)
+        file_name = os.path.basename(os.path.normpath(albumpath))
-        return new_folder
+        if not single:
+            prefix = "headphones_" + file_name + "_@hp@_"
+        else:
+            prefix = "headphones_" + os.path.splitext(file_name)[0] + "_@hp@_"
+        new_folder = tempfile.mkdtemp(prefix=prefix, dir=tempdir)
     except Exception as e:
-        logger.warn("Cannot copy/move files to temp folder: " +
+        logger.error("Cannot create temp directory: " + tempdir.decode(
-                    new_folder.decode(headphones.SYS_ENCODING, 'replace') +
+            headphones.SYS_ENCODING, 'replace') + ". Error: " + str(e))
-                    ". Not continuing. Error: " + str(e))
+        return None
+
+    # Attempt to stop multiple temp dirs being created for the same albumpath
+    if not forced:
+        try:
+            workdir = os.path.join(tempdir, prefix)
+            workdir = re.sub(r'\[', '[[]', workdir)
+            workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
+            if len(glob.glob(workdir + '*/')) >= 3:
+                logger.error(
+                    "Looks like a temp directory has previously been created for this albumpath, not continuing " + workdir.decode(
+                        headphones.SYS_ENCODING, 'replace'))
+                shutil.rmtree(new_folder)
+                return None
+        except Exception as e:
+            logger.warn("Cannot determine if already copied/processed, will copy anyway: Warning: " + str(e))
+
+    # Copy to temp dir
+    try:
+        subdir = os.path.join(new_folder, "headphones")
+        logger.info("Copying files to " + subdir.decode(headphones.SYS_ENCODING, 'replace'))
+        if not single:
+            shutil.copytree(albumpath, subdir)
+        else:
+            os.makedirs(subdir)
+            shutil.copy(albumpath, subdir)
+        # Update the album path with the new location
+        return subdir
+    except Exception as e:
+        logger.warn("Cannot copy/move files to temp directory: " + new_folder.decode(headphones.SYS_ENCODING,
+                                                                                  'replace') + ". Not continuing. Error: " + str(
+            e))
+        shutil.rmtree(new_folder)
         return None
 
 
-def cue_split(albumpath):
+def cue_split(albumpath, keep_original_folder=False):
     """
      Attempts to check and split audio files by a cue for the given directory.
      """
@@ -662,6 +749,15 @@ def cue_split(albumpath):
     # Split cue
     if cue_count and cue_count >= count and cue_dirs:
 
+        # Copy to temp directory
+        if keep_original_folder:
+            temppath = preserve_torrent_directory(albumpath)
+            if temppath:
+                cue_dirs = [cue_dir.replace(albumpath, temppath) for cue_dir in cue_dirs]
+                albumpath = temppath
+            else:
+                return None
+
         from headphones import logger, cuesplit
         logger.info("Attempting to split audio files by cue")
 
@@ -672,12 +768,12 @@ def cue_split(albumpath):
             except Exception as e:
                 os.chdir(cwd)
                 logger.warn("Cue not split: " + str(e))
-                return False
+                return None
 
         os.chdir(cwd)
-        return True
+        return albumpath
 
-    return False
+    return None
 
 
 def extract_logline(s):
@@ -729,9 +825,11 @@ def smartMove(src, dest, delete=True):
 
     source_dir = os.path.dirname(src)
     filename = os.path.basename(src)
+    source_path = os.path.join(source_dir, filename)
+    dest_path = os.path.join(dest, filename)
 
-    if os.path.isfile(os.path.join(dest, filename)):
+    if os.path.isfile(dest_path):
-        logger.info('Destination file exists: %s', os.path.join(dest, filename))
+        logger.info('Destination file exists: %s', dest_path)
         title = os.path.splitext(filename)[0]
         ext = os.path.splitext(filename)[1]
         i = 1
@@ -749,14 +847,28 @@ def smartMove(src, dest, delete=True):
                                 src.decode(headphones.SYS_ENCODING, 'replace'), e)
                 break
 
-    try:
     if delete:
-            shutil.move(os.path.join(source_dir, filename), os.path.join(dest, filename))
+        try:
+            logger.info('Moving "%s" to "%s"', source_path, dest_path)
+            shutil.move(source_path, dest_path)
+        except Exception as e:
+            exists = os.path.exists(dest_path)
+            if exists and os.path.getsize(source_path) == os.path.getsize(dest_path):
+                logger.warn('Successfully moved file "%s", but something went wrong: %s',
+                    filename.decode(headphones.SYS_ENCODING, 'replace'), e)
+                os.unlink(source_path)
             else:
-            shutil.copy(os.path.join(source_dir, filename), os.path.join(dest, filename))
+                # remove faultly copied file
+                if exists:
+                    os.unlink(dest_path)
+                raise
+    else:
+        try:
+            logger.info('Copying "%s" to "%s"', source_path, dest_path)
+            shutil.copy(source_path, dest_path)
             return True
         except Exception as e:
-        logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
+            logger.warn('Error copying file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
                         e)
 
 
@@ -898,3 +1010,24 @@ def create_https_certificates(ssl_cert, ssl_key):
         return False
 
     return True
+
+
+class BeetsLogCapture(beetslogging.Handler):
+
+    def __init__(self):
+        beetslogging.Handler.__init__(self)
+        self.messages = []
+
+    def emit(self, record):
+        self.messages.append(six.text_type(record.msg))
+
+
+@contextmanager
+def capture_beets_log(logger='beets'):
+    capture = BeetsLogCapture()
+    log = beetslogging.getLogger(logger)
+    log.addHandler(capture)
+    try:
+        yield capture.messages
+    finally:
+        log.removeHandler(capture)

headphones/importer.py

@@ -89,11 +89,13 @@ def artistlist_to_mbids(artistlist, forced=False):
             addArtisttoDB(artistid)
 
         # Just update the tracks if it does
-        else:
+
-            havetracks = len(
+        # not sure this is correct and we're updating during scanning in librarysync
-                myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+        # else:
-                myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+        #     havetracks = len(
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
+        #         myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+        #         myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+        #     myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
 
         # Delete it from the New Artists if the request came from there
         if forced:
@@ -105,7 +107,7 @@ def artistlist_to_mbids(artistlist, forced=False):
     try:
         lastfm.getSimilar()
     except Exception as e:
-        logger.warn('Failed to update arist information from Last.fm: %s' % e)
+        logger.warn('Failed to update artist information from Last.fm: %s' % e)
 
 
 def addArtistIDListToDB(artistidlist):
@@ -262,8 +264,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
             else:
                 if check_release_date is None or check_release_date == u"None":
+                    if headphones.CONFIG.MB_IGNORE_AGE_MISSING is not 1:
                         logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
+                    else:
+                        logger.info("[%s] Skipping update of: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                        new_releases = 0
                 else:
                     if len(check_release_date) == 10:
                         release_date = check_release_date
@@ -305,7 +311,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
             # This will be used later to build a hybrid release
             fullreleaselist = []
             # Search for releases within a release group
-            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
+            find_hybrid_releases = myDB.select("SELECT * from allalbums WHERE AlbumID=?",
                                                [rg['id']])
 
             # Build the dictionary for the fullreleaselist
@@ -512,7 +518,10 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
             logger.info(
                 u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+            try:
                 cache.getThumb(AlbumID=rg['id'])
+            except Exception as e:
+                logger.error("Error getting album art: %s", e)
 
             # Start a search for the album if it's new, hasn't been marked as
             # downloaded and autowant_all is selected. This search is deferred,
@@ -528,10 +537,16 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     finalize_update(artistid, artist['artist_name'], errors)
 
     logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
+    try:
         cache.getThumb(ArtistID=artistid)
+    except Exception as e:
+        logger.error("Error getting album art: %s", e)
 
     logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
+    try:
         metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
+    except Exception as e:
+        logger.error("Error getting Metacritic reviews: %s", e)
 
     if errors:
         logger.info(

headphones/lastfm.py

@@ -55,7 +55,7 @@ def request_lastfm(method, **kwargs):
         return
 
     if "error" in data:
-        logger.error("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.FM returned an error: %s", data["message"])
         return
 
     return data
@@ -149,7 +149,10 @@ def getTagTopArtists(tag, limit=50):
         logger.debug("Fetched %d artists from Last.FM", len(artists))
 
         for artist in artists:
+            try:
                 artist_mbid = artist["mbid"]
+            except KeyError:
+                continue
 
             if not any(artist_mbid in x for x in results):
                 artistlist.append(artist_mbid)

headphones/librarysync.py

@@ -50,27 +50,43 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
     logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
 
     if not append:
-        # Clean up bad filepaths
-        tracks = myDB.select(
-            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
 
+        # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
+        if ArtistID:
+            tracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
+                'IS NOT NULL',
+                [ArtistID, ArtistID])
+        else:
+            tracks = myDB.action(
+                'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL')
+
+        locations = []
         for track in tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
+            locations.append(track['Location'])
+        for location in locations:
+            encoded_track_string = location.encode(headphones.SYS_ENCODING, 'replace')
             if not os.path.isfile(encoded_track_string):
                 myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, location])
                 myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
-                            [None, None, None, track['Location']])
+                            [None, None, None, location])
 
-        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
+        if ArtistName:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName])
+        else:
+            del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have')
 
+        locations = []
         for track in del_have_tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
+            locations.append([track['Location'], track['ArtistName']])
+        for location in locations:
+            encoded_track_string = location[0].encode(headphones.SYS_ENCODING, 'replace')
             if not os.path.isfile(encoded_track_string):
-                if track['ArtistName']:
+                if location[1]:
                     # Make sure deleted files get accounted for when updating artist track counts
-                    new_artists.append(track['ArtistName'])
+                    new_artists.append(location[1])
-                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
+                myDB.action('DELETE FROM have WHERE Location=?', [location[0]])
                 logger.info(
                     'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
                         headphones.SYS_ENCODING, 'replace'))
@@ -216,6 +232,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
     song_count = 0
     latest_artist = []
     last_completion_percentage = 0
+    prev_artist_name = None
+    artistid = None
 
     for song in song_list:
 
@@ -237,83 +255,68 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
         # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+        albumid = None
 
-            track = myDB.action(
+        if song['ArtistName'] and song['CleanName']:
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+            artist_name = song['ArtistName']
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            clean_name = song['CleanName']
-            have_updated = False
+
+            # Only update if artist is in the db
+            if artist_name != prev_artist_name:
+                prev_artist_name = artist_name
+                artistid = None
+
+                artist_lookup = "\"" + artist_name.replace("\"", "\"\"") + "\""
+
+                try:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
+                except:
+                    dbartist = None
+                if not dbartist:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name])
+                    if not dbartist:
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name])
+                        if not dbartist:
+                            clean_artist = helpers.clean_name(artist_name)
+                            if clean_artist:
+                                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                                       [clean_artist, clean_artist + '{'])
+                                if not dbartist:
+                                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                                           [clean_artist, clean_artist + '{'])
+
+                if dbartist:
+                    artistid = dbartist[0][0]
+
+            if artistid:
+
+                # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index
+                # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like)
+                # Now selects/updates using CleanName index (may have to revert if not working)
+
+                # matching on CleanName should be enough, ensure it's the same artist just in case
+
+                # Update tracks
+                track = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
                 if track:
-                controlValueDict = {'ArtistName': track['ArtistName'],
+                    albumid = track['AlbumID']
-                                    'AlbumTitle': track['AlbumTitle'],
+                    myDB.action(
-                                    'TrackTitle': track['TrackTitle']}
+                        'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
-                newValueDict = {'Location': song['Location'],
+                        [song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
-                                'BitRate': song['BitRate'],
-                                'Format': song['Format']}
-                myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                controlValueDict2 = {'Location': song['Location']}
+                # Update alltracks
-                newValueDict2 = {'Matched': track['AlbumID']}
+                alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
-                myDB.upsert("have", newValueDict2, controlValueDict2)
-                have_updated = True
-            else:
-                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
-                                    [song['CleanName']]).fetchone()
-                if track:
-                    controlValueDict = {'CleanName': track['CleanName']}
-                    newValueDict = {'Location': song['Location'],
-                                    'BitRate': song['BitRate'],
-                                    'Format': song['Format']}
-                    myDB.upsert("tracks", newValueDict, controlValueDict)
-
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': track['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
-                else:
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': "Failed"}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                    have_updated = True
-
-            alltrack = myDB.action(
-                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
-                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
                 if alltrack:
-                controlValueDict = {'ArtistName': alltrack['ArtistName'],
+                    albumid = alltrack['AlbumID']
-                                    'AlbumTitle': alltrack['AlbumTitle'],
+                    myDB.action(
-                                    'TrackTitle': alltrack['TrackTitle']}
+                        'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
-                newValueDict = {'Location': song['Location'],
+                        [song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
-                                'BitRate': song['BitRate'],
-                                'Format': song['Format']}
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
 
+        # Update have
         controlValueDict2 = {'Location': song['Location']}
-                newValueDict2 = {'Matched': alltrack['AlbumID']}
+        if albumid:
-                myDB.upsert("have", newValueDict2, controlValueDict2)
+            newValueDict2 = {'Matched': albumid}
         else:
-                alltrack = myDB.action(
-                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
-                    [song['CleanName']]).fetchone()
-                if alltrack:
-                    controlValueDict = {'CleanName': alltrack['CleanName']}
-                    newValueDict = {'Location': song['Location'],
-                                    'BitRate': song['BitRate'],
-                                    'Format': song['Format']}
-                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-
-                    controlValueDict2 = {'Location': song['Location']}
-                    newValueDict2 = {'Matched': alltrack['AlbumID']}
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                else:
-                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
-                    if not have_updated:
-                        controlValueDict2 = {'Location': song['Location']}
-                        newValueDict2 = {'Matched': "Failed"}
-                        myDB.upsert("have", newValueDict2, controlValueDict2)
-
-        else:
-            controlValueDict2 = {'Location': song['Location']}
             newValueDict2 = {'Matched': "Failed"}
         myDB.upsert("have", newValueDict2, controlValueDict2)
 
@@ -327,49 +330,110 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
         # Clean up the new artist list
         unique_artists = {}.fromkeys(new_artists).keys()
-        current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
 
-        # There was a bug where artists with special characters (-,') would show up in new artists.
+        # # Don't think we need to do this, check the db instead below
-        artist_list = [
+        #
-            x for x in unique_artists
+        # # artist scan
-            if helpers.clean_name(x).lower() not in [
+        # if ArtistName:
-                helpers.clean_name(y[0]).lower()
+        #     current_artists = [[ArtistName]]
-                for y in current_artists
+        # # directory scan
-                ]
+        # else:
-            ]
+        #     current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL')
-        artists_checked = [
+        #
-            x for x in unique_artists
+        # # There was a bug where artists with special characters (-,') would show up in new artists.
-            if helpers.clean_name(x).lower() in [
+        #
-                helpers.clean_name(y[0]).lower()
+        # # artist_list = scanned artists not in the db
-                for y in current_artists
+        # artist_list = [
-                ]
+        #     x for x in unique_artists
-            ]
+        #     if helpers.clean_name(x).lower() not in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
+        #
+        # # artists_checked = scanned artists that exist in the db
+        # artists_checked = [
+        #     x for x in unique_artists
+        #     if helpers.clean_name(x).lower() in [
+        #         helpers.clean_name(y[0]).lower()
+        #         for y in current_artists
+        #         ]
+        #     ]
+
+        new_artist_list = []
+
+        for artist in unique_artists:
+
+            if not artist:
+                continue
+
+            logger.info('Processing artist: %s' % artist)
+
+            # check if artist is already in the db
+            artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\""
+
+            try:
+                dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
+            except:
+                dbartist = None
+            if not dbartist:
+                clean_artist = helpers.clean_name(artist)
+                if clean_artist:
+                    dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
+                                           [clean_artist, clean_artist + '{'])
+                    if not dbartist:
+                        dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
+                                               [clean_artist, clean_artist + '{'])
+
+            # new artist not in db, add to list
+            if not dbartist:
+                new_artist_list.append(artist)
+            else:
+
+                # artist in db, update have track counts
+                artistid = dbartist[0][0]
 
-        # Update track counts
-        for artist in artists_checked:
                 # Have tracks are selected from tracks table and not all tracks because of duplicates
                 # We update the track count upon an album switch to compliment this
+
+                # havetracks = (
+                #     len(myDB.select(
+                #         'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
+                #         [artist])) + len(myDB.select(
+                #             'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                #             [artist]))
+                # )
+
+                try:
                     havetracks = (
                         len(myDB.select(
-                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
+                            'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
-                    [artist])) + len(myDB.select(
+                            [artistid])) + len(myDB.select(
-                        'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
-                        [artist]))
                     )
+                except Exception as e:
+                    logger.warn('Error updating counts for artist: %s: %s' % (artist, e))
+
                 # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
                 # (can fix by getting rid of second len statement)
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
 
-        logger.info('Found %i new artists' % len(artist_list))
+                if havetracks:
+                    myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
 
-        if artist_list:
+                    # Update albums to downloaded
+                    update_album_status(ArtistID=artistid)
+
+        logger.info('Found %i new artists' % len(new_artist_list))
+
+        # Add scanned artists not in the db
+        if new_artist_list:
             if headphones.CONFIG.AUTO_ADD_ARTISTS:
-                logger.info('Importing %i new artists' % len(artist_list))
+                logger.info('Importing %i new artists' % len(new_artist_list))
-                importer.artistlist_to_mbids(artist_list)
+                importer.artistlist_to_mbids(new_artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
                 # myDB.action('DELETE from newartists')
-                for artist in artist_list:
+                for artist in new_artist_list:
                     myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
 
         if headphones.CONFIG.DETECT_BITRATE and bitrates:
@@ -379,51 +443,79 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
 
+        artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\""
+        try:
             havetracks = len(
-            myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
+                myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
                             [ArtistID])) + len(myDB.select(
-                            'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
-                            [ArtistName]))
+        except Exception as e:
+            logger.warn('Error updating counts for artist: %s: %s' % (ArtistName, e))
+
+        if havetracks:
             myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
 
-    if not append:
+    # Moved above to call for each artist
-        update_album_status()
+    # if not append:
+    #     update_album_status()
 
     if not append and not artistScan:
         lastfm.getSimilar()
 
+    if ArtistName:
+        logger.info('Scanning complete for artist: %s', ArtistName)
+    else:
         logger.info('Library scan complete')
 
 
 # ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
 
+# Think the above comment relates to calling from Manage Unmatched
 
-def update_album_status(AlbumID=None):
+# This used to select and update all albums and would clobber the db, changed to run by ArtistID.
+
+def update_album_status(AlbumID=None, ArtistID=None):
     myDB = db.DBConnection()
-    logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+    # logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+
     if AlbumID:
         album_status_updater = myDB.action(
-            'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+            'SELECT'
+            ' a.AlbumID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
+            'FROM'
+            ' albums AS a '
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
+            'WHERE'
+            ' a.AlbumID = ? AND a.Status != "Downloaded" '
+            'GROUP BY'
+            ' a.AlbumID '
+            'HAVING'
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
+            [AlbumID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
+        )
     else:
-        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
+        album_status_updater = myDB.action(
-    for album in album_status_updater:
+            'SELECT'
-        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?',
+            ' a.AlbumID, a.ArtistID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
-                                    [album['AlbumID']])
+            'FROM'
-        total_tracks = 0
+            ' albums AS a '
-        have_tracks = 0
+            'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
-        for track in track_counter:
+            'WHERE'
-            total_tracks += 1
+            ' a.ArtistID = ? AND a.Status != "Downloaded" '
-            if track['Location']:
+            'GROUP BY'
-                have_tracks += 1
+            ' a.AlbumID '
-        if total_tracks != 0:
+            'HAVING'
-            album_completion = float(float(have_tracks) / float(total_tracks)) * 100
+            ' AVG(t.Location IS NOT NULL) * 100 >= ?',
-        else:
+            [ArtistID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
-            album_completion = 0
+        )
-            logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
 
-        if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT:
     new_album_status = "Downloaded"
 
+    albums = []
+    for album in album_status_updater:
+        albums.append([album['AlbumID'], album['ArtistName'], album['AlbumTitle']])
+    for album in albums:
+
         # I don't think we want to change Downloaded->Skipped.....
         # I think we can only automatically change Skipped->Downloaded when updating
         # There was a bug report where this was causing infinite downloads if the album was
@@ -433,10 +525,13 @@ def update_album_status(AlbumID=None):
         #        new_album_status = "Skipped"
         #    else:
         #        new_album_status = album['Status']
-        else:
+        #     else:
-            new_album_status = album['Status']
+        #         new_album_status = album['Status']
+        #
+        #     myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
+        #     if new_album_status != album['Status']:
+        #         logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
+        # logger.info('Album status update complete')
 
-        myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
+        myDB.action('UPDATE albums SET Status = ? WHERE AlbumID = ?', [new_album_status, album[0]])
-        if new_album_status != album['Status']:
+        logger.info('Album: %s - %s. Status updated to %s' % (album[1], album[2], new_album_status))
-            logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
-    logger.info('Album status update complete')

headphones/mb.py

@@ -770,3 +770,50 @@ def findAlbumID(artist=None, album=None):
         return False
     rgid = unicode(results[0]['id'])
     return rgid
+
+
+def getArtistForReleaseGroup(rgid):
+    """
+    Returns artist name for a release group
+    Used for series where we store the series instead of the artist
+    """
+    releaseGroup = None
+    try:
+        with mb_lock:
+            releaseGroup = musicbrainzngs.get_release_group_by_id(
+                rgid, ["artists"])
+            releaseGroup = releaseGroup['release-group']
+    except musicbrainzngs.WebServiceError as e:
+        logger.warn(
+            'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (
+                rgid, str(e)))
+        mb_lock.snooze(5)
+
+    if not releaseGroup:
+        return False
+    else:
+        return releaseGroup['artist-credit'][0]['artist']['name']
+
+
+def getArtistRelationships(artistid):
+    """
+    Returns list of relationship urls. e.g. Discogs, Wikipedia etc.
+    """
+    urls = []
+    artist = None
+    try:
+        with mb_lock:
+            info = musicbrainzngs.get_artist_by_id(artistid, includes='url-rels')
+    except musicbrainzngs.WebServiceError as e:
+        logger.warn(
+            'Attempt to query MusicBrainz for %s failed "%s"' % (artistid, str(e)))
+        mb_lock.snooze(5)
+    if 'artist' in info:
+        artist = info['artist']
+        if 'url-relation-list' in artist:
+            for l in artist['url-relation-list']:
+                urls.append({
+                    'type': l['type'],
+                    'url': l['target']
+                })
+    return urls

headphones/metacritic.py

@@ -27,8 +27,9 @@ def update(artistid, artist_name, release_groups):
     # cut down on api calls. If it's ineffective then we'll switch to search
 
     replacements = {" & ": " ", ".": ""}
+    mc_artist_name = helpers.clean_musicbrainz_name(artist_name, return_as_string=False)
+    mc_artist_name = mc_artist_name.replace("'", " ")
     mc_artist_name = helpers.replace_all(artist_name.lower(), replacements)
-
     mc_artist_name = mc_artist_name.replace(" ", "-")
 
     headers = {
@@ -36,7 +37,7 @@ def update(artistid, artist_name, release_groups):
 
     url = "http://www.metacritic.com/person/" + mc_artist_name + "?filter-options=music&sort_options=date&num_items=100"
 
-    res = request.request_soup(url, headers=headers)
+    res = request.request_soup(url, headers=headers, whitelist_status_code=404)
 
     rows = None
 

headphones/metadata.py

@@ -289,6 +289,12 @@ def album_metadata(path, release, common_tags):
         first_char = sort_name[0]
 
     orig_folder = u''
+
+    # Get from temp path
+    if "_@hp@_" in path:
+        orig_folder = path.rsplit("headphones_", 1)[1].split("_@hp@_")[0]
+        orig_folder = orig_folder.decode(headphones.SYS_ENCODING, 'replace')
+    else:
         for r, d, f in os.walk(path):
             try:
                 orig_folder = os.path.basename(

headphones/metadata_test.py

@@ -168,8 +168,8 @@ class MetadataTest(TestCase):
         if not _h.SYS_ENCODING:
             _h.SYS_ENCODING = 'UTF-8'
 
-        res = _hp.replace_all(
+        res = _hp.pattern_substitute(
             "/music/$First/$Artist/$Artist - $Album{ [$Year]}", md, True)
 
         self.assertEqual(res, u"/music/A/artist/artist - Album",
-                         "check correct rendering of None via replace_all()")
+                         "check correct rendering of None via pattern_substitute()")

headphones/notifiers.py

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from urllib import urlencode
+from urllib import urlencode, quote_plus
 import urllib
 import subprocess
 import json
@@ -23,6 +23,7 @@ import email.utils
 from httplib import HTTPSConnection
 from urlparse import parse_qsl
 import urllib2
+import requests as requests
 
 import os.path
 from headphones import logger, helpers, common, request
@@ -148,7 +149,9 @@ class PROWL(object):
 
         http_handler.request("POST",
                              "/publicapi/add",
-                             headers={'Content-type': "application/x-www-form-urlencoded"},
+                             headers={
+                                 'Content-type':
+                                     "application/x-www-form-urlencoded"},
                              body=urlencode(data))
         response = http_handler.getresponse()
         request_status = response.status
@@ -203,28 +206,34 @@ class XBMC(object):
         url = host + '/xbmcCmds/xbmcHttp/?' + url_command
 
         if self.password:
-            return request.request_content(url, auth=(self.username, self.password))
+            return request.request_content(url,
+                                           auth=(self.username, self.password))
         else:
             return request.request_content(url)
 
     def _sendjson(self, host, method, params={}):
-        data = [{'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
+        data = [
+            {'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
         headers = {'Content-Type': 'application/json'}
         url = host + '/jsonrpc'
 
         if self.password:
-            response = request.request_json(url, method="post", data=json.dumps(data),
+            response = request.request_json(
-                                            headers=headers, auth=(self.username, self.password))
+                url, method="post",
+                data=json.dumps(data),
+                headers=headers, auth=(
+                    self.username, self.password))
         else:
-            response = request.request_json(url, method="post", data=json.dumps(data),
+            response = request.request_json(url, method="post",
+                                            data=json.dumps(data),
                                             headers=headers)
 
         if response:
             return response[0]['result']
 
     def update(self):
-        # From what I read you can't update the music library on a per directory or per path basis
+        # From what I read you can't update the music library on a per
-        # so need to update the whole thing
+        # directory or per path basis so need to update the whole thing
 
         hosts = [x.strip() for x in self.hosts.split(',')]
 
@@ -247,18 +256,23 @@ class XBMC(object):
             logger.info('Sending notification command to XMBC @ ' + host)
             try:
                 version = self._sendjson(host, 'Application.GetProperties',
-                                         {'properties': ['version']})['version']['major']
+                                         {'properties': ['version']})[
+                    'version']['major']
 
                 if version < 12:  # Eden
-                    notification = header + "," + message + "," + time + "," + albumartpath
+                    notification = header + "," + message + "," + time + \
+                                   "," + albumartpath
                     notifycommand = {'command': 'ExecBuiltIn',
-                                     'parameter': 'Notification(' + notification + ')'}
+                                     'parameter': 'Notification(' +
+                                                  notification + ')'}
                     request = self._sendhttp(host, notifycommand)
 
                 else:  # Frodo
-                    params = {'title': header, 'message': message, 'displaytime': int(time),
+                    params = {'title': header, 'message': message,
+                              'displaytime': int(time),
                               'image': albumartpath}
-                    request = self._sendjson(host, 'GUI.ShowNotification', params)
+                    request = self._sendjson(host, 'GUI.ShowNotification',
+                                             params)
 
                 if not request:
                     raise Exception
@@ -323,22 +337,28 @@ class Plex(object):
         url = host + '/xbmcCmds/xbmcHttp/?' + command
 
         if self.password:
-            response = request.request_response(url, auth=(self.username, self.password))
+            response = request.request_response(url, auth=(
+                self.username, self.password))
         else:
             response = request.request_response(url)
 
         return response
 
     def _sendjson(self, host, method, params={}):
-        data = [{'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
+        data = [
+            {'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
         headers = {'Content-Type': 'application/json'}
         url = host + '/jsonrpc'
 
         if self.password:
-            response = request.request_json(url, method="post", data=json.dumps(data),
+            response = request.request_json(
-                                            headers=headers, auth=(self.username, self.password))
+                url, method="post",
+                data=json.dumps(data),
+                headers=headers, auth=(
+                    self.username, self.password))
         else:
-            response = request.request_json(url, method="post", data=json.dumps(data),
+            response = request.request_json(url, method="post",
+                                            data=json.dumps(data),
                                             headers=headers)
 
         if response:
@@ -346,20 +366,60 @@ class Plex(object):
 
     def update(self):
 
-        # From what I read you can't update the music library on a per directory or per path basis
+        # Get token from user credentials
-        # so need to update the whole thing
+        if not self.token:
+            loginpage = 'https://plex.tv/users/sign_in.json'
+            post_params = {
+                'user[login]': self.username,
+                'user[password]': self.password
+            }
+            headers = {
+                'X-Plex-Device-Name': 'Headphones',
+                'X-Plex-Product': 'Headphones',
+                'X-Plex-Client-Identifier': common.USER_AGENT,
+                'X-Plex-Version': ''
+            }
+
+            logger.info("Getting plex.tv credentials for user %s", self.username)
+
+            try:
+                r = requests.post(loginpage, data=post_params, headers=headers)
+                r.raise_for_status()
+            except requests.RequestException as e:
+                logger.error("Error getting plex.tv credentials, check settings: %s", e)
+                return False
+
+            try:
+                data = r.json()
+            except ValueError as e:
+                logger.error("Error getting plex.tv credentials: %s", e)
+                return False
+
+            try:
+                self.token = data['user']['authentication_token']
+            except KeyError as e:
+                logger.error("Error getting plex.tv credentials: %s", e)
+                return False
+
+        # From what I read you can't update the music library on a per
+        # directory or per path basis so need to update the whole thing
 
         hosts = [x.strip() for x in self.server_hosts.split(',')]
 
         for host in hosts:
-            logger.info('Sending library update command to Plex Media Server@ ' + host)
+            logger.info(
+                'Sending library update command to Plex Media Server@ ' + host)
             url = "%s/library/sections" % host
             if self.token:
                 params = {'X-Plex-Token': self.token}
             else:
                 params = False
 
+            try:
                 r = request.request_minidom(url, params=params)
+                if not r:
+                    logger.warn("Error getting Plex Media Server details, check settings (possibly incorrect token)")
+                    return False
 
                 sections = r.getElementsByTagName('Directory')
 
@@ -369,9 +429,14 @@ class Plex(object):
 
                 for s in sections:
                     if s.getAttribute('type') == "artist":
-                    url = "%s/library/sections/%s/refresh" % (host, s.getAttribute('key'))
+                        url = "%s/library/sections/%s/refresh" % (
+                            host, s.getAttribute('key'))
                         request.request_response(url, params=params)
 
+            except Exception as e:
+                logger.error("Error getting Plex Media Server details: %s" % e)
+                return False
+
     def notify(self, artist, album, albumartpath):
 
         hosts = [x.strip() for x in self.client_hosts.split(',')]
@@ -381,27 +446,35 @@ class Plex(object):
         time = "3000"  # in ms
 
         for host in hosts:
-            logger.info('Sending notification command to Plex client @ ' + host)
+            logger.info(
+                'Sending notification command to Plex client @ ' + host)
             try:
                 version = self._sendjson(host, 'Application.GetProperties',
-                                         {'properties': ['version']})['version']['major']
+                                         {'properties': ['version']})[
+                    'version']['major']
 
                 if version < 12:  # Eden
-                    notification = header + "," + message + "," + time + "," + albumartpath
+                    notification = header + "," + message + "," + time + \
+                                   "," + albumartpath
                     notifycommand = {'command': 'ExecBuiltIn',
-                                     'parameter': 'Notification(' + notification + ')'}
+                                     'parameter': 'Notification(' +
+                                                  notification + ')'}
                     request = self._sendhttp(host, notifycommand)
 
                 else:  # Frodo
-                    params = {'title': header, 'message': message, 'displaytime': int(time),
+                    params = {'title': header, 'message': message,
+                              'displaytime': int(time),
                               'image': albumartpath}
-                    request = self._sendjson(host, 'GUI.ShowNotification', params)
+                    request = self._sendjson(host, 'GUI.ShowNotification',
+                                             params)
 
                 if not request:
                     raise Exception
 
             except Exception:
-                logger.error('Error sending notification request to Plex client @ ' + host)
+                logger.error(
+                    'Error sending notification request to Plex client @ ' +
+                    host)
 
 
 class NMA(object):
@@ -419,7 +492,8 @@ class NMA(object):
             message = "Headphones has snatched: " + snatched
         else:
             event = artist + ' - ' + album + ' complete!'
-            message = "Headphones has downloaded and postprocessed: " + artist + ' [' + album + ']'
+            message = "Headphones has downloaded and postprocessed: " + \
+                      artist + ' [' + album + ']'
 
         logger.debug(u"NMA event: " + event)
         logger.debug(u"NMA message: " + message)
@@ -433,7 +507,8 @@ class NMA(object):
         if len(keys) > 1:
             batch = True
 
-        response = p.push(title, event, message, priority=nma_priority, batch_mode=batch)
+        response = p.push(title, event, message, priority=nma_priority,
+                          batch_mode=batch)
 
         if not response[api][u'code'] == u'200':
             logger.error(u'Could not send notification to NotifyMyAndroid')
@@ -461,9 +536,11 @@ class PUSHBULLET(object):
             data['device_iden'] = self.deviceid
 
         headers = {'Content-type': "application/json",
-                   'Authorization': 'Bearer ' + headphones.CONFIG.PUSHBULLET_APIKEY}
+                   'Authorization': 'Bearer ' +
+                                    headphones.CONFIG.PUSHBULLET_APIKEY}
 
-        response = request.request_json(url, method="post", headers=headers, data=json.dumps(data))
+        response = request.request_json(url, method="post", headers=headers,
+                                        data=json.dumps(data))
 
         if response:
             logger.info(u"PushBullet notifications sent.")
@@ -492,7 +569,9 @@ class PUSHALOT(object):
 
         http_handler.request("POST",
                              "/api/sendmessage",
-                             headers={'Content-type': "application/x-www-form-urlencoded"},
+                             headers={
+                                 'Content-type':
+                                     "application/x-www-form-urlencoded"},
                              body=urlencode(data))
         response = http_handler.getresponse()
         request_status = response.status
@@ -512,6 +591,49 @@ class PUSHALOT(object):
             return False
 
 
+class JOIN(object):
+    def __init__(self):
+
+        self.enabled = headphones.CONFIG.JOIN_ENABLED
+        self.apikey = headphones.CONFIG.JOIN_APIKEY
+        self.deviceid = headphones.CONFIG.JOIN_DEVICEID
+        self.url = 'https://joinjoaomgcd.appspot.com/_ah/' \
+                   'api/messaging/v1/sendPush?apikey={apikey}' \
+                   '&title={title}&text={text}' \
+                   '&icon={icon}'
+
+    def notify(self, message, event):
+        if not headphones.CONFIG.JOIN_ENABLED or \
+                not headphones.CONFIG.JOIN_APIKEY:
+            return
+
+        icon = "https://cdn.rawgit.com/Headphones/" \
+               "headphones/develop/data/images/headphoneslogo.png"
+
+        if not self.deviceid:
+            self.deviceid = "group.all"
+        l = [x.strip() for x in self.deviceid.split(',')]
+        if len(l) > 1:
+            self.url += '&deviceIds={deviceid}'
+        else:
+            self.url += '&deviceId={deviceid}'
+
+        response = urllib2.urlopen(self.url.format(apikey=self.apikey,
+                                                   title=quote_plus(event),
+                                                   text=quote_plus(
+                                                       message.encode(
+                                                           "utf-8")),
+                                                   icon=icon,
+                                                   deviceid=self.deviceid))
+
+        if response:
+            logger.info(u"Join notifications sent.")
+            return True
+        else:
+            logger.error(u"Join notification failed.")
+            return False
+
+
 class Synoindex(object):
     def __init__(self, util_loc='/usr/syno/bin/synoindex'):
         self.util_loc = util_loc
@@ -524,7 +646,8 @@ class Synoindex(object):
 
         if not self.util_exists():
             logger.warn(
-                "Error sending notification: synoindex utility not found at %s" % self.util_loc)
+                "Error sending notification: synoindex utility "
+                "not found at %s" % self.util_loc)
             return
 
         if os.path.isfile(path):
@@ -533,16 +656,19 @@ class Synoindex(object):
             cmd_arg = '-A'
         else:
             logger.warn(
-                "Error sending notification: Path passed to synoindex was not a file or folder.")
+                "Error sending notification: Path passed to synoindex "
+                "was not a file or folder.")
             return
 
         cmd = [self.util_loc, cmd_arg, path]
         logger.info("Calling synoindex command: %s" % str(cmd))
         try:
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT,
                                  cwd=headphones.PROG_DIR)
             out, error = p.communicate()
-            # synoindex never returns any codes other than '0', highly irritating
+            # synoindex never returns any codes other than '0',
+            #  highly irritating
         except OSError, e:
             logger.warn("Error sending notification: %s" % str(e))
 
@@ -580,7 +706,8 @@ class PUSHOVER(object):
 
         headers = {'Content-type': "application/x-www-form-urlencoded"}
 
-        response = request.request_response(url, method="POST", headers=headers, data=data)
+        response = request.request_response(url, method="POST",
+                                            headers=headers, data=data)
 
         if response:
             logger.info(u"Pushover notifications sent.")
@@ -614,20 +741,25 @@ class TwitterNotifier(object):
     def notify_snatch(self, title):
         if headphones.CONFIG.TWITTER_ONSNATCH:
             self._notifyTwitter(
-                common.notifyStrings[common.NOTIFY_SNATCH] + ': ' + title + ' at ' + helpers.now())
+                common.notifyStrings[
+                    common.NOTIFY_SNATCH] + ': ' + title + ' at ' +
+                helpers.now())
 
     def notify_download(self, title):
         if headphones.CONFIG.TWITTER_ENABLED:
             self._notifyTwitter(common.notifyStrings[
-                                    common.NOTIFY_DOWNLOAD] + ': ' + title + ' at ' + helpers.now())
+                                    common.NOTIFY_DOWNLOAD] + ': ' +
+                                title + ' at ' + helpers.now())
 
     def test_notify(self):
         return self._notifyTwitter(
-            "This is a test notification from Headphones at " + helpers.now(), force=True)
+            "This is a test notification from Headphones at " + helpers.now(),
+            force=True)
 
     def _get_authorization(self):
 
-        oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret)
+        oauth_consumer = oauth.Consumer(key=self.consumer_key,
+                                        secret=self.consumer_secret)
         oauth_client = oauth.Client(oauth_consumer)
 
         logger.info('Requesting temp token from Twitter')
@@ -635,32 +767,42 @@ class TwitterNotifier(object):
         resp, content = oauth_client.request(self.REQUEST_TOKEN_URL, 'GET')
 
         if resp['status'] != '200':
-            logger.info('Invalid respond from Twitter requesting temp token: %s' % resp['status'])
+            logger.info(
+                'Invalid respond from Twitter requesting temp token: %s' %
+                resp['status'])
         else:
             request_token = dict(parse_qsl(content))
 
             headphones.CONFIG.TWITTER_USERNAME = request_token['oauth_token']
-            headphones.CONFIG.TWITTER_PASSWORD = request_token['oauth_token_secret']
+            headphones.CONFIG.TWITTER_PASSWORD = request_token[
+                'oauth_token_secret']
 
-            return self.AUTHORIZATION_URL + "?oauth_token=" + request_token['oauth_token']
+            return self.AUTHORIZATION_URL + "?oauth_token=" + request_token[
+                'oauth_token']
 
     def _get_credentials(self, key):
         request_token = {}
 
         request_token['oauth_token'] = headphones.CONFIG.TWITTER_USERNAME
-        request_token['oauth_token_secret'] = headphones.CONFIG.TWITTER_PASSWORD
+        request_token[
+            'oauth_token_secret'] = headphones.CONFIG.TWITTER_PASSWORD
         request_token['oauth_callback_confirmed'] = 'true'
 
-        token = oauth.Token(request_token['oauth_token'], request_token['oauth_token_secret'])
+        token = oauth.Token(request_token['oauth_token'],
+                            request_token['oauth_token_secret'])
         token.set_verifier(key)
 
-        logger.info('Generating and signing request for an access token using key ' + key)
+        logger.info(
+            'Generating and signing request for an access token using key ' +
+            key)
 
-        oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret)
+        oauth_consumer = oauth.Consumer(key=self.consumer_key,
+                                        secret=self.consumer_secret)
         logger.info('oauth_consumer: ' + str(oauth_consumer))
         oauth_client = oauth.Client(oauth_consumer, token)
         logger.info('oauth_client: ' + str(oauth_client))
-        resp, content = oauth_client.request(self.ACCESS_TOKEN_URL, method='POST',
+        resp, content = oauth_client.request(self.ACCESS_TOKEN_URL,
+                                             method='POST',
                                              body='oauth_verifier=%s' % key)
         logger.info('resp, content: ' + str(resp) + ',' + str(content))
 
@@ -669,14 +811,18 @@ class TwitterNotifier(object):
 
         logger.info('resp[status] = ' + str(resp['status']))
         if resp['status'] != '200':
-            logger.info('The request for a token with did not succeed: ' + str(resp['status']),
+            logger.info('The request for a token with did not succeed: ' + str(
+                resp['status']),
                         logger.ERROR)
             return False
         else:
-            logger.info('Your Twitter Access Token key: %s' % access_token['oauth_token'])
+            logger.info('Your Twitter Access Token key: %s' % access_token[
-            logger.info('Access Token secret: %s' % access_token['oauth_token_secret'])
+                'oauth_token'])
+            logger.info(
+                'Access Token secret: %s' % access_token['oauth_token_secret'])
             headphones.CONFIG.TWITTER_USERNAME = access_token['oauth_token']
-            headphones.CONFIG.TWITTER_PASSWORD = access_token['oauth_token_secret']
+            headphones.CONFIG.TWITTER_PASSWORD = access_token[
+                'oauth_token_secret']
             return True
 
     def _send_tweet(self, message=None):
@@ -688,7 +834,8 @@ class TwitterNotifier(object):
 
         logger.info(u"Sending tweet: " + message)
 
-        api = twitter.Api(username, password, access_token_key, access_token_secret)
+        api = twitter.Api(username, password, access_token_key,
+                          access_token_secret)
 
         try:
             api.PostUpdate(message)
@@ -714,7 +861,7 @@ class OSX_NOTIFY(object):
             self.AppKit = __import__("AppKit")
         except:
             logger.warn('OS X Notification: Cannot import objc or AppKit')
-            return False
+            pass
 
     def swizzle(self, cls, SEL, func):
         old_IMP = getattr(cls, SEL, None)
@@ -741,7 +888,8 @@ class OSX_NOTIFY(object):
             )
 
             NSUserNotification = self.objc.lookUpClass('NSUserNotification')
-            NSUserNotificationCenter = self.objc.lookUpClass('NSUserNotificationCenter')
+            NSUserNotificationCenter = self.objc.lookUpClass(
+                'NSUserNotificationCenter')
             NSAutoreleasePool = self.objc.lookUpClass('NSAutoreleasePool')
 
             if not NSUserNotification or not NSUserNotificationCenter:
@@ -756,14 +904,17 @@ class OSX_NOTIFY(object):
             if text:
                 notification.setInformativeText_(text)
             if sound:
-                notification.setSoundName_("NSUserNotificationDefaultSoundName")
+                notification.setSoundName_(
+                    "NSUserNotificationDefaultSoundName")
             if image:
-                source_img = self.AppKit.NSImage.alloc().initByReferencingFile_(image)
+                source_img = self.AppKit.NSImage.alloc().\
+                    initByReferencingFile_(image)
                 notification.setContentImage_(source_img)
                 # notification.set_identityImage_(source_img)
             notification.setHasActionButton_(False)
 
-            notification_center = NSUserNotificationCenter.defaultUserNotificationCenter()
+            notification_center = NSUserNotificationCenter.\
+                defaultUserNotificationCenter()
             notification_center.deliverNotification_(notification)
 
             del pool
@@ -784,13 +935,16 @@ class BOXCAR(object):
     def notify(self, title, message, rgid=None):
         try:
             if rgid:
-                message += '<br></br><a href="http://musicbrainz.org/release-group/%s">MusicBrainz</a>' % rgid
+                message += '<br></br><a href="http://musicbrainz.org/' \
+                           'release-group/%s">MusicBrainz</a>' % rgid
 
             data = urllib.urlencode({
                 'user_credentials': headphones.CONFIG.BOXCAR_TOKEN,
                 'notification[title]': title.encode('utf-8'),
                 'notification[long_message]': message.encode('utf-8'),
-                'notification[sound]': "done"
+                'notification[sound]': "done",
+                'notification[icon_url]': "https://raw.githubusercontent.com/rembo10/headphones/master/data/images"
+                                          "/headphoneslogo.png"
             })
 
             req = urllib2.Request(self.url)
@@ -818,7 +972,8 @@ class SubSonicNotifier(object):
             self.host = self.host + "/"
 
         # Invoke request
-        request.request_response(self.host + "musicFolderSettings.view?scanNow",
+        request.request_response(
+            self.host + "musicFolderSettings.view?scanNow",
             auth=(self.username, self.password))
 
 
@@ -827,12 +982,15 @@ class Email(object):
 
         message = MIMEText(message, 'plain', "utf-8")
         message['Subject'] = subject
-        message['From'] = email.utils.formataddr(('Headphones', headphones.CONFIG.EMAIL_FROM))
+        message['From'] = email.utils.formataddr(
+            ('Headphones', headphones.CONFIG.EMAIL_FROM))
         message['To'] = headphones.CONFIG.EMAIL_TO
+        message['Date'] = email.utils.formatdate(localtime=True)
 
         try:
             if headphones.CONFIG.EMAIL_SSL:
-                mailserver = smtplib.SMTP_SSL(headphones.CONFIG.EMAIL_SMTP_SERVER,
+                mailserver = smtplib.SMTP_SSL(
+                    headphones.CONFIG.EMAIL_SMTP_SERVER,
                     headphones.CONFIG.EMAIL_SMTP_PORT)
             else:
                 mailserver = smtplib.SMTP(headphones.CONFIG.EMAIL_SMTP_SERVER,
@@ -847,7 +1005,8 @@ class Email(object):
                 mailserver.login(headphones.CONFIG.EMAIL_SMTP_USER,
                                  headphones.CONFIG.EMAIL_SMTP_PASSWORD)
 
-            mailserver.sendmail(headphones.CONFIG.EMAIL_FROM, headphones.CONFIG.EMAIL_TO,
+            mailserver.sendmail(headphones.CONFIG.EMAIL_FROM,
+                                headphones.CONFIG.EMAIL_TO,
                                 message.as_string())
             mailserver.quit()
             return True
@@ -858,8 +1017,7 @@ class Email(object):
 
 
 class TELEGRAM(object):
-
+    def notify(self, message, status, rgid=None, image=None):
-    def notify(self, message, status):
         if not headphones.CONFIG.TELEGRAM_ENABLED:
             return
 
@@ -872,9 +1030,25 @@ class TELEGRAM(object):
         userid = headphones.CONFIG.TELEGRAM_USERID
 
         # Construct message
-        payload = {'chat_id': userid, 'text': status + ': ' + message}
+        message = '\n\n' + message
 
-        # Send message to user using Telegram's Bot API
+        # MusicBrainz link
+        if rgid:
+            message += '\n\n <a href="http://musicbrainz.org/' \
+                      'release-group/%s">MusicBrainz</a>' % rgid
+
+        # Send image
+        response = None
+        if image:
+            image_file = {'photo': (image, open(image, "rb"))}
+            payload = {'chat_id': userid, 'parse_mode': "HTML", 'caption': status + message}
+            try:
+                response = requests.post(TELEGRAM_API % (token, "sendPhoto"), data=payload, files=image_file)
+            except Exception, e:
+                logger.info(u'Telegram notify failed: ' + str(e))
+        # Sent text
+        else:
+            payload = {'chat_id': userid, 'parse_mode': "HTML", 'text': status + message}
             try:
                 response = requests.post(TELEGRAM_API % (token, "sendMessage"), data=payload)
             except Exception, e:
@@ -882,9 +1056,39 @@ class TELEGRAM(object):
 
         # Error logging
         sent_successfuly = True
-        if not response.status_code == 200:
+        if response and not response.status_code == 200:
-            logger.info(u'Could not send notification to TelegramBot (token=%s). Response: [%s]', (token, response.text))
+            logger.info("Could not send notification to TelegramBot (token=%s). Response: [%s]", token, response.text)
             sent_successfuly = False
 
         logger.info(u"Telegram notifications sent.")
         return sent_successfuly
+
+
+class SLACK(object):
+    def notify(self, message, status):
+        if not headphones.CONFIG.SLACK_ENABLED:
+            return
+
+        import requests
+
+        SLACK_URL = headphones.CONFIG.SLACK_URL
+        channel = headphones.CONFIG.SLACK_CHANNEL
+        emoji = headphones.CONFIG.SLACK_EMOJI
+
+        payload = {'channel': channel, 'text': status + ': ' + message,
+                   'icon_emoji': emoji}
+
+        try:
+            response = requests.post(SLACK_URL, json=payload)
+        except Exception, e:
+            logger.info(u'Slack notify failed: ' + str(e))
+
+        sent_successfuly = True
+        if not response.status_code == 200:
+            logger.info(
+                u'Could not send notification to Slack. Response: [%s]',
+                (response.text))
+            sent_successfuly = False
+
+        logger.info(u"Slack notifications sent.")
+        return sent_successfuly

headphones/postprocessor.py

@@ -17,7 +17,6 @@ import shutil
 import uuid
 import threading
 import itertools
-import tempfile
 
 import os
 import re
@@ -25,11 +24,12 @@ import beets
 import headphones
 from beets import autotag
 from beets import config as beetsconfig
+from beets import logging as beetslogging
 from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
 from beetsplug import lyrics as beetslyrics
-from headphones import notifiers, utorrent, transmission, deluge
+from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
 from headphones import db, albumart, librarysync
-from headphones import logger, helpers, request, mb, music_encoder
+from headphones import logger, helpers, mb, music_encoder
 from headphones import metadata
 
 postprocessor_lock = threading.Lock()
@@ -44,6 +44,8 @@ def checkFolder():
 
         for album in snatched:
             if album['FolderName']:
+                folder_name = album['FolderName']
+                single = False
                 if album['Kind'] == 'nzb':
                     download_dir = headphones.CONFIG.DOWNLOAD_DIR
                 else:
@@ -52,21 +54,32 @@ def checkFolder():
                     else:
                         download_dir = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
 
-                album_path = os.path.join(download_dir, album['FolderName']).encode(
+                    # Get folder from torrent hash
+                    if album['TorrentHash'] and headphones.CONFIG.TORRENT_DOWNLOADER:
+                        torrent_folder_name = None
+                        if headphones.CONFIG.TORRENT_DOWNLOADER == 1:
+                            torrent_folder_name, single = transmission.getFolder(album['TorrentHash'])
+                        elif headphones.CONFIG.TORRENT_DOWNLOADER == 4:
+                            torrent_folder_name, single = qbittorrent.getFolder(album['TorrentHash'])
+                        if torrent_folder_name:
+                            folder_name = torrent_folder_name
+
+                if folder_name:
+                    album_path = os.path.join(download_dir, folder_name).encode(
                         headphones.SYS_ENCODING, 'replace')
                     logger.debug("Checking if %s exists" % album_path)
 
                     if os.path.exists(album_path):
-                    logger.info('Found "' + album['FolderName'] + '" in ' + album[
+                        logger.info('Found "' + folder_name + '" in ' + album[
                             'Kind'] + ' download folder. Verifying....')
-                    verify(album['AlbumID'], album_path, album['Kind'])
+                        verify(album['AlbumID'], album_path, album['Kind'], single=single)
             else:
                 logger.info("No folder name found for " + album['Title'])
 
     logger.debug("Checking download folder finished.")
 
 
-def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=False):
+def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=False, single=False):
     myDB = db.DBConnection()
     release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
     tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
@@ -170,13 +183,17 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
             controlValueDict = {"TrackID": track['id'],
                                 "AlbumID": albumid}
 
+            clean_name = helpers.clean_name(
+                release_dict['artist_name'] + ' ' + release_dict['title'] + ' ' + track['title'])
+
             newValueDict = {"ArtistID": release_dict['artist_id'],
                             "ArtistName": release_dict['artist_name'],
                             "AlbumTitle": release_dict['title'],
                             "AlbumASIN": release_dict['asin'],
                             "TrackTitle": track['title'],
                             "TrackDuration": track['duration'],
-                            "TrackNumber": track['number']
+                            "TrackNumber": track['number'],
+                            "CleanName": clean_name
                             }
 
             myDB.upsert("tracks", newValueDict, controlValueDict)
@@ -207,13 +224,38 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
                                                                        'replace') + " isn't complete yet. Will try again on the next run")
                 return
 
-    # Split cue
+    # Force single file through
+    if single and not downloaded_track_list:
+        downloaded_track_list.append(albumpath)
+
+    # Check to see if we're preserving the torrent dir
+    if (headphones.CONFIG.KEEP_TORRENT_FILES and Kind == "torrent") or headphones.CONFIG.KEEP_ORIGINAL_FOLDER:
+        keep_original_folder = True
+
+    # Split cue before metadata check
     if headphones.CONFIG.CUE_SPLIT and downloaded_cuecount and downloaded_cuecount >= len(
             downloaded_track_list):
-        if headphones.CONFIG.KEEP_TORRENT_FILES and Kind == "torrent":
+        new_folder = None
-            albumpath = helpers.preserve_torrent_directory(albumpath)
+        new_albumpath = albumpath
-        if albumpath and helpers.cue_split(albumpath):
+        if keep_original_folder:
+            temp_path = helpers.preserve_torrent_directory(new_albumpath, forced)
+            if not temp_path:
+                markAsUnprocessed(albumid, new_albumpath, keep_original_folder)
+                return
+            else:
+                new_albumpath = temp_path
+                new_folder = os.path.split(new_albumpath)[0]
+                Kind = "cue_split"
+        cuepath = helpers.cue_split(new_albumpath)
+        if not cuepath:
+            if new_folder:
+                shutil.rmtree(new_folder)
+            markAsUnprocessed(albumid, albumpath, keep_original_folder)
+            return
+        else:
+            albumpath = cuepath
             downloaded_track_list = helpers.get_downloaded_track_list(albumpath)
+            keep_original_folder = False
 
     # test #1: metadata - usually works
     logger.debug('Verifying metadata...')
@@ -243,7 +285,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
 
         if metaartist == dbartist and metaalbum == dbalbum:
             doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind,
-                             keep_original_folder)
+                             keep_original_folder, forced, single)
             return
 
     # test #2: filenames
@@ -262,7 +304,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
 
             if dbtrack in filetrack:
                 doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind,
-                                 keep_original_folder)
+                                 keep_original_folder, forced, single)
                 return
 
     # test #3: number of songs and duration
@@ -295,46 +337,49 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
             delta = abs(downloaded_track_duration - db_track_duration)
             if delta < 240:
                 doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind,
-                                 keep_original_folder)
+                                 keep_original_folder, forced, single)
                 return
 
     logger.warn(u'Could not identify album: %s. It may not be the intended album.',
                 albumpath.decode(headphones.SYS_ENCODING, 'replace'))
+    markAsUnprocessed(albumid, albumpath, keep_original_folder)
+
+
+def markAsUnprocessed(albumid, albumpath, keep_original_folder=False):
+    myDB = db.DBConnection()
     myDB.action(
-        'UPDATE snatched SET status = "Unprocessed" WHERE status NOT LIKE "Seed%" and AlbumID=?',
+        'UPDATE snatched SET status = "Unprocessed" WHERE status NOT LIKE "Seed%" and AlbumID=?', [albumid])
-        [albumid])
     processed = re.search(r' \(Unprocessed\)(?:\[\d+\])?', albumpath)
     if not processed:
-        if headphones.CONFIG.RENAME_UNPROCESSED:
+        if headphones.CONFIG.RENAME_UNPROCESSED and not keep_original_folder:
             renameUnprocessedFolder(albumpath, tag="Unprocessed")
         else:
-            logger.warn(u"Won't rename %s to mark as 'Unprocessed', because it is disabled.",
+            logger.warn(u"Won't rename %s to mark as 'Unprocessed', because it is disabled or folder is being kept.",
                         albumpath.decode(headphones.SYS_ENCODING, 'replace'))
 
 
 def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind=None,
-                     keep_original_folder=False):
+                     keep_original_folder=False, forced=False, single=False):
     logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle']))
     new_folder = None
-    # Check to see if we're preserving the torrent dir
+
-    if (headphones.CONFIG.KEEP_TORRENT_FILES and Kind == "torrent" and 'headphones-modified' not in albumpath) or headphones.CONFIG.KEEP_ORIGINAL_FOLDER or keep_original_folder:
+    # Preserve the torrent dir
-        new_folder = tempfile.mkdtemp(prefix="headphones_")
+    if keep_original_folder or single:
-        subdir = os.path.join(new_folder, "headphones")
+        temp_path = helpers.preserve_torrent_directory(albumpath, forced, single)
-        logger.info("Copying files to " + subdir.decode(headphones.SYS_ENCODING, 'replace') + " subfolder to preserve downloaded files for seeding")
+        if not temp_path:
-        try:
+            markAsUnprocessed(albumid, albumpath, keep_original_folder)
-            shutil.copytree(albumpath, subdir)
-            # Update the album path with the new location
-            albumpath = subdir
-        except Exception as e:
-            logger.warn("Cannot copy/move files to temp folder: " + new_folder.decode(headphones.SYS_ENCODING, 'replace') + ". Not continuing. Error: " + str(e))
-            shutil.rmtree(new_folder)
             return
+        else:
+            albumpath = temp_path
+            new_folder = os.path.split(albumpath)[0]
+    elif Kind == "cue_split":
+        new_folder = os.path.split(albumpath)[0]
 
     # Need to update the downloaded track list with the new location.
     # Could probably just throw in the "headphones-modified" folder,
     # but this is good to make sure we're not counting files that may have failed to move
+    if new_folder:
         downloaded_track_list = []
-
         for r, d, f in os.walk(albumpath):
             for files in f:
                 if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
@@ -352,7 +397,8 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                          downloaded_track.decode(headphones.SYS_ENCODING, "replace"))
             return
         except IOError:
-            logger.error("Unable to find media file: %s. Not continuing.")
+            logger.error("Unable to find media file: %s. Not continuing.", downloaded_track.decode(
+                headphones.SYS_ENCODING, "replace"))
             if new_folder:
                 shutil.rmtree(new_folder)
             return
@@ -387,21 +433,13 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                 shutil.rmtree(new_folder)
             return
 
+    # get artwork and path
+    album_art_path = None
     artwork = None
-    album_art_path = albumart.getAlbumArt(albumid)
+    if headphones.CONFIG.EMBED_ALBUM_ART or headphones.CONFIG.ADD_ALBUM_ART or \
-    if headphones.CONFIG.EMBED_ALBUM_ART or headphones.CONFIG.ADD_ALBUM_ART:
+            (headphones.CONFIG.PLEX_ENABLED and headphones.CONFIG.PLEX_NOTIFY) or \
-
+            (headphones.CONFIG.XBMC_ENABLED and headphones.CONFIG.XBMC_NOTIFY):
-        if album_art_path:
+        album_art_path, artwork = albumart.getAlbumArt(albumid)
-            artwork = request.request_content(album_art_path)
-        else:
-            artwork = None
-
-        if not album_art_path or not artwork or len(artwork) < 100:
-            logger.info("No suitable album art found from Amazon. Checking Last.FM....")
-            artwork = albumart.getCachedArt(albumid)
-            if not artwork or len(artwork) < 100:
-                artwork = False
-                logger.info("No suitable album art found from Last.FM. Not adding album art")
 
     if headphones.CONFIG.EMBED_ALBUM_ART and artwork:
         embedAlbumArt(artwork, downloaded_track_list)
@@ -446,12 +484,12 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         [albumid])
 
     # Check if torrent has finished seeding
-    if headphones.CONFIG.TORRENT_DOWNLOADER == 1 or headphones.CONFIG.TORRENT_DOWNLOADER == 2:
+    if headphones.CONFIG.TORRENT_DOWNLOADER != 0:
         seed_snatched = myDB.action(
             'SELECT * from snatched WHERE Status="Seed_Snatched" and AlbumID=?',
             [albumid]).fetchone()
         if seed_snatched:
-            hash = seed_snatched['FolderName']
+            hash = seed_snatched['TorrentHash']
             torrent_removed = False
             logger.info(u'%s - %s. Checking if torrent has finished seeding and can be removed' % (
                 release['ArtistName'], release['AlbumTitle']))
@@ -459,8 +497,10 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                 torrent_removed = transmission.removeTorrent(hash, True)
             elif headphones.CONFIG.TORRENT_DOWNLOADER == 3:  # Deluge
                 torrent_removed = deluge.removeTorrent(hash, True)
-            else:
+            elif headphones.CONFIG.TORRENT_DOWNLOADER == 2:
                 torrent_removed = utorrent.removeTorrent(hash, True)
+            else:
+                torrent_removed = qbittorrent.removeTorrent(hash, True)
 
             # Torrent removed, delete the snatched record, else update Status for scheduled job to check
             if torrent_removed:
@@ -538,10 +578,15 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         pushbullet = notifiers.PUSHBULLET()
         pushbullet.notify(pushmessage, statusmessage)
 
+    if headphones.CONFIG.JOIN_ENABLED:
+        logger.info(u"Join request")
+        join = notifiers.JOIN()
+        join.notify(pushmessage, statusmessage)
+
     if headphones.CONFIG.TELEGRAM_ENABLED:
         logger.info(u"Telegram request")
         telegram = notifiers.TELEGRAM()
-        telegram.notify(pushmessage, statusmessage)
+        telegram.notify(statusmessage, pushmessage)
 
     if headphones.CONFIG.TWITTER_ENABLED:
         logger.info(u"Sending Twitter notification")
@@ -615,7 +660,7 @@ def addAlbumArt(artwork, albumpath, release, metadata_dict):
     if artwork[:4] == '\x89PNG':
         ext = ".png"
 
-    album_art_name = helpers.replace_all(
+    album_art_name = helpers.pattern_substitute(
         headphones.CONFIG.ALBUM_ART_FORMAT.strip(), md) + ext
 
     album_art_name = helpers.replace_illegal_chars(album_art_name).encode(
@@ -671,7 +716,7 @@ def moveFiles(albumpath, release, metadata_dict):
     logger.info("Moving files: %s" % albumpath)
 
     md = metadata.album_metadata(albumpath, release, metadata_dict)
-    folder = helpers.replace_all(
+    folder = helpers.pattern_substitute(
         headphones.CONFIG.FOLDER_FORMAT.strip(), md, normalize=True)
 
     if headphones.CONFIG.FILE_UNDERSCORES:
@@ -918,12 +963,29 @@ def correctMetadata(albumid, release, downloaded_track_list):
         if not items:
             continue
 
+        search_ids = []
+        logger.debug('Getting recommendation from beets. Artist: %s. Album: %s. Tracks: %s', release['ArtistName'],
+                     release['AlbumTitle'], len(items))
+
+        # Try with specific release, e.g. alternate release selected from albumPage
+        if release['ReleaseID'] != release['AlbumID']:
+            logger.debug('trying beets with specific Release ID: %s', release['ReleaseID'])
+            search_ids = [release['ReleaseID']]
+
         try:
-            cur_artist, cur_album, candidates, rec = autotag.tag_album(items,
+            beetslog = beetslogging.getLogger('beets')
-                                                                       search_artist=helpers.latinToAscii(
+            beetslog.set_global_level(beetslogging.DEBUG) if headphones.VERBOSE else beetslog.set_global_level(
-                                                                           release['ArtistName']),
+                beetslogging.CRITICAL)
-                                                                       search_album=helpers.latinToAscii(
+            with helpers.capture_beets_log() as logs:
-                                                                           release['AlbumTitle']))
+                cur_artist, cur_album, prop = autotag.tag_album(items,
+                                                                search_artist=release['ArtistName'],
+                                                                search_album=release['AlbumTitle'],
+                                                                search_ids=search_ids)
+                candidates = prop.candidates
+                rec = prop.recommendation
+                for log in logs:
+                    logger.debug('Beets: %s', log)
+            beetslog.set_global_level(beetslogging.NOTSET)
         except Exception as e:
             logger.error('Error getting recommendation: %s. Not writing metadata', e)
             return False
@@ -1033,7 +1095,7 @@ def renameFiles(albumpath, downloaded_track_list, release):
             title = md[metadata.Vars.TITLE]
             new_file_name = helpers.cleanTitle(title) + ext
         else:
-            new_file_name = helpers.replace_all(
+            new_file_name = helpers.pattern_substitute(
                 headphones.CONFIG.FILE_FORMAT.strip(), md
             ).replace('/', '_') + ext
 
@@ -1186,7 +1248,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                     'Found a match in the database: %s. Verifying to make sure it is the correct album',
                     snatched['Title'])
                 verify(snatched['AlbumID'], folder, snatched['Kind'],
-                       keep_original_folder=keep_original_folder)
+                       forced=True, keep_original_folder=keep_original_folder)
                 continue
 
         # Attempt 2: strip release group id from filename
@@ -1213,7 +1275,8 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
             else:
                 logger.info(
                     'Found a (possibly) valid Musicbrainz release group id in album folder name.')
-                verify(rgid, folder, forced=True)
+                verify(rgid, folder, forced=True,
+                       keep_original_folder=keep_original_folder)
                 continue
 
         # Attempt 3a: parse the folder name into a valid format
@@ -1232,7 +1295,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                 logger.info(
                     'Found a match in the database: %s - %s. Verifying to make sure it is the correct album',
                     release['ArtistName'], release['AlbumTitle'])
-                verify(release['AlbumID'], folder, keep_original_folder=keep_original_folder)
+                verify(release['AlbumID'], folder, forced=True, keep_original_folder=keep_original_folder)
                 continue
             else:
                 logger.info('Querying MusicBrainz for the release group id for: %s - %s', name,
@@ -1244,7 +1307,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                     rgid = None
 
                 if rgid:
-                    verify(rgid, folder, keep_original_folder=keep_original_folder)
+                    verify(rgid, folder, forced=True, keep_original_folder=keep_original_folder)
                     continue
                 else:
                     logger.info('No match found on MusicBrainz for: %s - %s', name, album)
@@ -1257,12 +1320,23 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
         except Exception:
             name = album = None
 
-        # Check if there's a cue to split
+        # Not found from meta data, check if there's a cue to split and try meta data again
-        if headphones.CONFIG.CUE_SPLIT and not name and not album and helpers.cue_split(folder):
+        kind = None
+        if headphones.CONFIG.CUE_SPLIT and not name and not album:
+            cue_folder = helpers.cue_split(folder, keep_original_folder=keep_original_folder)
+            if cue_folder:
                 try:
-                name, album, year = helpers.extract_metadata(folder)
+                    name, album, year = helpers.extract_metadata(cue_folder)
                 except Exception:
                     name = album = None
+                if name:
+                    folder = cue_folder
+                    if keep_original_folder:
+                        keep_original_folder = False
+                        kind = "cue_split"
+                elif folder != cue_folder:
+                    cue_folder = os.path.split(cue_folder)[0]
+                    shutil.rmtree(cue_folder)
 
         if name and album:
             release = myDB.action(
@@ -1272,7 +1346,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                 logger.info(
                     'Found a match in the database: %s - %s. Verifying to make sure it is the correct album',
                     release['ArtistName'], release['AlbumTitle'])
-                verify(release['AlbumID'], folder, keep_original_folder=keep_original_folder)
+                verify(release['AlbumID'], folder, Kind=kind, forced=True, keep_original_folder=keep_original_folder)
                 continue
             else:
                 logger.info('Querying MusicBrainz for the release group id for: %s - %s', name,
@@ -1284,7 +1358,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                     rgid = None
 
                 if rgid:
-                    verify(rgid, folder, keep_original_folder=keep_original_folder)
+                    verify(rgid, folder, Kind=kind, forced=True, keep_original_folder=keep_original_folder)
                     continue
                 else:
                     logger.info('No match found on MusicBrainz for: %s - %s', name, album)
@@ -1301,7 +1375,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                 logger.info(
                     'Found a match in the database: %s - %s. Verifying to make sure it is the correct album',
                     release['ArtistName'], release['AlbumTitle'])
-                verify(release['AlbumID'], folder, keep_original_folder=keep_original_folder)
+                verify(release['AlbumID'], folder, forced=True, keep_original_folder=keep_original_folder)
                 continue
             else:
                 logger.info('Querying MusicBrainz for the release group id for: %s',
@@ -1313,7 +1387,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
                     rgid = None
 
                 if rgid:
-                    verify(rgid, folder, keep_original_folder=keep_original_folder)
+                    verify(rgid, folder, forced=True, keep_original_folder=keep_original_folder)
                     continue
                 else:
                     logger.info('No match found on MusicBrainz for: %s - %s', name, album)

headphones/qbittorrent.py

@@ -0,0 +1,358 @@
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
+
+import urllib
+import urllib2
+import cookielib
+import json
+import time
+import mimetypes
+import random
+import string
+import os
+
+import headphones
+
+from headphones import logger
+from collections import namedtuple
+
+from qbittorrentv2 import Client
+
+
+class qbittorrentclient(object):
+
+    TOKEN_REGEX = "<div id='token' style='display:none;'>([^<>]+)</div>"
+    UTSetting = namedtuple("UTSetting", ["name", "int", "str", "access"])
+
+    def __init__(self, base_url=None, username=None, password=None,):
+
+        host = headphones.CONFIG.QBITTORRENT_HOST
+        if not host.startswith('http'):
+            host = 'http://' + host
+
+        if host.endswith('/'):
+            host = host[:-1]
+
+        if host.endswith('/gui'):
+            host = host[:-4]
+
+        self.base_url = host
+        self.username = headphones.CONFIG.QBITTORRENT_USERNAME
+        self.password = headphones.CONFIG.QBITTORRENT_PASSWORD
+
+        # Try new v2 api
+        try:
+            self.qb = Client(self.base_url)
+            login_text = self.qb.login(self.username, self.password)
+            if login_text:
+                logger.warning("Could not login to qBittorrent v2 api, check credentials: %s", login_text)
+            self.version = 2
+        except Exception as e:
+            logger.warning("Error with qBittorrent v2 api, check settings or update, will try v1: %s" % e)
+            self.cookiejar = cookielib.CookieJar()
+            self.opener = self._make_opener()
+            self._get_sid(self.base_url, self.username, self.password)
+            self.version = 1
+
+    def _make_opener(self):
+        # create opener with cookie handler to carry QBitTorrent SID cookie
+        cookie_handler = urllib2.HTTPCookieProcessor(self.cookiejar)
+        handlers = [cookie_handler]
+        return urllib2.build_opener(*handlers)
+
+    def _get_sid(self, base_url, username, password):
+        # login so we can capture SID cookie
+        login_data = urllib.urlencode({'username': username, 'password': password})
+        try:
+            self.opener.open(base_url + '/login', login_data)
+        except urllib2.URLError as err:
+            logger.debug('Error getting SID. qBittorrent responded with error: ' + str(err.reason))
+            return
+        for cookie in self.cookiejar:
+                logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value)
+        return
+
+    def _command(self, command, args=None, content_type=None, files=None):
+        logger.debug('QBittorrent WebAPI Command: %s' % command)
+
+        url = self.base_url + '/' + command
+
+        data = None
+        headers = dict()
+        if content_type == 'multipart/form-data':
+            data, headers = encode_multipart(args, files)
+        else:
+            if args:
+                data = urllib.urlencode(args)
+            if content_type:
+                headers['Content-Type'] = content_type
+
+        logger.debug('%s' % json.dumps(headers, indent=4))
+        logger.debug('%s' % data)
+
+        request = urllib2.Request(url, data, headers)
+        try:
+            response = self.opener.open(request)
+            info = response.info()
+            if info:
+                if info.getheader('content-type'):
+                    if info.getheader('content-type') == 'application/json':
+                        resp = ''
+                        for line in response:
+                            resp = resp + line
+                        logger.debug('response code: %s' % str(response.code))
+                        logger.debug('response: %s' % resp)
+                        return response.code, json.loads(resp)
+            logger.debug('response code: %s' % str(response.code))
+            return response.code, None
+        except urllib2.URLError as err:
+            logger.debug('Failed URL: %s' % url)
+            logger.debug('QBitTorrent webUI raised the following error: %s' % str(err))
+            return None, None
+
+    def _get_list(self, **args):
+        return self._command('query/torrents', args)
+
+    def _get_settings(self):
+        status, value = self._command('query/preferences')
+        logger.debug('get_settings() returned %d items' % len(value))
+        return value
+
+    def get_savepath(self, hash):
+        logger.debug('qb.get_savepath(%s)' % hash)
+        status, torrentList = self._get_list()
+        for torrent in torrentList:
+            if torrent['hash']:
+                if torrent['hash'].upper() == hash.upper():
+                    return torrent['save_path']
+        return None
+
+    def start(self, hash):
+        logger.debug('qb.start(%s)' % hash)
+        args = {'hash': hash}
+        return self._command('command/resume', args, 'application/x-www-form-urlencoded')
+
+    def pause(self, hash):
+        logger.debug('qb.pause(%s)' % hash)
+        args = {'hash': hash}
+        return self._command('command/pause', args, 'application/x-www-form-urlencoded')
+
+    def getfiles(self, hash):
+        logger.debug('qb.getfiles(%s)' % hash)
+        return self._command('query/propertiesFiles/' + hash)
+
+    def getprops(self, hash):
+        logger.debug('qb.getprops(%s)' % hash)
+        return self._command('query/propertiesGeneral/' + hash)
+
+    def setprio(self, hash, priority):
+        logger.debug('qb.setprio(%s,%d)' % (hash, priority))
+        args = {'hash': hash, 'priority': priority}
+        return self._command('command/setFilePrio', args, 'application/x-www-form-urlencoded')
+
+    def remove(self, hash, remove_data=False):
+        logger.debug('qb.remove(%s,%s)' % (hash, remove_data))
+
+        args = {'hashes': hash}
+        if remove_data:
+            command = 'command/deletePerm'
+        else:
+            command = 'command/delete'
+        return self._command(command, args, 'application/x-www-form-urlencoded')
+
+
+def removeTorrent(hash, remove_data=False):
+    logger.debug('removeTorrent(%s,%s)' % (hash, remove_data))
+
+    qbclient = qbittorrentclient()
+    if qbclient.version == 2:
+        torrentlist = qbclient.qb.torrents(hashes=hash.lower())
+    else:
+        status, torrentlist = qbclient._get_list()
+    for torrent in torrentlist:
+        if torrent['hash'].lower() == hash.lower():
+            if torrent['ratio'] >= torrent['ratio_limit'] and torrent['ratio_limit'] >= 0:
+                if qbclient.version == 2:
+                    if remove_data:
+                        logger.info(
+                            '%s has finished seeding, removing torrent and data. '
+                            'Ratio: %s, Ratio Limit: %s' % (torrent['name'], torrent['ratio'], torrent['ratio_limit']))
+                        qbclient.qb.delete_permanently(hash)
+                    else:
+                        logger.info('%s has finished seeding, removing torrent' % torrent['name'])
+                        qbclient.qb.delete(hash)
+                else:
+                    qbclient.remove(hash, remove_data)
+                return True
+            else:
+                logger.info(
+                    '%s has not finished seeding yet, torrent will not be removed, will try again on next run. '
+                    'Ratio: %s, Ratio Limit: %s' % (torrent['name'], torrent['ratio'], torrent['ratio_limit']))
+                return False
+    return False
+
+
+def addTorrent(link):
+    logger.debug('addTorrent(%s)' % link)
+
+    qbclient = qbittorrentclient()
+    if qbclient.version == 2:
+        return qbclient.qb.download_from_link(link, savepath=headphones.CONFIG.DOWNLOAD_TORRENT_DIR,
+                                              category=headphones.CONFIG.QBITTORRENT_LABEL)
+    else:
+        args = {'urls': link, 'savepath': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
+        if headphones.CONFIG.QBITTORRENT_LABEL:
+            args['category'] = headphones.CONFIG.QBITTORRENT_LABEL
+
+        return qbclient._command('command/download', args, 'multipart/form-data')
+
+
+def addFile(data):
+    logger.debug('addFile(data)')
+
+    qbclient = qbittorrentclient()
+    if qbclient.version == 2:
+        return qbclient.qb.download_from_file(data, savepath=headphones.CONFIG.DOWNLOAD_TORRENT_DIR,
+                                              category=headphones.CONFIG.QBITTORRENT_LABEL)
+    else:
+        files = {'torrents': {'filename': '', 'content': data}}
+        return qbclient._command('command/upload', filelist=files)
+
+
+def getName(hash):
+    logger.debug('getName(%s)' % hash)
+
+    qbclient = qbittorrentclient()
+
+    tries = 1
+    while tries <= 6:
+        time.sleep(10)
+        if qbclient.version == 2:
+            torrentlist = qbclient.qb.torrents(hashes=hash.lower())
+        else:
+            status, torrentlist = qbclient._get_list()
+        for torrent in torrentlist:
+            if torrent['hash'].lower() == hash.lower():
+                return torrent['name']
+        tries += 1
+
+    return None
+
+
+def getFolder(hash):
+    logger.debug('getFolder(%s)' % hash)
+
+    torrent_folder = None
+    single_file = False
+
+    qbclient = qbittorrentclient()
+
+    try:
+        if qbclient.version == 2:
+            torrent_files = qbclient.qb.get_torrent_files(hash.lower())
+        else:
+            status, torrent_files = qbclient.getfiles(hash.lower())
+        if torrent_files:
+            if len(torrent_files) == 1:
+                torrent_folder = torrent_files[0]['name']
+                single_file = True
+            else:
+                torrent_folder = os.path.split(torrent_files[0]['name'])[0]
+                torrent_folder = torrent_folder.split(os.sep)[0]
+                single_file = False
+    except:
+        torrent_folder = None
+        single_file = False
+
+    return torrent_folder, single_file
+
+
+def setSeedRatio(hash, ratio):
+    logger.debug('setSeedRatio(%s)' % hash)
+
+    qbclient = qbittorrentclient()
+
+    if qbclient.version == 2:
+        ratio = -1 if ratio == 0 else ratio
+        return qbclient.qb.set_share_ratio(hash.lower(), ratio)
+    else:
+        logger.warn('setSeedRatio only available with qBittorrent v2 api')
+        return
+
+
+def apiVersion2():
+    logger.debug('getApiVersion')
+
+    qbclient = qbittorrentclient()
+
+    if qbclient.version == 2:
+        return True
+    else:
+        return False
+
+
+_BOUNDARY_CHARS = string.digits + string.ascii_letters
+
+
+# Taken from http://code.activestate.com/recipes/578668-encode-multipart-form-data-for-uploading-files-via/
+# "MIT License" which is compatible with GPL
+def encode_multipart(args, files, boundary=None):
+    logger.debug('encode_multipart()')
+
+    def escape_quote(s):
+        return s.replace('"', '\\"')
+
+    if boundary is None:
+        boundary = ''.join(random.choice(_BOUNDARY_CHARS) for i in range(30))
+    lines = []
+
+    if args:
+        for name, value in args.items():
+            lines.extend((
+                '--{0}'.format(boundary),
+                'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
+                '',
+                str(value),
+            ))
+    logger.debug(''.join(lines))
+
+    if files:
+        for name, value in files.items():
+            filename = value['filename']
+            if 'mimetype' in value:
+                mimetype = value['mimetype']
+            else:
+                mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+            lines.extend((
+                '--{0}'.format(boundary),
+                'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(
+                    escape_quote(name), escape_quote(filename)),
+                'Content-Type: {0}'.format(mimetype),
+                '',
+                value['content'],
+            ))
+
+    lines.extend((
+        '--{0}--'.format(boundary),
+        '',
+    ))
+    body = '\r\n'.join(lines)
+
+    headers = {
+        'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
+        'Content-Length': str(len(body)),
+    }
+
+    return (body, headers)

headphones/rutracker.py

@@ -6,6 +6,7 @@ from urlparse import urlparse
 import re
 
 import requests as requests
+# from requests.auth import HTTPDigestAuth
 from bs4 import BeautifulSoup
 
 import headphones
@@ -24,7 +25,7 @@ class Rutracker(object):
         return self.loggedin
 
     def still_logged_in(self, html):
-        if not html or "action=\"http://login.rutracker.org/forum/login.php\">" in html:
+        if not html or "action=\"http://rutracker.org/forum/login.php\">" in html:
             return False
         else:
             return True
@@ -34,7 +35,7 @@ class Rutracker(object):
         Logs in user
         """
 
-        loginpage = 'http://login.rutracker.org/forum/login.php'
+        loginpage = 'http://rutracker.org/forum/login.php'
         post_params = {
             'login_username': headphones.CONFIG.RUTRACKER_USER,
             'login_password': headphones.CONFIG.RUTRACKER_PASSWORD,
@@ -46,10 +47,14 @@ class Rutracker(object):
         try:
             r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
             # try again
-            if not self.has_bb_data_cookie(r):
+            if not self.has_bb_session_cookie(r):
                 time.sleep(10)
+                if headphones.CONFIG.RUTRACKER_COOKIE:
+                    logger.info("Attempting to log in using predefined cookie...")
+                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE})
+                else:
                     r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
-            if self.has_bb_data_cookie(r):
+            if self.has_bb_session_cookie(r):
                 self.loggedin = True
                 logger.info("Successfully logged in to rutracker")
             else:
@@ -62,11 +67,11 @@ class Rutracker(object):
             self.loggedin = False
             return self.loggedin
 
-    def has_bb_data_cookie(self, response):
+    def has_bb_session_cookie(self, response):
-        if 'bb_data' in response.cookies.keys():
+        if 'bb_session' in response.cookies.keys():
             return True
         # Rutracker randomly send a 302 redirect code, cookie may be present in response history
-        return next(('bb_data' in r.cookies.keys() for r in response.history), False)
+        return next(('bb_session' in r.cookies.keys() for r in response.history), False)
 
     def searchurl(self, artist, album, year, format):
         """
@@ -82,10 +87,10 @@ class Rutracker(object):
         searchterm = searchterm + year
 
         if format == 'lossless':
-            format = '+lossless'
+            format = '+lossless||TR24'
             self.maxsize = 10000000000
         elif format == 'lossless+mp3':
-            format = '+lossless||mp3||aac'
+            format = '+lossless||TR24||mp3||aac'
             self.maxsize = 10000000000
         else:
             format = '+mp3||aac'
@@ -93,8 +98,13 @@ class Rutracker(object):
 
         # sort by size, descending.
         sort = '&o=7&s=2'
+        try:
+            searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort)
+        except:
+            searchterm = searchterm.encode('utf-8')
             searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort)
         logger.info("Searching rutracker using term: %s", searchterm)
+
         return searchurl
 
     def search(self, searchurl):
@@ -169,7 +179,7 @@ class Rutracker(object):
         return the .torrent data
         """
         torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
-        downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id
+        downloadurl = 'http://rutracker.org/forum/dl.php?t=' + torrent_id
         cookie = {'bb_dl': torrent_id}
         try:
             headers = {'Referer': url}
@@ -216,3 +226,37 @@ class Rutracker(object):
             self.session.post(url, params={'action': 'add-file'}, files=files)
         except Exception as e:
             logger.exception('Error adding file to utorrent %s', e)
+
+    # TODO get this working in qbittorrent.py
+    def qbittorrent_add_file(self, data):
+        host = headphones.CONFIG.QBITTORRENT_HOST
+        if not host.startswith('http'):
+            host = 'http://' + host
+        if host.endswith('/'):
+            host = host[:-1]
+        if host.endswith('/gui'):
+            host = host[:-4]
+        base_url = host
+
+        # self.session.auth = HTTPDigestAuth(headphones.CONFIG.QBITTORRENT_USERNAME, headphones.CONFIG.QBITTORRENT_PASSWORD)
+
+        url = base_url + '/login'
+        try:
+            self.session.post(url, data={'username': headphones.CONFIG.QBITTORRENT_USERNAME,
+                                         'password': headphones.CONFIG.QBITTORRENT_PASSWORD})
+        except Exception as e:
+            logger.exception('Error adding file to qbittorrent %s', e)
+            return
+
+        url = base_url + '/command/upload'
+
+        args = {'savepath': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
+        if headphones.CONFIG.QBITTORRENT_LABEL:
+            args['category'] = headphones.CONFIG.QBITTORRENT_LABEL
+
+        torrent_files = {'torrents': data}
+
+        try:
+            self.session.post(url, data=args, files=torrent_files)
+        except Exception as e:
+            logger.exception('Error adding file to qbittorrent %s', e)

headphones/searcher.py

@@ -14,7 +14,6 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
-# t411 support added by a1ex, @likeitneverwentaway on github for maintenance
 
 from base64 import b16encode, b32decode
 from hashlib import sha1
@@ -25,29 +24,31 @@ import datetime
 import subprocess
 import unicodedata
 import urlparse
-from json import loads
 
 import os
 import re
 from pygazelle import api as gazelleapi
 from pygazelle import encoding as gazelleencoding
 from pygazelle import format as gazelleformat
+from pygazelle import release_type as gazellerelease_type
 import headphones
 from headphones.common import USER_AGENT
 from headphones import logger, db, helpers, classes, sab, nzbget, request
-from headphones import utorrent, transmission, notifiers, rutracker, deluge
+from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent
 from bencode import bencode, bdecode
 
 # Magnet to torrent services, for Black hole. Stolen from CouchPotato.
 TORRENT_TO_MAGNET_SERVICES = [
-    # 'https://zoink.it/torrent/%s.torrent',
+    'http://itorrents.org/torrent/%s.torrent',
-    # 'http://torrage.com/torrent/%s.torrent',
+    'https://cache.torrentgalaxy.org/get/%s',
-    'https://torcache.net/torrent/%s.torrent',
+    'https://www.seedpeer.me/torrent/%s'
 ]
 
-# Persistent What.cd API object
+# Persistent Orpheus.network API object
-gazelle = None
+orpheusobj = None
 ruobj = None
+# Persistent RED API object
+redobj = None
 
 
 def fix_url(s, charset="utf-8"):
@@ -152,28 +153,31 @@ def calculate_torrent_hash(link, data=None):
 
 def get_seed_ratio(provider):
     """
-    Return the seed ratio for the specified provider, if applicable. Defaults to
+    Return the seed ratio for the specified provider if applicable. Defaults to
     None in case of an error.
     """
 
     if provider == 'rutracker.org':
         seed_ratio = headphones.CONFIG.RUTRACKER_RATIO
-    elif provider == 'Kick Ass Torrents':
+    elif provider == 'Orpheus.network':
-        seed_ratio = headphones.CONFIG.KAT_RATIO
+        seed_ratio = headphones.CONFIG.ORPHEUS_RATIO
-    elif provider == 'What.cd':
+    elif provider == 'Redacted':
-        seed_ratio = headphones.CONFIG.WHATCD_RATIO
+        seed_ratio = headphones.CONFIG.REDACTED_RATIO
-    elif provider == 'PassTheHeadphones.Me':
-        seed_ratio = headphones.CONFIG.PTH_RATIO
     elif provider == 'The Pirate Bay':
         seed_ratio = headphones.CONFIG.PIRATEBAY_RATIO
     elif provider == 'Old Pirate Bay':
         seed_ratio = headphones.CONFIG.OLDPIRATEBAY_RATIO
     elif provider == 'Waffles.ch':
         seed_ratio = headphones.CONFIG.WAFFLES_RATIO
-    elif provider == 'Mininova':
+    elif provider.startswith("Jackett_"):
-        seed_ratio = headphones.CONFIG.MININOVA_RATIO
+        provider = provider.split("Jackett_")[1]
-    elif provider == 'Strike':
+        if provider in headphones.CONFIG.TORZNAB_HOST:
-        seed_ratio = headphones.CONFIG.STRIKE_RATIO
+            seed_ratio = headphones.CONFIG.TORZNAB_RATIO
+        else:
+            for torznab in headphones.CONFIG.get_extra_torznabs():
+                if provider in torznab[0]:
+                    seed_ratio = torznab[2]
+                    break
     else:
         seed_ratio = None
 
@@ -268,15 +272,12 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
                        headphones.CONFIG.NZBGET_HOST)
 
     TORRENT_PROVIDERS = (headphones.CONFIG.TORZNAB or
-                         headphones.CONFIG.KAT or
                          headphones.CONFIG.PIRATEBAY or
                          headphones.CONFIG.OLDPIRATEBAY or
-                         headphones.CONFIG.MININOVA or
                          headphones.CONFIG.WAFFLES or
                          headphones.CONFIG.RUTRACKER or
-                         headphones.CONFIG.WHATCD or
+                         headphones.CONFIG.ORPHEUS or
-                         headphones.CONFIG.PTH or
+                         headphones.CONFIG.REDACTED)
-                         headphones.CONFIG.STRIKE)
 
     results = []
     myDB = db.DBConnection()
@@ -435,6 +436,7 @@ def sort_search_results(resultlist, album, new, albumlength):
 
     resultlist = temp_list
 
+    # if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and result[3] != 'Orpheus.network':
     if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE:
 
         try:
@@ -482,6 +484,10 @@ def sort_search_results(resultlist, album, new, albumlength):
 
         finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), reverse=True)
 
+        # keep number of seeders order for Orpheus.network
+        # if result[3] == 'Orpheus.network':
+        #    finallist = resultlist
+
     if not len(finallist):
         logger.info('No appropriate matches found for %s - %s', album['ArtistName'],
                     album['AlbumTitle'])
@@ -737,7 +743,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
         }
 
         data = request.request_json(
-            url='http://api.omgwtfnzbs.org/json/',
+            url='http://api.omgwtfnzbs.me/json/',
             params=params, headers=headers
         )
 
@@ -836,19 +842,6 @@ def send_to_downloader(data, bestqual, album):
             torrent_name = helpers.replace_illegal_chars(folder_name) + '.torrent'
             download_path = os.path.join(headphones.CONFIG.TORRENTBLACKHOLE_DIR, torrent_name)
 
-            # Blackhole for t411
-            if bestqual[2].lower().startswith("http://api.t411"):
-                if headphones.CONFIG.MAGNET_LINKS == 2:
-                    try:
-                        url = bestqual[2].split('TOKEN')[0]
-                        token = bestqual[2].split('TOKEN')[1]
-                        data = request.request_content(url, headers={'Authorization': token})
-                        torrent_to_file(download_path, data)
-                        logger.info('Successfully converted magnet to torrent file')
-                    except Exception as e:
-                        logger.error("Error converting magnet link: %s" % str(e))
-                        return
-
             if bestqual[2].lower().startswith("magnet:"):
                 if headphones.CONFIG.MAGNET_LINKS == 1:
                     try:
@@ -874,12 +867,11 @@ def send_to_downloader(data, bestqual, album):
                     services = TORRENT_TO_MAGNET_SERVICES[:]
                     random.shuffle(services)
                     headers = {'User-Agent': USER_AGENT}
-                    headers['Referer'] = 'https://torcache.net/'
 
                     for service in services:
 
                         data = request.request_content(service % torrent_hash, headers=headers)
-                        if data and "torcache" in data:
+                        if data:
                             if not torrent_to_file(download_path, data):
                                 return
                             # Extract folder name from torrent
@@ -925,11 +917,11 @@ def send_to_downloader(data, bestqual, album):
                 logger.error("Error sending torrent to Transmission. Are you sure it's running?")
                 return
 
-            folder_name = transmission.getTorrentFolder(torrentid)
+            folder_name = transmission.getName(torrentid)
             if folder_name:
-                logger.info('Torrent folder name: %s' % folder_name)
+                logger.info('Torrent name: %s' % folder_name)
             else:
-                logger.error('Torrent folder name could not be determined')
+                logger.error('Torrent name could not be determined')
                 return
 
             # Set Seed Ratio
@@ -979,7 +971,7 @@ def send_to_downloader(data, bestqual, album):
             except Exception as e:
                 logger.error('Error sending torrent to Deluge: %s' % str(e))
 
-        else:  # if headphones.CONFIG.TORRENT_DOWNLOADER == 2:
+        elif headphones.CONFIG.TORRENT_DOWNLOADER == 2:
             logger.info("Sending torrent to uTorrent")
 
             # Add torrent
@@ -1010,19 +1002,50 @@ def send_to_downloader(data, bestqual, album):
             seed_ratio = get_seed_ratio(bestqual[3])
             if seed_ratio is not None:
                 utorrent.setSeedRatio(torrentid, seed_ratio)
+        else:  # if headphones.CONFIG.TORRENT_DOWNLOADER == 4:
+            logger.info("Sending torrent to QBiTorrent")
+
+            # Add torrent
+            if bestqual[3] == 'rutracker.org':
+                if qbittorrent.apiVersion2:
+                    qbittorrent.addFile(data)
+                else:
+                    ruobj.qbittorrent_add_file(data)
+            else:
+                qbittorrent.addTorrent(bestqual[2])
+
+            # Get hash
+            torrentid = calculate_torrent_hash(bestqual[2], data)
+            torrentid = torrentid.lower()
+            if not torrentid:
+                logger.error('Torrent id could not be determined')
+                return
+
+            # Get name
+            folder_name = qbittorrent.getName(torrentid)
+            if folder_name:
+                logger.info('Torrent name: %s' % folder_name)
+            else:
+                logger.error('Torrent name could not be determined')
+                return
+
+            # Set Seed Ratio
+            seed_ratio = get_seed_ratio(bestqual[3])
+            if seed_ratio is not None:
+                qbittorrent.setSeedRatio(torrentid, seed_ratio)
 
     myDB = db.DBConnection()
     myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']])
-    myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)',
+    myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
                 [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name,
-                 kind])
+                 kind, torrentid])
 
     # Store the torrent id so we can check later if it's finished seeding and can be removed
     if seed_ratio is not None and seed_ratio != 0 and torrentid:
         myDB.action(
-            'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)',
+            'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
-            [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", torrentid,
+            [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", folder_name,
-             kind])
+             kind, torrentid])
 
     # notify
     artist = album[1]
@@ -1050,10 +1073,22 @@ def send_to_downloader(data, bestqual, album):
         logger.info(u"Sending PushBullet notification")
         pushbullet = notifiers.PUSHBULLET()
         pushbullet.notify(name, "Download started")
+    if headphones.CONFIG.JOIN_ENABLED and headphones.CONFIG.JOIN_ONSNATCH:
+        logger.info(u"Sending Join notification")
+        join = notifiers.JOIN()
+        join.notify(name, "Download started")
+    if headphones.CONFIG.SLACK_ENABLED and headphones.CONFIG.SLACK_ONSNATCH:
+        logger.info(u"Sending Slack notification")
+        slack = notifiers.SLACK()
+        slack.notify(name, "Download started")
     if headphones.CONFIG.TELEGRAM_ENABLED and headphones.CONFIG.TELEGRAM_ONSNATCH:
         logger.info(u"Sending Telegram notification")
+        from headphones import cache
+        c = cache.Cache()
+        album_art = c.get_artwork_from_cache(None, rgid)
         telegram = notifiers.TELEGRAM()
-        telegram.notify(name, "Download started")
+        message = 'Snatched from ' + provider + '. ' + name
+        telegram.notify(message, "Snatched: " + title, rgid, image=album_art)
     if headphones.CONFIG.TWITTER_ENABLED and headphones.CONFIG.TWITTER_ONSNATCH:
         logger.info(u"Sending Twitter notification")
         twitter = notifiers.TwitterNotifier()
@@ -1176,7 +1211,8 @@ def verifyresult(title, artistterm, term, lossless):
 
 def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                   choose_specific_download=False):
-    global gazelle  # persistent what.cd api object to reduce number of login attempts
+    global orpheusobj  # persistent orpheus.network api object to reduce number of login attempts
+    global redobj  # persistent redacted api object to reduce number of login attempts
     global ruobj  # and rutracker
 
     reldate = album['ReleaseDate']
@@ -1248,18 +1284,21 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
 
         if headphones.CONFIG.TORZNAB_HOST and headphones.CONFIG.TORZNAB_ENABLED:
             torznab_hosts.append((headphones.CONFIG.TORZNAB_HOST, headphones.CONFIG.TORZNAB_APIKEY,
-                                  headphones.CONFIG.TORZNAB_ENABLED))
+                                  headphones.CONFIG.TORZNAB_RATIO, headphones.CONFIG.TORZNAB_ENABLED))
 
         for torznab_host in headphones.CONFIG.get_extra_torznabs():
-            if torznab_host[2] == '1' or torznab_host[2] == 1:
+            if torznab_host[3] == '1' or torznab_host[3] == 1:
                 torznab_hosts.append(torznab_host)
 
         if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:
             categories = "3040"
+            maxsize = 10000000000
         elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:
-            categories = "3040,3010"
+            categories = "3040,3010,3050"
+            maxsize = 10000000000
         else:
-            categories = "3010"
+            categories = "3010,3050"
+            maxsize = 300000000
 
         if album['Type'] == 'Other':
             categories = "3030"
@@ -1269,8 +1308,12 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
 
             provider = torznab_host[0]
 
+            # Format Jackett provider
+            if "api/v2.0/indexers" in torznab_host[0]:
+                provider = "Jackett_" + provider.split("/indexers/", 1)[1].split('/', 1)[0]
+
             # Request results
-            logger.info('Parsing results from %s using search term: %s' % (torznab_host[0], term))
+            logger.info('Parsing results from %s using search term: %s' % (provider, term))
 
             headers = {'User-Agent': USER_AGENT}
             params = {
@@ -1281,25 +1324,49 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 "q": term
             }
 
-            data = request.request_feed(
+            data = request.request_soup(
-                url=torznab_host[0] + '/api?',
+                url=torznab_host[0],
                 params=params, headers=headers
             )
 
             # Process feed
             if data:
-                if not len(data.entries):
+                items = data.find_all('item')
-                    logger.info(u"No results found from %s for %s", torznab_host[0], term)
+                if not items:
+                    logger.info(u"No results found from %s for %s", provider, term)
                 else:
-                    for item in data.entries:
+                    for item in items:
                         try:
-                            url = item.link
+                            title = item.title.get_text()
-                            title = item.title
+                            url = item.find("link").next_sibling.strip()
-                            size = int(item.links[1]['length'])
+                            seeders = int(item.find("torznab:attr", attrs={"name": "seeders"}).get('value'))
+
+                            # Torrentech hack - size currently not returned, make it up
+                            if 'torrentech' in torznab_host[0]:
+                                if albumlength:
+                                    if 'Lossless' in title:
+                                        size = albumlength / 1000 * 800 * 128
+                                    elif 'MP3' in title:
+                                        size = albumlength / 1000 * 320 * 128
+                                    else:
+                                        size = albumlength / 1000 * 256 * 128
+                                else:
+                                    logger.info('Skipping %s, could not determine size' % title)
+                                    continue
+                            elif item.size:
+                                size = int(item.size.string)
+                            else:
+                                size = int(item.find("torznab:attr", attrs={"name": "size"}).get('value'))
+
                             if all(word.lower() in title.lower() for word in term.split()):
-                                logger.info(
+                                if size < maxsize and minimumseeders < seeders:
-                                    'Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
+                                    logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
                                     resultlist.append((title, size, url, provider, 'torrent', True))
+                                else:
+                                    logger.info(
+                                        '%s is larger than the maxsize or has too little seeders for this category, '
+                                        'skipping. (Size: %i bytes, Seeders: %d)',
+                                        title, size, seeders)
                             else:
                                 logger.info('Skipping %s, not all search term words found' % title)
 
@@ -1307,66 +1374,6 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                             logger.exception(
                                 "An unknown error occurred trying to parse the feed: %s" % e)
 
-    if headphones.CONFIG.KAT:
-        provider = "Kick Ass Torrents"
-        ka_term = term.replace("!", "")
-
-        # Use proxy if specified
-        if headphones.CONFIG.KAT_PROXY_URL:
-            providerurl = fix_url(set_proxy(headphones.CONFIG.KAT_PROXY_URL))
-        else:
-            providerurl = fix_url("https://kat.cr")
-
-        # Build URL
-        providerurl = providerurl + "/usearch/" + ka_term
-
-        # Set max size and category
-        if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:
-            maxsize = 10000000000
-            providerurl += " category:lossless/"
-        elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:
-            maxsize = 10000000000
-            providerurl += " category:music/"
-        else:
-            maxsize = 300000000
-            providerurl += " category:music/"
-
-        # Requesting content
-        logger.info("Searching %s using term: %s" % (provider, ka_term))
-
-        params = {
-            "field": "seeders",
-            "sorder": "desc",
-            "rss": "1"
-        }
-
-        data = request.request_feed(url=providerurl, params=params,
-                                    whitelist_status_code=[404])
-
-        # Process feed
-        if data:
-            if not len(data.entries):
-                logger.info("No results found on %s using search term: %s" % (provider, ka_term))
-            else:
-                for item in data.entries:
-                    try:
-                        title = item['title']
-                        seeders = item['torrent_seeds']
-                        if headphones.CONFIG.TORRENT_DOWNLOADER == 0:
-                            url = item['links'][1]['href']
-                        else:
-                            url = item['torrent_magneturi']
-                        size = int(item['links'][1]['length'])
-                        if size < maxsize and minimumseeders < int(seeders):
-                            resultlist.append((title, size, url, provider, 'torrent', True))
-                            logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
-                        else:
-                            logger.info(
-                                '%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %d)',
-                                title, size, int(seeders))
-                    except Exception as e:
-                        logger.exception("Unhandled exception in the KAT parser")
-
     if headphones.CONFIG.WAFFLES:
         provider = "Waffles.ch"
         providerurl = fix_url("https://waffles.ch/browse.php")
@@ -1391,8 +1398,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             query_items = [usersearchterm]
 
         query_items.extend(['format:(%s)' % format,
-                            'size:[0 TO %d]' % maxsize,
+                            'size:[0 TO %d]' % maxsize])
-                            '-seeders:0'])  # cut out dead torrents
+        # (25/03/2017 Waffles back up after 5 months, all torrents currently have no seeders, remove for now)
+        # '-seeders:0'])  cut out dead torrents
 
         if bitrate:
             query_items.append('bitrate:"%s"' % bitrate)
@@ -1468,9 +1476,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 if rulist:
                     resultlist.extend(rulist)
 
-    if headphones.CONFIG.WHATCD:
+    if headphones.CONFIG.ORPHEUS:
-        provider = "What.cd"
+        provider = "Orpheus.network"
-        providerurl = "http://what.cd/"
+        providerurl = "http://orpheus.network/"
 
         bitrate = None
         bitrate_string = bitrate
@@ -1493,7 +1501,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                         bitrate_string = encoding_string
                 if bitrate_string not in gazelleencoding.ALL_ENCODINGS:
                     logger.info(
-                        u"Your preferred bitrate is not one of the available What.cd filters, so not using it as a search parameter.")
+                        u"Your preferred bitrate is not one of the available Orpheus.network filters, so not using it as a search parameter.")
             maxsize = 10000000000
         elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:  # Highest quality including lossless
             search_formats = [gazelleformat.FLAC, gazelleformat.MP3]
@@ -1502,31 +1510,64 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             search_formats = [gazelleformat.MP3]
             maxsize = 300000000
 
-        if not gazelle or not gazelle.logged_in():
+        if not orpheusobj or not orpheusobj.logged_in():
             try:
-                logger.info(u"Attempting to log in to What.cd...")
+                logger.info(u"Attempting to log in to Orpheus.network...")
-                gazelle = gazelleapi.GazelleAPI(headphones.CONFIG.WHATCD_USERNAME,
+                orpheusobj = gazelleapi.GazelleAPI(headphones.CONFIG.ORPHEUS_USERNAME,
-                                                headphones.CONFIG.WHATCD_PASSWORD,
+                                                headphones.CONFIG.ORPHEUS_PASSWORD,
-                                                headphones.CONFIG.WHATCD_URL)
+                                                headphones.CONFIG.ORPHEUS_URL)
-                gazelle._login()
+                orpheusobj._login()
             except Exception as e:
-                gazelle = None
+                orpheusobj = None
-                logger.error(u"What.cd credentials incorrect or site is down. Error: %s %s" % (
+                logger.error(u"Orpheus.network credentials incorrect or site is down. Error: %s %s" % (
                     e.__class__.__name__, str(e)))
 
-        if gazelle and gazelle.logged_in():
+        if orpheusobj and orpheusobj.logged_in():
             logger.info(u"Searching %s..." % provider)
             all_torrents = []
+
+            album_type = ""
+
+            # Specify release types to filter by
+            if album['Type'] == 'Album':
+                album_type = [gazellerelease_type.ALBUM]
+            if album['Type'] == 'Soundtrack':
+                album_type = [gazellerelease_type.SOUNDTRACK]
+            if album['Type'] == 'EP':
+                album_type = [gazellerelease_type.EP]
+            # No musicbrainz match for this type
+            # if album['Type'] == 'Anthology':
+            #   album_type = [gazellerelease_type.ANTHOLOGY]
+            if album['Type'] == 'Compilation':
+                album_type = [gazellerelease_type.COMPILATION]
+            if album['Type'] == 'DJ-mix':
+                album_type = [gazellerelease_type.DJ_MIX]
+            if album['Type'] == 'Single':
+                album_type = [gazellerelease_type.SINGLE]
+            if album['Type'] == 'Live':
+                album_type = [gazellerelease_type.LIVE_ALBUM]
+            if album['Type'] == 'Remix':
+                album_type = [gazellerelease_type.REMIX]
+            if album['Type'] == 'Bootleg':
+                album_type = [gazellerelease_type.BOOTLEG]
+            if album['Type'] == 'Interview':
+                album_type = [gazellerelease_type.INTERVIEW]
+            if album['Type'] == 'Mixtape/Street':
+                album_type = [gazellerelease_type.MIXTAPE]
+            if album['Type'] == 'Other':
+                album_type = [gazellerelease_type.UNKNOWN]
+
             for search_format in search_formats:
                 if usersearchterm:
                     all_torrents.extend(
-                        gazelle.search_torrents(searchstr=usersearchterm, format=search_format,
+                        orpheusobj.search_torrents(searchstr=usersearchterm, format=search_format,
-                                                encoding=bitrate_string)['results'])
+                                                encoding=bitrate_string, releasetype=album_type)['results'])
                 else:
-                    all_torrents.extend(gazelle.search_torrents(artistname=semi_clean_artist_term,
+                    all_torrents.extend(orpheusobj.search_torrents(artistname=semi_clean_artist_term,
                                                                 groupname=semi_clean_album_term,
                                                                 format=search_format,
-                                                                encoding=bitrate_string)['results'])
+                                                                encoding=bitrate_string,
+                                                                releasetype=album_type)['results'])
 
             # filter on format, size, and num seeders
             logger.info(u"Filtering torrents by format, maximum size, and minimum seeders...")
@@ -1542,13 +1583,11 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             elif len(match_torrents) > 1:
                 logger.info(u"Found %d matching releases from %s for %s - %s after filtering" %
                             (len(match_torrents), provider, artistterm, albumterm))
-                logger.info(
+                logger.info('Sorting torrents by number of seeders...')
-                    "Sorting torrents by times snatched and preferred bitrate %s..." % bitrate_string)
+                match_torrents.sort(key=lambda x: int(x.seeders), reverse=True)
-                match_torrents.sort(key=lambda x: int(x.snatched), reverse=True)
                 if gazelleformat.MP3 in search_formats:
-                    # sort by size after rounding to nearest 10MB...hacky, but will favor highest quality
+                    logger.info('Sorting torrents by seeders...')
-                    match_torrents.sort(key=lambda x: int(10 * round(x.size / 1024. / 1024. / 10.)),
+                    match_torrents.sort(key=lambda x: int(x.seeders), reverse=True)
-                                        reverse=True)
                 if search_formats and None not in search_formats:
                     match_torrents.sort(
                         key=lambda x: int(search_formats.index(x.format)))  # prefer lossless
@@ -1563,14 +1602,14 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                     torrent.group.update_group_data()  # will load the file_path for the individual torrents
                 resultlist.append((torrent.file_path,
                                    torrent.size,
-                                   gazelle.generate_torrent_link(torrent.id),
+                                   orpheusobj.generate_torrent_link(torrent.id),
                                    provider,
                                    'torrent', True))
 
-    # PassTheHeadphones.me - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable
+    # Redacted - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable
-    if headphones.CONFIG.PTH:
+    if headphones.CONFIG.REDACTED:
-        provider = "PassTheHeadphones.me"
+        provider = "Redacted"
-        providerurl = "https://passtheheadphones.me/"
+        providerurl = "https://redacted.ch"
 
         bitrate = None
         bitrate_string = bitrate
@@ -1593,7 +1632,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                         bitrate_string = encoding_string
                 if bitrate_string not in gazelleencoding.ALL_ENCODINGS:
                     logger.info(
-                        u"Your preferred bitrate is not one of the available What.cd filters, so not using it as a search parameter.")
+                        u"Your preferred bitrate is not one of the available RED filters, so not using it as a search parameter.")
             maxsize = 10000000000
         elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:  # Highest quality including lossless
             search_formats = [gazelleformat.FLAC, gazelleformat.MP3]
@@ -1602,28 +1641,28 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             search_formats = [gazelleformat.MP3]
             maxsize = 300000000
 
-        if not gazelle or not gazelle.logged_in():
+        if not redobj or not redobj.logged_in():
             try:
-                logger.info(u"Attempting to log in to PassTheHeadphones.me...")
+                logger.info(u"Attempting to log in to Redacted...")
-                gazelle = gazelleapi.GazelleAPI(headphones.CONFIG.PTH_USERNAME,
+                redobj = gazelleapi.GazelleAPI(headphones.CONFIG.REDACTED_USERNAME,
-                                                headphones.CONFIG.PTH_PASSWORD,
+                                                headphones.CONFIG.REDACTED_PASSWORD,
-                                                headphones.CONFIG.PTH_URL)
+                                                providerurl)
-                gazelle._login()
+                redobj._login()
             except Exception as e:
-                gazelle = None
+                redobj = None
-                logger.error(u"PassTheHeadphones credentials incorrect or site is down. Error: %s %s" % (
+                logger.error(u"Redacted credentials incorrect or site is down. Error: %s %s" % (
                     e.__class__.__name__, str(e)))
 
-        if gazelle and gazelle.logged_in():
+        if redobj and redobj.logged_in():
             logger.info(u"Searching %s..." % provider)
             all_torrents = []
             for search_format in search_formats:
                 if usersearchterm:
                     all_torrents.extend(
-                        gazelle.search_torrents(searchstr=usersearchterm, format=search_format,
+                        redobj.search_torrents(searchstr=usersearchterm, format=search_format,
                                                 encoding=bitrate_string)['results'])
                 else:
-                    all_torrents.extend(gazelle.search_torrents(artistname=semi_clean_artist_term,
+                    all_torrents.extend(redobj.search_torrents(artistname=semi_clean_artist_term,
                                                                 groupname=semi_clean_album_term,
                                                                 format=search_format,
                                                                 encoding=bitrate_string)['results'])
@@ -1661,22 +1700,23 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             for torrent in match_torrents:
                 if not torrent.file_path:
                     torrent.group.update_group_data()  # will load the file_path for the individual torrents
+                use_token = headphones.CONFIG.REDACTED_USE_FLTOKEN and torrent.can_use_token
                 resultlist.append((torrent.file_path,
                                    torrent.size,
-                                   gazelle.generate_torrent_link(torrent.id),
+                                   redobj.generate_torrent_link(torrent.id, use_token),
                                    provider,
                                    'torrent', True))
 
     # Pirate Bay
     if headphones.CONFIG.PIRATEBAY:
         provider = "The Pirate Bay"
-        tpb_term = term.replace("!", "")
+        tpb_term = term.replace("!", "").replace("'", " ")
 
         # Use proxy if specified
         if headphones.CONFIG.PIRATEBAY_PROXY_URL:
             providerurl = fix_url(set_proxy(headphones.CONFIG.PIRATEBAY_PROXY_URL))
         else:
-            providerurl = fix_url("https://thepiratebay.se")
+            providerurl = fix_url("https://thepiratebay.org")
 
         # Build URL
         providerurl = providerurl + "/search/" + tpb_term + "/0/7/"  # 7 is sort by seeders
@@ -1709,7 +1749,6 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 for item in rows:
                     try:
                         url = None
-                        rightformat = True
                         title = ''.join(item.find("a", {"class": "detLink"}))
                         seeds = int(''.join(item.find("td", {"align": "right"})))
 
@@ -1778,7 +1817,6 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                     try:
                         links = item.select("td.title-row a")
 
-                        rightformat = True
                         title = links[1].text
                         seeds = int(item.select("td.seeders-row")[0].text)
                         url = links[0][
@@ -1800,184 +1838,13 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                         logger.error(
                             u"An unknown error occurred in the Old Pirate Bay parser: %s" % e)
 
-    # Strike
-    if headphones.CONFIG.STRIKE:
-        provider = "Strike"
-        s_term = term.replace("!", "")
-        providerurl = fix_url("https://getstrike.net/api/v2/torrents/search/?phrase=")
-
-        providerurl = providerurl + s_term + "&category=Music"
-
-        if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:
-            providerurl = providerurl + "&subcategory=Lossless"
-            maxsize = 10000000000
-        elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:
-            maxsize = 10000000000
-        else:
-            maxsize = 300000000
-
-        logger.info("Searching %s using term: %s" % (provider, s_term))
-        data = request.request_json(url=providerurl,
-                                    whitelist_status_code=[404])
-
-        if not data or not data.get('torrents'):
-            logger.info("No results found on %s using search term: %s" % (provider, s_term))
-        else:
-            for item in data['torrents']:
-                try:
-                    title = item['torrent_title']
-                    seeders = item['seeds']
-                    url = item['magnet_uri']
-                    size = int(item['size'])
-
-                    if size < maxsize and minimumseeders < int(seeders):
-                        resultlist.append((title, size, url, provider, 'torrent', True))
-                        logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
-                    else:
-                        logger.info(
-                            '%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %d)',
-                            title, size, int(seeders))
-                except Exception as e:
-                    logger.exception("Unhandled exception in the Strike parser")
-
-    # Mininova
-    if headphones.CONFIG.MININOVA:
-        provider = "Mininova"
-        providerurl = fix_url("http://www.mininova.org/rss/" + term + "/5")
-
-        if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:
-            # categories = "7"        #music
-            format = "2"  # flac
-            maxsize = 10000000000
-        elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:
-            # categories = "7"        #music
-            format = "10"  # mp3+flac
-            maxsize = 10000000000
-        else:
-            # categories = "7"        #music
-            format = "8"  # mp3
-            maxsize = 300000000
-
-        # Requesting content
-        logger.info('Parsing results from Mininova')
-
-        data = request.request_feed(
-            url=providerurl,
-            timeout=20
-        )
-
-        # Process feed
-        if data:
-            if not len(data.entries):
-                logger.info(u"No results found from %s for %s" % (provider, term))
-            else:
-                for item in data.entries:
-                    try:
-                        rightformat = True
-                        title = item.title
-                        sxstart = item.description.find("Ratio: ") + 7
-                        seeds = ""
-                        while item.description[sxstart:sxstart + 1] != " ":
-                            seeds = seeds + item.description[sxstart:sxstart + 1]
-                            sxstart = sxstart + 1
-                        url = item.links[1]['url']
-                        size = int(item.links[1]['length'])
-                        if format == "2":
-                            torrent = request.request_content(url)
-                            if not torrent or (int(torrent.find(".mp3")) > 0 and int(
-                                    torrent.find(".flac")) < 1):
-                                rightformat = False
-
-                        if rightformat and size < maxsize and minimumseeders < seeds:
-                            match = True
-                            logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
-                        else:
-                            match = False
-                            logger.info('%s is larger than the maxsize, the wrong format or has too little seeders'
-                                        ' for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (
-                                            title, size, int(seeds), rightformat))
-
-                        resultlist.append((title, size, url, provider, 'torrent', match))
-                    except Exception as e:
-                        logger.exception("Unhandled exception in Mininova Parser")
-    # t411
-    if headphones.CONFIG.TQUATTRECENTONZE:
-        username = headphones.CONFIG.TQUATTRECENTONZE_USER
-        password = headphones.CONFIG.TQUATTRECENTONZE_PASSWORD
-        API_URL = "http://api.t411.li"
-        AUTH_URL = API_URL + '/auth'
-        DL_URL = API_URL + '/torrents/download/'
-        provider = "t411"
-        t411_term = term.replace(" ", "%20")
-        SEARCH_URL = API_URL + '/torrents/search/' + t411_term + "?limit=15&cid=395&subcat=623"
-        headers_login = {'username': username, 'password': password}
-
-        # Requesting content
-        logger.info('Parsing results from t411 using search term: %s' % term)
-        req = request.request_content(AUTH_URL, method='post', data=headers_login)
-
-        if len(req.split('"')) == 9:
-            token = req.split('"')[7]
-            headers_auth = {'Authorization': token}
-            logger.info('t411 - User %s logged in' % username)
-        else:
-            logger.info('t411 - Login error : %s' % req.split('"')[3])
-
-        # Quality
-        if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:
-            providerurl = fix_url(SEARCH_URL + "&term[16][]=529&term[16][]=1184")
-        elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:
-            providerurl = fix_url(SEARCH_URL + "&term[16][]=685&term[16][]=527&term[16][]=1070&term[16][]=528&term[16][]=1167&term[16][]=1166&term[16][]=530&term[16][]=529&term[16][]=1184&term[16][]=532&term[16][]=533&term[16][]=1085&term[16][]=534&term[16][]=535&term[16][]=1069&term[16][]=537&term[16][]=538")
-        elif headphones.CONFIG.PREFERRED_QUALITY == 0:
-            providerurl = fix_url(SEARCH_URL + "&term[16][]=685&term[16][]=527&term[16][]=1070&term[16][]=528&term[16][]=1167&term[16][]=1166&term[16][]=530&term[16][]=532&term[16][]=533&term[16][]=1085&term[16][]=534&term[16][]=535&term[16][]=1069&term[16][]=537&term[16][]=538")
-        else:
-            providerurl = fix_url(SEARCH_URL)
-
-        # Tracker search
-        req = request.request_content(providerurl, headers=headers_auth)
-        req = loads(req)
-        total = req['total']
-
-        # Process feed
-        if total == '0':
-            logger.info("No results found from t411 for %s" % term)
-        else:
-            logger.info('Found %s results from t411' % total)
-            torrents = req['torrents']
-            for torrent in torrents:
-                try:
-                    title = torrent['name']
-                    if torrent['seeders'] < minimumseeders:
-                        logger.info('Skipping torrent %s : seeders below minimum set' % title)
-                        continue
-                    id = torrent['id']
-                    size = int(torrent['size'])
-                    data = request.request_content(DL_URL + id, headers=headers_auth)
-
-                    # Blackhole
-                    if headphones.CONFIG.TORRENT_DOWNLOADER == 0 and headphones.CONFIG.MAGNET_LINKS == 2:
-                        url = DL_URL + id + 'TOKEN' + token
-                        resultlist.append((title, size, url, provider, 'torrent', True))
-
-                    # Build magnet
-                    else:
-                        metadata = bdecode(data)
-                        hashcontents = bencode(metadata['info'])
-                        digest = sha1(hashcontents).hexdigest()
-                        trackers = [metadata["announce"]][0]
-                        url = 'magnet:?xt=urn:btih:%s&tr=%s' % (digest, trackers)
-                        resultlist.append((title, size, url, provider, 'torrent', True))
-
-                except Exception as e:
-                    logger.error("Error converting magnet link: %s" % str(e))
-                    return
-
     # attempt to verify that this isn't a substring result
     # when looking for "Foo - Foo" we don't want "Foobar"
     # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
     results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)]
 
     # Additional filtering for size etc
+    # if results and not choose_specific_download and result[3] != 'Orpheus.network':
     if results and not choose_specific_download:
         results = more_filtering(results, album, albumlength, new)
 
@@ -1991,30 +1858,47 @@ def preprocess(resultlist):
     for result in resultlist:
         if result[4] == 'torrent':
 
+            headers = {}
+
             # rutracker always needs the torrent data
             if result[3] == 'rutracker.org':
                 return ruobj.get_torrent_data(result[2]), result
 
-            # Get out of here if we're using Transmission
+            # Jackett sometimes redirects
-            if headphones.CONFIG.TORRENT_DOWNLOADER == 1:  # if not a magnet link still need the .torrent to generate hash... uTorrent support labeling
+            jackett_content = None
+            if result[3].startswith('Jackett_') or 'torznab' in result[3].lower():
+                r = request.request_response(url=result[2], headers=headers, allow_redirects=False)
+                if r:
+                    jackett_content = r.content
+                    link = r.headers.get('Location')
+                    if link and link != result[2]:
+                        if link.startswith('magnet:'):
+                            result = (result[0], result[1], link, result[3], "magnet", result[5])
+                            return "d10:magnet-uri%d:%se" % (len(link), link), result
+                        else:
+                            result = (result[0], result[1], link, result[3], result[4], result[5])
                             return True, result
+
+            # Get out of here if we're using Transmission or Deluge
+            # if not a magnet link still need the .torrent to generate hash... uTorrent support labeling
+            if headphones.CONFIG.TORRENT_DOWNLOADER in [1, 3]:
+                return True, result
+
             # Get out of here if it's a magnet link
             if result[2].lower().startswith("magnet:"):
                 return True, result
 
             # Download the torrent file
-            headers = {}
 
-            if result[3] == 'Kick Ass Torrents':
+            if result[3] == 'Orpheus.network':
-                headers['Referer'] = 'https://torcache.net/'
-                headers['User-Agent'] = USER_AGENT
-            elif result[3] == 'What.cd':
                 headers['User-Agent'] = 'Headphones'
-            elif result[3] == 'PassTheHeadphones.me':
+            elif result[3] == 'Redacted':
                 headers['User-Agent'] = 'Headphones'
             elif result[3] == "The Pirate Bay" or result[3] == "Old Pirate Bay":
-                headers[
+                headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
-                    'User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
+            elif jackett_content:
+                return jackett_content, result
+
             return request.request_content(url=result[2], headers=headers), result
 
         if result[4] == 'magnet':

headphones/torrentfinished.py

@@ -13,13 +13,10 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import threading
 
-from headphones import db, utorrent, transmission, logger
+from headphones import db, utorrent, transmission, deluge, qbittorrent, logger
 import headphones
 
-postprocessor_lock = threading.Lock()
-
 
 def checkTorrentFinished():
     """
@@ -28,18 +25,22 @@ def checkTorrentFinished():
 
     logger.info("Checking if any torrents have finished seeding and can be removed")
 
-    with postprocessor_lock:
     myDB = db.DBConnection()
     results = myDB.select('SELECT * from snatched WHERE Status="Seed_Processed"')
 
     for album in results:
-            hash = album['FolderName']
+        hash = album['TorrentHash']
         albumid = album['AlbumID']
         torrent_removed = False
+
         if headphones.CONFIG.TORRENT_DOWNLOADER == 1:
             torrent_removed = transmission.removeTorrent(hash, True)
-            else:
+        elif headphones.CONFIG.TORRENT_DOWNLOADER == 2:
             torrent_removed = utorrent.removeTorrent(hash, True)
+        elif headphones.CONFIG.TORRENT_DOWNLOADER == 3:
+            torrent_removed = deluge.removeTorrent(hash, True)
+        else:
+            torrent_removed = qbittorrent.removeTorrent(hash, True)
 
         if torrent_removed:
             myDB.action('DELETE from snatched WHERE status = "Seed_Processed" and AlbumID=?',

headphones/transmission.py

@@ -17,6 +17,7 @@ import time
 import json
 import base64
 import urlparse
+import os
 
 from headphones import logger, request
 import headphones
@@ -33,7 +34,7 @@ _session_id = None
 def addTorrent(link, data=None):
     method = 'torrent-add'
 
-    if link.endswith('.torrent') and not link.startswith('http') or data:
+    if link.endswith('.torrent') and not link.startswith(('http', 'magnet')) or data:
         if data:
             metainfo = str(base64.b64encode(data))
         else:
@@ -64,7 +65,32 @@ def addTorrent(link, data=None):
         return False
 
 
-def getTorrentFolder(torrentid):
+def getFolder(torrentid):
+    torrent_folder = None
+    single_file = False
+    method = 'torrent-get'
+    arguments = {'ids': torrentid, 'fields': ['files']}
+
+    response = torrentAction(method, arguments)
+
+    try:
+        torrent_files = response['arguments']['torrents'][0]['files']
+        if torrent_files:
+            if len(torrent_files) == 1:
+                torrent_folder = torrent_files[0]['name']
+                single_file = True
+            else:
+                torrent_folder = os.path.split(torrent_files[0]['name'])[0]
+                torrent_folder = torrent_folder.split(os.sep)[0]
+                single_file = False
+    except:
+        torrent_folder = None
+        single_file = False
+
+    return torrent_folder, single_file
+
+
+def getName(torrentid):
     method = 'torrent-get'
     arguments = {'ids': torrentid, 'fields': ['name', 'percentDone']}
 

headphones/versioncheck.py

@@ -39,14 +39,15 @@ def runGit(args):
 
         try:
             logger.debug('Trying to execute: "' + cmd + '" with shell in ' + headphones.PROG_DIR)
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True,
+            p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                 shell=True,
                                  cwd=headphones.PROG_DIR)
             output, err = p.communicate()
             output = output.strip()
 
             logger.debug('Git output: ' + output)
-        except OSError:
+        except OSError as e:
-            logger.debug('Command failed: %s', cmd)
+            logger.debug('Command failed: %s. Error: %s' % (cmd, e))
             continue
 
         if 'not found' in output or "not recognized as an internal or external command" in output:

headphones/webserve.py

@@ -14,7 +14,6 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
-# t411 support added by a1ex, @likeitneverwentaway on github for maintenance
 
 from operator import itemgetter
 import threading
@@ -252,7 +251,10 @@ class WebInterface(object):
         namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID])
         for name in namecheck:
             artistname = name['ArtistName']
+        try:
             logger.info(u"Deleting all traces of artist: " + artistname)
+        except TypeError:
+            logger.info(u"Deleting all traces of artist: null")
         myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
 
         from headphones import cache
@@ -324,7 +326,7 @@ class WebInterface(object):
                   '$first': firstchar.lower(),
                   }
 
-        folder = helpers.replace_all(folder_format.strip(), values, normalize=True)
+        folder = helpers.pattern_substitute(folder_format.strip(), values, normalize=True)
 
         folder = helpers.replace_illegal_chars(folder, type="folder")
         folder = folder.replace('./', '_/').replace('/.', '/_')
@@ -345,6 +347,7 @@ class WebInterface(object):
 
         dirs = set(dirs)
 
+        try:
             for dir in dirs:
                 artistfolder = os.path.join(dir, folder)
                 if not os.path.isdir(artistfolder.encode(headphones.SYS_ENCODING)):
@@ -353,6 +356,9 @@ class WebInterface(object):
                 threading.Thread(target=librarysync.libraryScan,
                                  kwargs={"dir": artistfolder, "artistScan": True, "ArtistID": ArtistID,
                                          "ArtistName": artist_name}).start()
+        except Exception as e:
+            logger.error('Unable to complete the scan: %s' % e)
+
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
 
     @cherrypy.expose
@@ -534,7 +540,7 @@ class WebInterface(object):
         myDB = db.DBConnection()
         upcoming = myDB.select(
             "SELECT * from albums WHERE ReleaseDate > date('now') order by ReleaseDate ASC")
-        wanted = myDB.select("SELECT * from albums WHERE Status='Wanted'")
+        wanted = myDB.select("SELECT * from albums WHERE Status='Wanted' order by ReleaseDate ASC")
         return serve_template(templatename="upcoming.html", title="Upcoming", upcoming=upcoming,
                               wanted=wanted)
 
@@ -634,6 +640,7 @@ class WebInterface(object):
                     'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
                     [existing_artist])
                 update_count = 0
+                artist_id = None
                 for entry in have_tracks:
                     old_clean_filename = entry['CleanName']
                     if old_clean_filename.startswith(existing_artist_clean):
@@ -642,31 +649,30 @@ class WebInterface(object):
                         myDB.action(
                             'UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?',
                             [new_clean_filename, existing_artist, old_clean_filename])
-                        controlValueDict = {"CleanName": new_clean_filename}
+
-                        newValueDict = {"Location": entry['Location'],
-                                        "BitRate": entry['BitRate'],
-                                        "Format": entry['Format']
-                                        }
                         # Attempt to match tracks with new CleanName
                         match_alltracks = myDB.action(
-                            'SELECT CleanName from alltracks WHERE CleanName=?',
+                            'SELECT CleanName FROM alltracks WHERE CleanName = ?',
                             [new_clean_filename]).fetchone()
                         if match_alltracks:
-                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                            myDB.action(
+                                'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
+                                [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
+
                         match_tracks = myDB.action(
-                            'SELECT CleanName, AlbumID from tracks WHERE CleanName=?',
+                            'SELECT ArtistID, CleanName, AlbumID FROM tracks WHERE CleanName = ?',
                             [new_clean_filename]).fetchone()
                         if match_tracks:
-                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action(
+                                'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
+                                [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
                             myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?',
                                         [new_clean_filename])
                             update_count += 1
-                            # This was throwing errors and I don't know why, but it seems to be working fine.
+                            artist_id = match_tracks['Artist_ID']
-                            # else:
-                            # logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist)
                 logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist))
-                if update_count > 0:
+                if artist_id:
-                    librarysync.update_album_status()
+                    librarysync.update_album_status(ArtistID=artist_id)
             else:
                 logger.info(
                     "Artist %s already named appropriately; nothing to modify" % existing_artist)
@@ -692,29 +698,28 @@ class WebInterface(object):
                             'UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?',
                             [new_clean_filename, existing_artist, existing_album,
                              old_clean_filename])
-                        controlValueDict = {"CleanName": new_clean_filename}
+
-                        newValueDict = {"Location": entry['Location'],
-                                        "BitRate": entry['BitRate'],
-                                        "Format": entry['Format']
-                                        }
                         # Attempt to match tracks with new CleanName
                         match_alltracks = myDB.action(
-                            'SELECT CleanName from alltracks WHERE CleanName=?',
+                            'SELECT CleanName FROM alltracks WHERE CleanName = ?',
                             [new_clean_filename]).fetchone()
                         if match_alltracks:
-                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                            myDB.action(
+                                'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
+                                [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
+
                         match_tracks = myDB.action(
-                            'SELECT CleanName, AlbumID from tracks WHERE CleanName=?',
+                            'SELECT CleanName, AlbumID FROM tracks WHERE CleanName = ?',
                             [new_clean_filename]).fetchone()
                         if match_tracks:
-                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action(
+                                'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
+                                [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
                             myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?',
                                         [new_clean_filename])
                             album_id = match_tracks['AlbumID']
                             update_count += 1
-                            # This was throwing errors and I don't know why, but it seems to be working fine.
+
-                            # else:
-                            # logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string))
                 logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (
                     update_count, new_artist, new_album))
                 if update_count > 0:
@@ -779,6 +784,10 @@ class WebInterface(object):
                 album = tracks['AlbumTitle']
                 track_title = tracks['TrackTitle']
                 if tracks['CleanName'] != original_clean:
+                    artist_id_check = myDB.action('SELECT ArtistID FROM tracks WHERE CleanName = ?',
+                                                 [tracks['CleanName']]).fetchone()
+                    if artist_id_check:
+                        artist_id = artist_id_check[0]
                     myDB.action(
                         'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?',
                         [None, None, None, tracks['CleanName']])
@@ -790,14 +799,14 @@ class WebInterface(object):
                         (original_clean, artist, album, track_title))
                     update_count += 1
             if update_count > 0:
-                librarysync.update_album_status()
+                librarysync.update_album_status(ArtistID=artist_id)
             logger.info("Artist: %s successfully restored to unmatched list" % artist)
 
         elif action == "unmatchAlbum":
             artist = existing_artist
             album = existing_album
             update_clean = myDB.select(
-                'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?',
+                'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched FROM have WHERE ArtistName=? AND AlbumTitle=?',
                 (artist, album))
             update_count = 0
             for tracks in update_clean:
@@ -806,7 +815,7 @@ class WebInterface(object):
                         'TrackTitle']).lower()
                 track_title = tracks['TrackTitle']
                 if tracks['CleanName'] != original_clean:
-                    album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?',
+                    album_id_check = myDB.action('SELECT AlbumID FROM tracks WHERE CleanName = ?',
                                                  [tracks['CleanName']]).fetchone()
                     if album_id_check:
                         album_id = album_id_check[0]
@@ -871,11 +880,19 @@ class WebInterface(object):
     def musicScan(self, path, scan=0, redirect=None, autoadd=0, libraryscan=0):
         headphones.CONFIG.LIBRARYSCAN = libraryscan
         headphones.CONFIG.AUTO_ADD_ARTISTS = autoadd
+
+        try:
+            params = {}
             headphones.CONFIG.MUSIC_DIR = path
             headphones.CONFIG.write()
+        except Exception as e:
+            logger.warn("Cannot save scan directory to config: %s", e)
+            if scan:
+                params = {"dir": path}
+
         if scan:
             try:
-                threading.Thread(target=librarysync.libraryScan).start()
+                threading.Thread(target=librarysync.libraryScan, kwargs=params).start()
             except Exception as e:
                 logger.error('Unable to complete the scan: %s' % e)
         if redirect:
@@ -1145,6 +1162,7 @@ class WebInterface(object):
             "download_scan_interval": headphones.CONFIG.DOWNLOAD_SCAN_INTERVAL,
             "update_db_interval": headphones.CONFIG.UPDATE_DB_INTERVAL,
             "mb_ignore_age": headphones.CONFIG.MB_IGNORE_AGE,
+            "mb_ignore_age_missing": checked(headphones.CONFIG.MB_IGNORE_AGE_MISSING),
             "search_interval": headphones.CONFIG.SEARCH_INTERVAL,
             "libraryscan_interval": headphones.CONFIG.LIBRARYSCAN_INTERVAL,
             "sab_host": headphones.CONFIG.SAB_HOST,
@@ -1157,6 +1175,10 @@ class WebInterface(object):
             "nzbget_password": headphones.CONFIG.NZBGET_PASSWORD,
             "nzbget_category": headphones.CONFIG.NZBGET_CATEGORY,
             "nzbget_priority": headphones.CONFIG.NZBGET_PRIORITY,
+            "qbittorrent_host": headphones.CONFIG.QBITTORRENT_HOST,
+            "qbittorrent_username": headphones.CONFIG.QBITTORRENT_USERNAME,
+            "qbittorrent_password": headphones.CONFIG.QBITTORRENT_PASSWORD,
+            "qbittorrent_label": headphones.CONFIG.QBITTORRENT_LABEL,
             "transmission_host": headphones.CONFIG.TRANSMISSION_HOST,
             "transmission_username": headphones.CONFIG.TRANSMISSION_USERNAME,
             "transmission_password": headphones.CONFIG.TRANSMISSION_PASSWORD,
@@ -1177,6 +1199,7 @@ class WebInterface(object):
             "torrent_downloader_transmission": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 1),
             "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2),
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
+            "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
@@ -1190,6 +1213,7 @@ class WebInterface(object):
             "use_torznab": checked(headphones.CONFIG.TORZNAB),
             "torznab_host": headphones.CONFIG.TORZNAB_HOST,
             "torznab_apikey": headphones.CONFIG.TORZNAB_APIKEY,
+            "torznab_ratio": headphones.CONFIG.TORZNAB_RATIO,
             "torznab_enabled": checked(headphones.CONFIG.TORZNAB_ENABLED),
             "extra_torznabs": headphones.CONFIG.get_extra_torznabs(),
             "use_nzbsorg": checked(headphones.CONFIG.NZBSORG),
@@ -1205,17 +1229,12 @@ class WebInterface(object):
             "torrentblackhole_dir": headphones.CONFIG.TORRENTBLACKHOLE_DIR,
             "download_torrent_dir": headphones.CONFIG.DOWNLOAD_TORRENT_DIR,
             "numberofseeders": headphones.CONFIG.NUMBEROFSEEDERS,
-            "use_kat": checked(headphones.CONFIG.KAT),
-            "kat_proxy_url": headphones.CONFIG.KAT_PROXY_URL,
-            "kat_ratio": headphones.CONFIG.KAT_RATIO,
             "use_piratebay": checked(headphones.CONFIG.PIRATEBAY),
             "piratebay_proxy_url": headphones.CONFIG.PIRATEBAY_PROXY_URL,
             "piratebay_ratio": headphones.CONFIG.PIRATEBAY_RATIO,
             "use_oldpiratebay": checked(headphones.CONFIG.OLDPIRATEBAY),
             "oldpiratebay_url": headphones.CONFIG.OLDPIRATEBAY_URL,
             "oldpiratebay_ratio": headphones.CONFIG.OLDPIRATEBAY_RATIO,
-            "use_mininova": checked(headphones.CONFIG.MININOVA),
-            "mininova_ratio": headphones.CONFIG.MININOVA_RATIO,
             "use_waffles": checked(headphones.CONFIG.WAFFLES),
             "waffles_uid": headphones.CONFIG.WAFFLES_UID,
             "waffles_passkey": headphones.CONFIG.WAFFLES_PASSKEY,
@@ -1224,21 +1243,17 @@ class WebInterface(object):
             "rutracker_user": headphones.CONFIG.RUTRACKER_USER,
             "rutracker_password": headphones.CONFIG.RUTRACKER_PASSWORD,
             "rutracker_ratio": headphones.CONFIG.RUTRACKER_RATIO,
-            "use_whatcd": checked(headphones.CONFIG.WHATCD),
+            "rutracker_cookie": headphones.CONFIG.RUTRACKER_COOKIE,
-            "whatcd_username": headphones.CONFIG.WHATCD_USERNAME,
+            "use_orpheus": checked(headphones.CONFIG.ORPHEUS),
-            "whatcd_password": headphones.CONFIG.WHATCD_PASSWORD,
+            "orpheus_username": headphones.CONFIG.ORPHEUS_USERNAME,
-            "whatcd_ratio": headphones.CONFIG.WHATCD_RATIO,
+            "orpheus_password": headphones.CONFIG.ORPHEUS_PASSWORD,
-            "whatcd_url": headphones.CONFIG.WHATCD_URL,
+            "orpheus_ratio": headphones.CONFIG.ORPHEUS_RATIO,
-            "use_pth": checked(headphones.CONFIG.PTH),
+            "orpheus_url": headphones.CONFIG.ORPHEUS_URL,
-            "pth_username": headphones.CONFIG.PTH_USERNAME,
+            "use_redacted": checked(headphones.CONFIG.REDACTED),
-            "pth_password": headphones.CONFIG.PTH_PASSWORD,
+            "redacted_username": headphones.CONFIG.REDACTED_USERNAME,
-            "pth_ratio": headphones.CONFIG.PTH_RATIO,
+            "redacted_password": headphones.CONFIG.REDACTED_PASSWORD,
-            "pth_url": headphones.CONFIG.PTH_URL,
+            "redacted_ratio": headphones.CONFIG.REDACTED_RATIO,
-            "use_strike": checked(headphones.CONFIG.STRIKE),
+            "redacted_use_fltoken": checked(headphones.CONFIG.REDACTED_USE_FLTOKEN),
-            "strike_ratio": headphones.CONFIG.STRIKE_RATIO,
-            "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE),
-            "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER,
-            "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD,
             "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0),
             "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1),
             "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2),
@@ -1262,6 +1277,8 @@ class WebInterface(object):
             "keep_nfo": checked(headphones.CONFIG.KEEP_NFO),
             "add_album_art": checked(headphones.CONFIG.ADD_ALBUM_ART),
             "album_art_format": headphones.CONFIG.ALBUM_ART_FORMAT,
+            "album_art_min_width": headphones.CONFIG.ALBUM_ART_MIN_WIDTH,
+            "album_art_max_width": headphones.CONFIG.ALBUM_ART_MAX_WIDTH,
             "embed_album_art": checked(headphones.CONFIG.EMBED_ALBUM_ART),
             "embed_lyrics": checked(headphones.CONFIG.EMBED_LYRICS),
             "replace_existing_folders": checked(headphones.CONFIG.REPLACE_EXISTING_FOLDERS),
@@ -1288,6 +1305,7 @@ class WebInterface(object):
             "magnet_links_3": radio(headphones.CONFIG.MAGNET_LINKS, 3),
             "log_dir": headphones.CONFIG.LOG_DIR,
             "cache_dir": headphones.CONFIG.CACHE_DIR,
+            "keep_torrent_files_dir": headphones.CONFIG.KEEP_TORRENT_FILES_DIR,
             "interface_list": interface_list,
             "music_encoder": checked(headphones.CONFIG.MUSIC_ENCODER),
             "encoder": headphones.CONFIG.ENCODER,
@@ -1388,7 +1406,16 @@ class WebInterface(object):
             "email_ssl": checked(headphones.CONFIG.EMAIL_SSL),
             "email_tls": checked(headphones.CONFIG.EMAIL_TLS),
             "email_onsnatch": checked(headphones.CONFIG.EMAIL_ONSNATCH),
-            "idtag": checked(headphones.CONFIG.IDTAG)
+            "idtag": checked(headphones.CONFIG.IDTAG),
+            "slack_enabled": checked(headphones.CONFIG.SLACK_ENABLED),
+            "slack_url": headphones.CONFIG.SLACK_URL,
+            "slack_channel": headphones.CONFIG.SLACK_CHANNEL,
+            "slack_emoji": headphones.CONFIG.SLACK_EMOJI,
+            "slack_onsnatch": checked(headphones.CONFIG.SLACK_ONSNATCH),
+            "join_enabled": checked(headphones.CONFIG.JOIN_ENABLED),
+            "join_onsnatch": checked(headphones.CONFIG.JOIN_ONSNATCH),
+            "join_apikey": headphones.CONFIG.JOIN_APIKEY,
+            "join_deviceid": headphones.CONFIG.JOIN_DEVICEID
         }
 
         for k, v in config.iteritems():
@@ -1433,17 +1460,17 @@ class WebInterface(object):
         checked_configs = [
             "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer",
             "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled",
-            "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay",
+            "use_nzbsorg", "use_omgwtfnzbs", "use_piratebay", "use_oldpiratebay",
-            "use_mininova", "use_waffles", "use_rutracker",
+            "use_waffles", "use_rutracker",
-            "use_whatcd", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate",
+            "use_orpheus", "use_redacted", "redacted_use_fltoken", "preferred_bitrate_allow_lossless",
-            "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
+            "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
             "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
             "embed_album_art", "embed_lyrics",
             "replace_existing_folders", "keep_original_folder", "file_underscores",
             "include_extras", "official_releases_only",
             "wait_until_release_date", "autowant_upcoming", "autowant_all",
             "autowant_manually_added", "do_not_process_unmatched", "keep_torrent_files",
-            "music_encoder",
+            "music_encoder", "mb_ignore_age_missing",
             "encoderlossless", "encoder_multicore", "delete_lossless_files", "growl_enabled",
             "growl_onsnatch", "prowl_enabled",
             "prowl_onsnatch", "xbmc_enabled", "xbmc_update", "xbmc_notify", "lms_enabled",
@@ -1456,7 +1483,8 @@ class WebInterface(object):
             "osx_notify_enabled", "osx_notify_onsnatch", "boxcar_enabled", "boxcar_onsnatch",
             "songkick_enabled", "songkick_filter_enabled",
             "mpc_enabled", "email_enabled", "email_ssl", "email_tls", "email_onsnatch",
-            "customauth", "idtag", "deluge_paused"
+            "customauth", "idtag", "deluge_paused",
+            "join_enabled", "join_onsnatch"
         ]
         for checked_config in checked_configs:
             if checked_config not in kwargs:
@@ -1509,13 +1537,15 @@ class WebInterface(object):
             if len(torznab_number):
                 torznab_api_key = 'torznab_api' + torznab_number
                 torznab_enabled_key = 'torznab_enabled' + torznab_number
+                torznab_ratio_key = 'torznab_ratio' + torznab_number
                 torznab_host = kwargs.get(torznab_host_key, '')
                 torznab_api = kwargs.get(torznab_api_key, '')
                 torznab_enabled = int(kwargs.get(torznab_enabled_key, 0))
-                for key in [torznab_host_key, torznab_api_key, torznab_enabled_key]:
+                torznab_ratio = kwargs.get(torznab_ratio_key, '')
+                for key in [torznab_host_key, torznab_api_key, torznab_enabled_key, torznab_ratio_key]:
                     if key in kwargs:
                         del kwargs[key]
-                extra_torznabs.append((torznab_host, torznab_api, torznab_enabled))
+                extra_torznabs.append((torznab_host, torznab_api, torznab_ratio, torznab_enabled))
 
         # Convert the extras to list then string. Coming in as 0 or 1 (append new extras to the end)
         temp_extras_list = []
@@ -1709,9 +1739,9 @@ class WebInterface(object):
 
     @cherrypy.expose
     def testPlex(self):
-        logger.info(u"Testing plex notifications")
+        logger.info(u"Testing plex update")
         plex = notifiers.Plex()
-        plex.notify("hellooooo", "test album!", "")
+        plex.update()
 
     @cherrypy.expose
     def testPushbullet(self):
@@ -1725,6 +1755,12 @@ class WebInterface(object):
         telegram = notifiers.TELEGRAM()
         telegram.notify("it works!", "lazers pew pew")
 
+    @cherrypy.expose
+    def testJoin(self):
+        logger.info("Testing Join notifications")
+        join = notifiers.JOIN()
+        join.notify("it works!", "Test message")
+
 
 class Artwork(object):
     @cherrypy.expose