Fixed .pep8 for travis build (v0.5.12)

This reverts commit 56de4c63a7.

.gitignore

@@ -1,3 +1,13 @@
+[Tt]est[Rr]esult*
+/cache
+/logs
+.project
+.pydevproject
+
+# coverage generated:
+/cover-html/
+.coverage
+.coveralls.yml
 
 # Compiled source #
 ###################
@@ -61,8 +71,4 @@ Thumbs.db
 obj/
 [Rr]elease*/
 _ReSharper*/
-[Tt]est[Rr]esult*
-/cache
-/logs
-.project
-.pydevproject
+.vscode

.pep8

@@ -1,5 +1,4 @@
 [pep8]
-# E111 indentation is not a multiple of four
 # E121 continuation line under-indented for hanging indent
 # E122 continuation line missing indentation or outdented
 # E124 closing bracket does not match visual indentation
@@ -10,8 +9,7 @@
 # E261 at least two spaces before inline comment
 # E262 inline comment should start with '# '
 # E265 block comment should start with '# '
-# E302 expected 2 blank lines, found 1
 # E501 line too long (312 > 160 characters)
 # E502 the backslash is redundant between brackets
-ignore = E111,E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E302,E501,E502
-max-line-length = 160
+ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E501,E502
+max-line-length = 160

.travis.yml

@@ -2,20 +2,36 @@
 # http://about.travis-ci.org/docs/
 
 language: python
+sudo: false
+
+cache:
+  pip: true
+  directories:
+  - lib
 
 # Available Python versions:
 # http://about.travis-ci.org/docs/user/ci-environment/#Python-VM-images
 python:
   - "2.6"
-  - "2.7"
+matrix:
+  include:
+    - python: "2.7"
+      env: SENDCOVERAGE=1
+
 # pylint 1.4 does not run under python 2.6
 install:
- - pip install pyOpenSSL
- - pip install pylint==1.3.1
- - pip install pyflakes
- - pip install pep8
+  - pip install pyOpenSSL
+  - pip install pylint==1.3.1
+  - pip install pyflakes
+  - pip install pep8
+  # coverage stuff:
+  - pip install coveralls
+  - pip install coverage
 script:
   - pep8 headphones
-  - pylint --rcfile=pylintrc headphones
   - pyflakes headphones
-  - nosetests headphones
+  - nosetests
+
+after_success:
+  # coverage stuff:
+  - if [ $SENDCOVERAGE ]; then coveralls; fi

API.md

@@ -66,7 +66,7 @@ Unmark album as wanted / i.e. mark as skipped
 
 ### forceSearch
 force search for wanted albums - not launched in a separate thread so it may take a bit to complete
-### forceProcess
+### forceProcess[&dir=/path/to/folder]
 Force post process albums in download directory - also not launched in a separate thread
 ### forceActiveArtistsUpdate
 force Active Artist Update - also not launched in a separate thread
@@ -108,4 +108,4 @@ See above.
 Gives you a list of results from searcher.searchforalbum(). Basically runs a normal search, but rather than sorting them and downloading the best result, it dumps the data, which you can then pass on to download_specific_release(). Returns a list of dictionaries with params: title, size, url, provider & kind - all of these values must be passed back to download_specific_release
 
 ### download_specific_release&id=albumid&title=$title&size=$size&url=$url&provider=$provider&kind=$kind
-Allows you to manually pass a choose_specific_download release back to searcher.send_to_downloader()
+Allows you to manually pass a choose_specific_download release back to searcher.send_to_downloader()

CHANGELOG.md

@@ -1,5 +1,64 @@
 # Changelog
 
+## v0.5.12
+Released 25 February 2016
+
+This is mostly a hotfix update
+
+Highlights:
+* Added: Experimental Deluge Support
+* Fixed: Some pep8 stuff
+* Improved: Use curly braces for pathrender optional variables
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.11...v0.5.12).
+
+## v0.5.11
+Released 20 February 2016
+
+Highlights:
+* Added: Soft chroot option
+* Fixed: Post processing temporary directory fix (#2504)
+* Fixed: Ubuntu init script (#2509)
+* Fixed: Image cache uncaught exception (#2485)
+* Improved: $Date/$date variable in folder renaming
+* Improved: Reuse transmission session id
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.10...v0.5.11).
+
+## v0.5.10
+Released 29 January 2016
+
+Highlights:
+* Added: API option to post-process single folders
+* Added: Ability to specify extension when re-encoding
+* Added: Option to stop renaming folders
+* Fixed: Utorrent torrents not being removed (#2385)
+* Fixed: Torznab to transmission
+* Fixed: Magnet folder names in history
+* Fixed: Multiple torcache fixes
+* Fixed: Updated requests & urllib3 to latest versions to fix errors with pyOpenSSL
+* Improved: Use a temporary folder during post-processing
+* Improved: Added verify_ssl_cert option
+* Improved: Fixed track matching progress
+* Improved: pylint, pep8 & pylint fixes
+* Improved: Stop JS links from scrolling to the top of the page
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.9...v0.5.10).
+
+## v0.5.9
+Released 05 September 2015
+
+Highlights:
+* Added: Providers Strike, Jackett, custom Torznabs
+* Added: Option to stop post-processing if no good match found (#2343)
+* Fixed: Blackhole -> Magnet, limit to torcache
+* Fixed: Kat 403 flac error
+* Fixed: Last.fm errors
+* Fixed: Pushover notifications
+* Improved: Rutracker logging, switched to requests lib
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.8...v0.5.9).
+
 ## v0.5.8
 Released 13 July 2015
 
@@ -16,7 +75,7 @@ Highlights:
 * Improved: Set localhost as default
 * Improved: Better single artist scanning
 
-The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.6...v0.5.7).
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.7...v0.5.8).
 
 ## v0.5.7
 Released 01 July 2015

Headphones.py

@@ -152,7 +152,10 @@ def main():
     headphones.DB_FILE = os.path.join(headphones.DATA_DIR, 'headphones.db')
 
     # Read config and start logging
-    headphones.initialize(config_file)
+    try:
+        headphones.initialize(config_file)
+    except headphones.exceptions.SoftChrootError as e:
+        raise SystemExit('FATAL ERROR')
 
     if headphones.DAEMON:
         headphones.daemonize()

README.md

@@ -1,6 +1,9 @@
-#![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
+##![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones
 
-Headphones is an automated music downloader for NZB and Torrent, written in Python. It supports SABnzbd, NZBget, Transmission, µTorrent and Blackhole.
+**Master Branch:**  [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=master)](https://travis-ci.org/rembo10/headphones)
+**Develop Branch:** [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=develop)](https://travis-ci.org/rembo10/headphones)
+
+Headphones is an automated music downloader for NZB and Torrent, written in Python. It supports SABnzbd, NZBget, Transmission, µTorrent, Deluge and Blackhole.
 
 ## Support & Discuss
 You are free to join the Headphones support community on IRC where you can ask questions, hang around and discuss anything related to HP.
@@ -18,8 +21,8 @@ You are free to join the Headphones support community on IRC where you can ask q
 
 1. Analyze your log, you just might find the solution yourself!
 2. You read the wiki and searched existing issues, but this is not solving your problem.
-3. Post the issue with a clear title, description and the HP log and use [proper markdown syntax](https://help.github.com/articles/github-flavored-markdown) to structure your text (code/log in code blocks). 
-4. Close your issue when it's solved! If you found the solution yourself please comment so that others benefit from it.
+3. Post the issue with a clear title, description and the HP log and use [proper markdown syntax](https://help.github.com/articles/github-flavored-markdown) to structure your text (code/log in code blocks).
+4. Close your issue when it's solved! If you found the solution yourself, please comment so that others benefit from it.
 
 **Feature requests** can be reported on the GitHub issue tracker too:
 

contrib/sni_test.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+# Ensure that we use the Headphones provided libraries.
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib"))
+
+import urlparse
+
+
+def can_import(module):
+    """
+    Return True if a given module can be imported or not.
+    """
+
+    try:
+        __import__(module)
+    except ImportError:
+        return False
+
+    # Module can be imported
+    return True
+
+
+def check_installation():
+    """
+    Check if some core modules are available. Info is based on this topic:
+    https://github.com/rembo10/headphones/issues/2210.
+    """
+
+    if can_import("requests"):
+        import requests
+        requests_version = requests.__version__
+    else:
+        requests_version = "no"
+
+    if can_import("OpenSSL"):
+        import OpenSSL
+        openssl_version = OpenSSL.__version__
+    else:
+        openssl_version = "no"
+
+    if can_import("cryptography"):
+        import cryptography
+        cryptography_version = cryptography.__version__
+    else:
+        cryptography_version = "no"
+
+    if can_import("pyasn1"):
+        import pyasn1
+        pyasn1_version = pyasn1.__version__
+    else:
+        pyasn1_version = "no"
+
+    if can_import("ndg.httpsclient"):
+        from ndg import httpsclient
+        ndg_version = httpsclient.__date__
+    else:
+        ndg_version = "no"
+
+    # Print some system information.
+    sys.stdout.write(
+        "* Checking Python version: %s.%s.%s\n" % sys.version_info[:3])
+    sys.stdout.write("* Operating system: %s\n" % sys.platform)
+
+    sys.stdout.write(
+        "* Checking if requests can be imported: %s\n" % requests_version)
+    sys.stdout.write(
+        "* Checking if pyOpenSSL is installed: %s\n" % openssl_version)
+    sys.stdout.write(
+        "* Checking if cryptography is installed: %s\n" % cryptography_version)
+    sys.stdout.write(
+        "* Checking if pyasn1 is installed: %s\n" % pyasn1_version)
+    sys.stdout.write(
+        "* Checking if ndg.httpsclient is installed: %s\n" % ndg_version)
+
+
+def main():
+    """
+    Test if the current Headphones installation can connect to SNI-enabled
+    servers.
+    """
+
+    # Read the URL to test.
+    if len(sys.argv) == 1:
+        url = "https://sni.velox.ch/"
+    else:
+        url = sys.argv[1]
+
+    # Check if it is a HTTPS website.
+    parts = urlparse.urlparse(url)
+
+    if parts.scheme.lower() != "https":
+        sys.stderr.write(
+            "Error: provided URL does not start with https://\n")
+        return 1
+
+    # Gather information
+    check_installation()
+
+    # Do the request.
+    if not can_import("requests"):
+        sys.stderr.exit("Error: cannot continue without requests module!\n")
+        return 1
+
+    sys.stdout.write("* Performing request: %s\n" % url)
+
+    import requests
+    requests.packages.urllib3.disable_warnings()
+
+    try:
+        try:
+            response = requests.get(url)
+        except requests.exceptions.SSLError as e:
+            sys.stdout.write(
+                "- Server certificate seems invalid. I will disable "
+                "certificate check and try again. You'll see the real "
+                "exception if it fails again.\n")
+            sys.stdout.write(
+                "* Retrying request with certificate verification off.\n")
+            response = requests.get(url)
+    except Exception as e:
+        sys.stdout.write(
+            "- An error occured while performing the request. The "
+            "exception was: %s\n" % e.message)
+        sys.stdout.write(
+            "- Consult the Troubleshooting wiki (https://github.com/"
+            "rembo10/headphones/wiki/Troubleshooting) before you post an "
+            "issue!")
+        return 0
+
+    # Verify the response.
+    if response.status_code == 200:
+        sys.stdout.write("+ Got a valid response. All seems OK!\n")
+    else:
+        sys.stdout.write(
+            "- Server returned status code %s. Expected a status code 200.\n",
+            response.status_code)
+        sys.stdout.write(
+            "- However, I was able to communicate to the server!\n")
+
+# E.g. `python sni_test.py https://example.org'.
+if __name__ == "__main__":
+    sys.exit(main())

data/interfaces/default/album.html

@@ -14,15 +14,15 @@
 
 			<a id="menu_link_delete" href="deleteAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}"><i class="fa fa-trash-o"></i> Delete Album</a>
 			%if album['Status'] == 'Skipped' or album['Status'] == 'Ignored':
-			<a id="menu_link_wanted" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=False', $(this),true)" data-success="'${album['AlbumTitle']}' added to queue"><i class="fa fa-heart"></i> Mark Album as Wanted</a>
+			<a id="menu_link_wanted" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=False', $(this),true)" data-success="'${album['AlbumTitle']}' added to queue"><i class="fa fa-heart"></i> Mark Album as Wanted</a>
 			%elif album['Status'] == 'Wanted':
-			<a id="menu_link_check" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this));" data-success="Forced checking successful"><i class="fa fa-search"></i> Force Check</a>
-			<a id="menu_link_skipped" href="#" onclick="doAjaxCall('unqueueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),true);" data-success="'${album['AlbumTitle']}' marked as Skipped"><i class="fa fa-step-forward"></i> Mark Album as Skipped</a>
+			<a id="menu_link_check" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this));" data-success="Forced checking successful"><i class="fa fa-search"></i> Force Check</a>
+			<a id="menu_link_skipped" href="javascript:void(0)" onclick="doAjaxCall('unqueueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),true);" data-success="'${album['AlbumTitle']}' marked as Skipped"><i class="fa fa-step-forward"></i> Mark Album as Skipped</a>
 			%else:
-			<a id="menu_link_retry" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=False', $(this),true);" data-success="Retrying the same version of '${album['AlbumTitle']}'"><i class="fa fa-refresh"></i> Retry Download</a>
-			<a id="menu_link_new" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this),true);" data-success="Looking for a new version of '${album['AlbumTitle']}'"><i class="fa fa-download"></i> Try New Version</a>
+			<a id="menu_link_retry" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=False', $(this),true);" data-success="Retrying the same version of '${album['AlbumTitle']}'"><i class="fa fa-refresh"></i> Retry Download</a>
+			<a id="menu_link_new" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this),true);" data-success="Looking for a new version of '${album['AlbumTitle']}'"><i class="fa fa-download"></i> Try New Version</a>
 			%endif		
-			<a class="menu_link_edit" id="album_chooser" href="#"><i class="fa fa-pencil"></i> Choose Alternate Release</a>
+			<a class="menu_link_edit" id="album_chooser" href="javascript:void(0)"><i class="fa fa-pencil"></i> Choose Alternate Release</a>
 				<div id="dialog" title="Choose an Alternate Release" style="display:none" class="configtable">
 					<div class="links">
 						<%
@@ -30,7 +30,7 @@
 						%>
 						%if not alternate_albums:
 							<p>No alternate releases found. Try refreshing the artist (if the artist is being refreshed, please wait until it's finished)</p>
-							<h2><a id="refresh_artist" onclick="doAjaxCall('refreshArtist?ArtistID=${album['ArtistID']}', $(this)), true" href="#" data-success="'${album['ArtistName']}' is being refreshed">Refresh Artist</a></h2>
+							<h2><a id="refresh_artist" onclick="doAjaxCall('refreshArtist?ArtistID=${album['ArtistID']}', $(this)), true" href="javascript:void(0)" data-success="'${album['ArtistName']}' is being refreshed">Refresh Artist</a></h2>
 						%else:
 							%for alternate_album in alternate_albums:
 								<%
@@ -43,12 +43,12 @@
 										alternate_album_name = alternate_album['AlbumTitle'] + " (" + alternate_album['ReleaseCountry'] + ", " + str(alternate_album['ReleaseDate']) + ", " + alternate_album['ReleaseFormat'] + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
 									
 								%>
-								<a href="#" onclick="doAjaxCall('switchAlbum?AlbumID=${album['AlbumID']}&ReleaseID=${alternate_album['ReleaseID']}', $(this), 'table');" data-success="Switched release to: ${alternate_album_name}">${alternate_album_name}</a><a href="${mb_link}" target="_blank">MB</a><br>
+								<a href="javascript:void(0)" onclick="doAjaxCall('switchAlbum?AlbumID=${album['AlbumID']}&ReleaseID=${alternate_album['ReleaseID']}', $(this), 'table');" data-success="Switched release to: ${alternate_album_name}">${alternate_album_name}</a><a href="${mb_link}" target="_blank">MB</a><br>
 							%endfor
 						%endif
 					</div>
 				</div>
-			<a class="menu_link_edit" id="edit_search_term" href="#"><i class="fa fa-pencil"></i> Edit Search Term</a>
+			<a class="menu_link_edit" id="edit_search_term" href="javascript:void(0)"><i class="fa fa-pencil"></i> Edit Search Term</a>
 			<div id="dialog2" title="Enter your own search term for this album" style="display:none" class="configtable">
 				<form action="editSearchTerm" method="GET" id="editSearchTerm">
 					<input type="hidden" name="AlbumID" value="${album['AlbumID']}">
@@ -64,7 +64,7 @@
 					<input type="button" value="Save changes" onclick="doAjaxCall('editSearchTerm',$(this),'tabs',true);return false;" data-success="Search term updated"/>
 				</form>
 			</div>
-			<a class="menu_link_edit" id="choose_specific_download" href="#" onclick="getAvailableDownloads()"><i class="fa fa-search"></i> Choose Specific Download</a>
+			<a class="menu_link_edit" id="choose_specific_download" href="javascript:void(0)" onclick="getAvailableDownloads()"><i class="fa fa-search"></i> Choose Specific Download</a>
 			<div id="choose_specific_download_dialog" title="Choose a specific download for this album" style="display:none" class="configtable">
 				<table class="display" id="downloads_table">
 					<thead>
@@ -238,7 +238,7 @@
 			feedback.fadeOut();
 			search_results = data
 			for( var i = 0, len = data.length; i < len; i++ ) {
-    			$('#downloads_table_body').append('<tr><td id="title"><a href="#" onclick="downloadSpecificRelease('+i+')">'+data[i].title+'</a></td><td id="size"><span title='+data[i].size+'></span>'+(data[i].size / (1024*1024)).toFixed(2)+' MB</td><td id="provider">'+data[i].provider+'</td><td id="kind">'+data[i].kind+'</td><td id="matches">'+data[i].matches+'</td></tr>');
+    			$('#downloads_table_body').append('<tr><td id="title"><a href="javascript:void(0)" onclick="downloadSpecificRelease('+i+')">'+data[i].title+'</a></td><td id="size"><span title='+data[i].size+'></span>'+(data[i].size / (1024*1024)).toFixed(2)+' MB</td><td id="provider">'+data[i].provider+'</td><td id="kind">'+data[i].kind+'</td><td id="matches">'+data[i].matches+'</td></tr>');
     		}
 			$('#downloads_table').dataTable({
 				"aoColumns": [

data/interfaces/default/artist.html

@@ -8,19 +8,19 @@
 <%def name="headerIncludes()">
 	<div id="subhead_container">
 		<div id="subhead_menu">
-			<a id="menu_link_refresh" onclick="doSimpleAjaxCall('refreshArtist?ArtistID=${artist['ArtistID']}')" href="#"><i class="fa fa-refresh"></i> Refresh Artist</a>
+			<a id="menu_link_refresh" onclick="doSimpleAjaxCall('refreshArtist?ArtistID=${artist['ArtistID']}')" href="javascript:void(0)"><i class="fa fa-refresh"></i> Refresh Artist</a>
 			<a id="menu_link_delete" href="deleteArtist?ArtistID=${artist['ArtistID']}"><i class="fa fa-trash-o"></i> Delete Artist</a>
-			<a id="menu_link_scan" onclick="doAjaxCall('scanArtist?ArtistID=${artist['ArtistID']}', $(this)),'table'" href="#" data-success="'${artist['ArtistName']}' was scanned"><i class="fa fa-refresh"></i> Scan Artist</a>
+			<a id="menu_link_scan" onclick="doAjaxCall('scanArtist?ArtistID=${artist['ArtistID']}', $(this)),'table'" href="javascript:void(0)" data-success="'${artist['ArtistName']}' was scanned"><i class="fa fa-refresh"></i> Scan Artist</a>
 			%if artist['Status'] == 'Paused':
-				<a id="menu_link_resume" href="#" onclick="doAjaxCall('resumeArtist?ArtistID=${artist['ArtistID']}',$(this),true)" data-success="${artist['ArtistName']} resumed"><i class="fa fa-play"></i> Resume Artist</a>
+				<a id="menu_link_resume" href="javascript:void(0)" onclick="doAjaxCall('resumeArtist?ArtistID=${artist['ArtistID']}',$(this),true)" data-success="${artist['ArtistName']} resumed"><i class="fa fa-play"></i> Resume Artist</a>
 			%else:
-				<a id="menu_link_pauze" href="#" onclick="doAjaxCall('pauseArtist?ArtistID=${artist['ArtistID']}',$(this),true)" data-success="${artist['ArtistName']} paused"><i class="fa fa-pause"></i> Pause Artist</a>
+				<a id="menu_link_pauze" href="javascript:void(0)" onclick="doAjaxCall('pauseArtist?ArtistID=${artist['ArtistID']}',$(this),true)" data-success="${artist['ArtistName']} paused"><i class="fa fa-pause"></i> Pause Artist</a>
 			%endif
 			%if artist['IncludeExtras']:
-				<a id="menu_link_removeextra" href="#" onclick="doAjaxCall('removeExtras?ArtistID=${artist['ArtistID']}&ArtistName=${artist['ArtistName']}',$(this),'submenu&table')" data-success="Extras removed for ${artist['ArtistName']}"><i class="fa fa-minus"></i> Remove Extras</a>
-				<a class="menu_link_edit" id="menu_link_modifyextra" href="#"><i class="fa fa-pencil"></i> Modify Extras</a>
+				<a id="menu_link_removeextra" href="javascript:void(0)" onclick="doAjaxCall('removeExtras?ArtistID=${artist['ArtistID']}&ArtistName=${artist['ArtistName']}',$(this),'submenu&table')" data-success="Extras removed for ${artist['ArtistName']}"><i class="fa fa-minus"></i> Remove Extras</a>
+				<a class="menu_link_edit" id="menu_link_modifyextra" href="javascript:void(0)"><i class="fa fa-pencil"></i> Modify Extras</a>
 			%else:
-				<a id="menu_link_getextra" href="#"><i class="fa fa-plus"></i> Get Extras</a>
+				<a id="menu_link_getextra" href="javascript:void(0)"><i class="fa fa-plus"></i> Get Extras</a>
 			%endif
 				<div id="dialog" title="Choose Which Extras to Fetch" style="display:none" class="configtable">
 					<form action="getExtras" method="get" class="form">
@@ -129,16 +129,16 @@
 				<td id="score">${album['CriticScore']}/${album['UserScore']}</td>
 				<td id="status">${album['Status']}
 				%if album['Status'] == 'Skipped' or album['Status'] == 'Ignored':
-					[<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}',$(this),'table')" data-success="'${album['AlbumTitle']}' added to Wanted list">want</a>]
+					[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}',$(this),'table')" data-success="'${album['AlbumTitle']}' added to Wanted list">want</a>]
 				%elif (album['Status'] == 'Wanted' or album['Status'] == 'Wanted Lossless'):
-                    [<a href="#" onclick="doAjaxCall('unqueueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}',$(this),'table')" data-success="'${album['AlbumTitle']}' skipped">skip</a>] [<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),'table')" data-success="Trying to download'${album['AlbumTitle']}'" title="Search if available for download">search</a>]
+                    [<a href="javascript:void(0)" onclick="doAjaxCall('unqueueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}',$(this),'table')" data-success="'${album['AlbumTitle']}' skipped">skip</a>] [<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),'table')" data-success="Trying to download'${album['AlbumTitle']}'" title="Search if available for download">search</a>]
 				%else:
-					[<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),'table')" data-success="Retrying the same version of '${album['AlbumTitle']}'" title="Retry the same download again">retry</a>][<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this),'table')" title="Try a new download, skipping all previously tried nzbs"  data-success="Downloading new version for '${album['AlbumTitle']}'" data-success="Looking for a new version of '${album['AlbumTitle']}'">new</a>]
+					[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}', $(this),'table')" data-success="Retrying the same version of '${album['AlbumTitle']}'" title="Retry the same download again">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True', $(this),'table')" title="Try a new download, skipping all previously tried nzbs"  data-success="Downloading new version for '${album['AlbumTitle']}'" data-success="Looking for a new version of '${album['AlbumTitle']}'">new</a>]
 				%endif
 				%if albumformat in lossy_formats and album['Status'] == 'Skipped':
-					[<a id="wantlossless" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&lossless=True', $(this),'table')" data-success="Lossless version of '${album['AlbumTitle']}' added to queue">want lossless</a>]
+					[<a id="wantlossless" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&lossless=True', $(this),'table')" data-success="Lossless version of '${album['AlbumTitle']}' added to queue">want lossless</a>]
 				%elif albumformat in lossy_formats and (album['Status'] == 'Snatched' or album['Status'] == 'Downloaded'):
-					[<a id="wantlossless" href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&lossless=True', $(this),'table')" data-success="Retrying the same lossless version of '${album['AlbumTitle']}'">retry lossless</a>]
+					[<a id="wantlossless" href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&lossless=True', $(this),'table')" data-success="Retrying the same lossless version of '${album['AlbumTitle']}'">retry lossless</a>]
 				%endif
 				</td>
 				<td id="have"><span title="${percent}"><span><div class="progress-container"><div style="width:${percent}%"><div class="havetracks">${havetracks}/${totaltracks}</div></div></div></td>

data/interfaces/default/base.html

@@ -34,11 +34,11 @@
 		% if headphones.CONFIG.CHECK_GITHUB and not headphones.CURRENT_VERSION:
 			<div id="updatebar">
 				You're running an unknown version of Headphones. <a href="update">Update</a> or
-				<a href="#" onclick="$('#updatebar').slideUp('slow');">Close</a>
+				<a href="javascript:void(0)" onclick="$('#updatebar').slideUp('slow');">Close</a>
 			</div>
 		% elif headphones.CONFIG.CHECK_GITHUB and headphones.CURRENT_VERSION != headphones.LATEST_VERSION and headphones.COMMITS_BEHIND > 0 and headphones.INSTALL_TYPE != 'win':
 			<div id="updatebar">
-				A <a href="https://github.com/${headphones.CONFIG.GIT_USER}/headphones/compare/${headphones.CURRENT_VERSION}...${headphones.LATEST_VERSION}"> newer version</a> is available. You're ${headphones.COMMITS_BEHIND} commits behind. <a href="update">Update</a> or <a href="#" onclick="$('#updatebar').slideUp('slow');">Close</a>
+				A <a href="https://github.com/${headphones.CONFIG.GIT_USER}/headphones/compare/${headphones.CURRENT_VERSION}...${headphones.LATEST_VERSION}"> newer version</a> is available. You're ${headphones.COMMITS_BEHIND} commits behind. <a href="update">Update</a> or <a href="javascript:void(0)" onclick="$('#updatebar').slideUp('slow');">Close</a>
 			</div>
 		% endif
 
@@ -89,7 +89,7 @@
 				<small>
 				<a href="shutdown"><i class="fa fa-power-off"></i> Shutdown</a> |
 				<a href="restart"><i class="fa fa-power-off"></i> Restart</a> |
-				<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><i class="fa fa-refresh"></i> Check for new version</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><i class="fa fa-refresh"></i> Check for new version</a>
 				</small>
 			</div>
 			<div id="version">

data/interfaces/default/config.html

@@ -311,6 +311,7 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_blackhole" value="0" ${config['torrent_downloader_blackhole']}> Black Hole
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
                                 <div class="row">
@@ -336,9 +337,14 @@
                                         <input type="radio" name="magnet_links" id="magnet_links_2" value="2" ${config['magnet_links_2']}>
                                         Convert
                                     </label>
+
+                                    <label class="inline">
+                                        <input type="radio" name="magnet_links" id="magnet_links_3" value="3" ${config['magnet_links_3']}>
+                                        Embed
+                                    </label>
                                     <div style="clear: both"></div>
 
-                                    <small>Note: opening magnet URL's is not suitable for headless/console/terminal servers.</small>
+                                    <small>Note: Opening magnet URLs is not suitable for headless/console/terminal servers.<br />Embed only works for rTorrent.</small>
                                 </div>
                             </fieldset>
                             <fieldset id="transmission_options">
@@ -380,6 +386,35 @@
                                     <input type="text" name="utorrent_label" value="${config['utorrent_label']}" size="30">
                                 </div>
                             </fieldset>
+                            <fieldset id="deluge_options">
+                                <div class="row">
+                                    <label>Deluge WebUI Host and Port</label>
+                                    <input type="text" name="deluge_host" value="${config['deluge_host']}" size="30">
+                                    <small>Usually http://localhost:8112 (requires WebUI plugin)</small>
+                                </div>
+                                <div class="row">
+                                    <label>Deluge Password</label>
+                                    <input type="password" name="deluge_password" value="${config['deluge_password']}" size="30">
+                                </div>
+                                <div class="row">
+                                    <small>Note: With Deluge, you can specify a different download directory for downloads sent from Headphones.
+                                    Set it in the Music Download Directory below</small>
+                                </div>
+                                <div class="row">
+                                    <label>Deluge Label</label>
+                                    <input type="text" name="deluge_label" value="${config['deluge_label']}" size="30">
+                                    <small>Labels shouldn't contain spaces (requires Label plugin)</small>
+                                </div>
+                                <div class="row">
+                                    <label>Move When Completed</label>
+                                    <input type="text" name="deluge_done_directory" value="${config['deluge_done_directory']}" size="30">
+                                    <small>Directory where Deluge should move completed downloads</small>
+                                </div>
+                                <div class="row checkbox">
+                                    <label>Add Torrent Paused</label>
+                                    <input type="checkbox" name="deluge_paused" value="1" ${config['deluge_paused']}>
+                                </div>
+                            </fieldset>
                             <fieldset id="general_torrent_options">
                                 <div class="row">
                                     <label>Minimum seeders</label>
@@ -533,22 +568,6 @@
                                     </div>
                                 </fieldset>
 
-                                <fieldset>
-                                    <div class="row checkbox left">
-                                        <input id="use_oldpiratebay" type="checkbox" class="bigcheck" name="use_oldpiratebay" value="1" ${config['use_oldpiratebay']} /><label for="use_oldpiratebay"><span class="option">Old Pirate Bay</span></label>
-                                    </div>
-                                    <div class="config">
-                                        <div class="row">
-                                            <label>URL</label>
-                                            <input type="text" name="oldpiratebay_url" value="${config['oldpiratebay_url']}" size="36">
-                                        </div>
-                                        <div class="row">
-                                            <label>Seed Ratio</label>
-                                            <input type="text" class="override-float" name="oldpiratebay_ratio" value="${config['oldpiratebay_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
-                                        </div>
-                                    </div>
-                                </fieldset>
-
                                 <fieldset>
                                     <div class="row checkbox left">
                                         <input id="use_kat" type="checkbox" class="bigcheck" name="use_kat" value="1" ${config['use_kat']} /><label for="use_kat"><span class="option">Kick Ass Torrents</span></label>
@@ -626,6 +645,87 @@
                                     </div>
                                 </fieldset>
 
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_strike" type="checkbox" class="bigcheck" name="use_strike" value="1" ${config['use_strike']} /><label for="use_strike"><span class="option">Strike Search</span></label>
+                                    </div>
+                                    <div class="config">
+                                        <div class="row">
+                                            <label>Seed Ratio</label>
+                                            <input type="text" class="override-float" name="strike_ratio" value="${config['strike_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                        </div>
+                                    </div>
+                                </fieldset>
+
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_torznab" type="checkbox" class="bigcheck" name="use_torznab" value="1" ${config['use_torznab']} /><label for="use_torznab"><span class="option">Jackett / Torznab Providers</span></label>
+                                    </div>
+                                    <div id="torznab_providers">
+                                        <div class="config" id="torznab1">
+                                            <div class="row">
+                                                <label>Torznab Host</label>
+                                                <input type="text" name="torznab_host" value="${config['torznab_host']}" size="30">
+                                                <small>e.g. http://localhost:9117/torznab/iptorrents</small>
+                                            </div>
+                                            <div class="row">
+                                                <label>Torznab API</label>
+                                                <input type="text" name="torznab_apikey" value="${config['torznab_apikey']}" size="36">
+                                            </div>
+                                            <div class="row checkbox">
+                                                <input id="torznab_enabled" type="checkbox" name="torznab_enabled" value="1" ${config['torznab_enabled']} /><label>Enabled</label>
+                                            </div>
+                                        </div>
+                                        <%
+                                            torznab_number = 2
+                                        %>
+                                        %for torznab in config['extra_torznabs']:
+                                            <%
+                                                if torznab[2] == '1' or torznab[2] == 1:
+                                                    torznab_enabled = "checked"
+                                                else:
+                                                    torznab_enabled = ""
+                                            %>
+                                            <div class="config" id="torznab${torznab_number}">
+                                                <div class="row">
+                                                    <label>Torznab Host</label>
+                                                    <input type="text" name="torznab_host${torznab_number}" value="${torznab[0]}" size="30">
+                                                </div>
+                                                <div class="row">
+                                                    <label>Torznab API</label>
+                                                    <input type="text" name="torznab_api${torznab_number}" value="${torznab[1]}" size="36">
+                                                </div>
+                                                <div class="row checkbox">
+                                                    <input id="torznab_enabled" type="checkbox" name="torznab_enabled${torznab_number}" value="1" ${torznab_enabled} /><label>Enabled</label>
+                                                </div>
+                                                <div class="row">
+                                                    <input type="button" class="remove_torznab" id="torznab${torznab_number}" value="Remove ${torznab[0]}">
+                                                </div>
+                                            </div>
+                                            <%
+                                                torznab_number += 1
+                                            %>
+                                        %endfor
+                                        <input type="button" value="Add Torznab" class="add_torznab" id="add_torznab" />
+                                    </div>
+                                </fieldset>
+
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_oldpiratebay" type="checkbox" class="bigcheck" name="use_oldpiratebay" value="1" ${config['use_oldpiratebay']} /><label for="use_oldpiratebay"><span class="option">Old Pirate Bay</span></label>
+                                    </div>
+                                    <div class="config">
+                                        <div class="row">
+                                            <label>URL</label>
+                                            <input type="text" name="oldpiratebay_url" value="${config['oldpiratebay_url']}" size="36">
+                                        </div>
+                                        <div class="row">
+                                            <label>Seed Ratio</label>
+                                            <input type="text" class="override-float" name="oldpiratebay_ratio" value="${config['oldpiratebay_ratio']}" size="10" title="Stop seeding when ratio met, 0 = unlimited. Scheduled job will remove torrent when post processed and finished seeding.">
+                                        </div>
+                                    </div>
+                                </fieldset>
+
                                 <fieldset>
                                      <div class="row checkbox left">
                                         <input id="use_mininova" type="checkbox" class="bigcheck" name="use_mininova" value="1" ${config['use_mininova']} /><label for="use_mininova"><span class="option">Mininova</span></label>
@@ -683,7 +783,7 @@
                                 </div>
                                 <div id="preferred_bitrate_options" class="suboptions">
                                     <div class="row nopad">
-                                        Target bitrate: 
+                                        Target bitrate:
                                         <input type="text" class="override-float" name="preferred_bitrate" value="${config['preferred_bitrate']}" size="3">kbps<br>
                                     </div>
                                     <div class="row nopad">
@@ -788,7 +888,7 @@
                                     <div class="row">
                                         as <input type="text" class="override-float" name="album_art_format" value="${config['album_art_format']}" size="10">.jpg
                                     </div>
-                                    <small>Use $Artist/$artist, $Album/$album, $Year/$year</small>
+                                    <small>Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in square brackets, use single-quote marks to escape square brackets literally ('[', ']').</small>
                                 </div>
                                 <div class="row checkbox left clearfix nopad">
                                     <label>
@@ -1025,7 +1125,7 @@
                                     </div>
                                     <div class="row">
                                         <label>Plex Token</label><input type="text" name="plex_token" value="${config['plex_token']}" size="30">
-                              <small>Plex Token (for use with Plex Home)</small>
+                                        <small>Plex Token (for use with Plex Home)</small>
                                     </div>
                                     <div class="checkbox row">
                                         <input type="checkbox" name="plex_update" value="1" ${config['plex_update']} /><label>Update Plex Library</label>
@@ -1152,6 +1252,23 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <div class="row checkbox left">
+                                    <input type="checkbox" class="bigcheck" name="telegram_enabled" id="telegram" value="1" ${config['telegram_enabled']} /><label for="telegram"><span class="option">Telegram</span></label>
+                                </div>
+                                <div id="telegramoptions">
+                                    <div class="row">
+                                        <label>Bot Token</label><input type="text" name="telegram_token" value="${config['telegram_token']}" size="50"><small>Contact <a href="http://telegram.me/BotFather">@BotFather</a> to create a bot and get its token</small>
+                                    </div>
+                                    <div class="row">
+                                        <label>User ID</label><input type="text" name="telegram_userid" value="${config['telegram_userid']}" size="50"><small>Contact <a href="http://telegram.me/myidbot">@myidbot</a> to get your user ID</small>
+                                    </div>
+                                    <div class="row checkbox">
+                                        <input type="checkbox" name="telegram_onsnatch" value="1" ${config['telegram_onsnatch']} /><label>Notify on snatch?</label>
+                                    </div>
+                                </div>
+                            </fieldset>
+
                         </td>
                     </tr>
                 </table>
@@ -1166,14 +1283,13 @@
                             <div class="row">
                                 <label>Folder Format</label>
                                 <input type="text" name="folder_format" value="${config['folder_format']}" size="43">
-                                <small>Use: $Artist/$artist, $SortArtist/$sortartist, $Album/$album, $Year/$year, $Type/$type (release type) and $First/$first (first letter in artist name), $OriginalFolder/$originalfolder (downloaded directory name)
-                                                                E.g.: $Type/$First/$artist/$album [$year] = Album/G/girl talk/all day [2010]</small>
+                                <small>Use: $Artist/$artist, $SortArtist/$sortartist, $Album/$album, $Year/$year, $Type/$type (release type) and $First/$first (first letter in artist name), $OriginalFolder/$originalfolder (downloaded directory name). Put optional variables in square brackets, use single-quote marks to escape square brackets literally ('[', ']').<br>E.g.: $Type/$First/$artist/$album '['$year']' = Album/G/girl talk/all day [2010]</small>
 
                             </div>
                             <div class="row">
                                 <label>File Format</label>
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
-                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year</small>
+                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year. Put optional variables in square brackets, use single-quote marks to escape square brackets literally ('[', ']').</small>
                             </div>
                             <div class="checkbox row clearfix">
                                 <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
@@ -1337,6 +1453,11 @@
                                     <input type="text" name="xldprofile" value="${config['xldprofile']}" size="43">
                                 </div>
                             </div>
+                            <div class="row">
+                                <label>Extension</label>
+                                <input type="text" name="encoderoutputformat" value="${config['encoderoutputformat']}" size="43">
+                                <small>If different from format selected above</small>
+                            </div>
                             <div class="row">
                                 <label>Path to Encoder</label>
                                 <input type="text" name="encoder_path" value="${config['encoder_path']}" size="43">
@@ -1358,7 +1479,7 @@
                                         else:
                                             which_extras_selected = "None"
                                     %>
-                                    <small>Currently Selected: ${which_extras_selected} <a href="#" id="modify_extras">(Change)</a></small></label>
+                                    <small>Currently Selected: ${which_extras_selected} <a href="javascript:void(0)" id="modify_extras">(Change)</a></small></label>
                                     <div id="dialog" title="Choose Which Extras to Include" style="display:none" class="configtable">
                                     %for extra in config['extras']:
                                         <input type="checkbox" id="${extra}_temp" name="${extra}_temp" value="1" ${config['extras'][extra]} />${string.capwords(extra)}<br>
@@ -1393,6 +1514,10 @@
                                         <input type="checkbox" name="freeze_db" id="freeze_db" value="1" ${config['freeze_db']} />
                                     </label>
                                 </div>
+                                <div class="row checkbox left clearfix nopad">
+                                    <label><input type="checkbox" title="Stop post-processing if no good metadata match found"
+                                           name="do_not_process_unmatched" value="1" ${config['do_not_process_unmatched']} />Stop post-processing if no good metadata match found</label>
+                                </div>
                                 <div class="row checkbox left clearfix">
                                     <label title="Use ID3v2.3 instead of ID3v2.4">
                                         Tag using ID3v2.3 instead of ID3v2.4
@@ -1924,6 +2049,26 @@
                         }
                 });
 
+                if ($("#telegram").is(":checked"))
+                        {
+                            $("#telegramoptions").show();
+                        }
+                else
+                        {
+                            $("#telegramoptions").hide();
+                        }
+
+                $("#telegram").click(function(){
+                        if ($("#telegram").is(":checked"))
+                        {
+                            $("#telegramoptions").slideDown();
+                        }
+                        else
+                        {
+                            $("#telegramoptions").slideUp();
+                        }
+                });
+
                 if ($("#osx_notify").is(":checked"))
                         {
                             $("#osx_notify_options").show();
@@ -2080,19 +2225,25 @@
 
                 if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options").hide();
+                            $("#transmission_options,#utorrent_options,#deluge_options").hide();
                             $("#torrent_blackhole_options").show();
                         }
                 if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options").hide();
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").hide();
                             $("#transmission_options").show();
                         }
                 if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options").hide();
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").hide();
                             $("#utorrent_options").show();
                         }
+                if ($("#torrent_downloader_deluge").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#transmission_options,#utorrent_options").hide();
+                            $("#deluge_options").show();
+                        }
+
 
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
@@ -2129,15 +2280,19 @@
                         }
                         if ($("#torrent_downloader_blackhole").is(":checked"))
                         {
-                            $("#transmission_options,#utorrent_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
+                            $("#transmission_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#torrent_blackhole_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_transmission").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#utorrent_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
+                            $("#torrent_blackhole_options,#utorrent_options,#deluge_options").fadeOut("fast", function() { $("#transmission_options").fadeIn() });
                         }
                         if ($("#torrent_downloader_utorrent").is(":checked"))
                         {
-                            $("#torrent_blackhole_options,#transmission_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                            $("#torrent_blackhole_options,#transmission_options,#deluge_options").fadeOut("fast", function() { $("#utorrent_options").fadeIn() });
+                        }
+                        if ($("#torrent_downloader_deluge").is(":checked"))
+                        {
+                            $("#torrent_blackhole_options,#utorrent_options,#transmission_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
                 });
 
@@ -2157,6 +2312,13 @@
                     deletedNewznabs = deletedNewznabs + 1;
                 });
 
+                var deletedTorznabs = 0;
+
+                $(".remove_torznab").click(function() {
+                    $(this).parent().parent().remove();
+                    deletedTorznabs = deletedTorznabs + 1;
+                });
+
                 $("#modify_extras").click(openExtrasDialog);
 
                 $("#include_extras").click(function(){
@@ -2190,6 +2352,20 @@
                     $("#add_newznab").before(formfields);
                 });
 
+                $("#add_torznab").click(function() {
+                    var intId = $("#torznab_providers > div").size() + deletedTorznabs + 1;
+                    var formfields = $("<div class=\"config\" id=\"torznab" + intId + "\"><div class=\"row\"><label>Torznab Host</label><input type=\"text\" name=\"torznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Torznab API</label><input type=\"text\" name=\"torznab_api" + intId + "\" size=\"36\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"torznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
+                    var removeTorznabButton = $("<div class=\"row\"><input type=\"button\" class=\"remove_torznab\" value=\"Remove\" /></div>");
+                    removeTorznabButton.click(function() {
+                        $(this).parent().remove();
+                        deletedTorznabs = deletedTorznabs + 1;
+
+                    });
+                    formfields.append(removeTorznabButton);
+                    formfields.append("</div>");
+                    $("#add_torznab").before(formfields);
+                });
+
                 $(".hpuser").keyup(function() {
                     $(".hpuser").val($(this).val());
                 });
@@ -2206,6 +2382,7 @@
                 initConfigCheckbox("#use_newznab");
                 initConfigCheckbox("#use_nzbsorg");
                 initConfigCheckbox("#use_omgwtfnzbs");
+                initConfigCheckbox("#use_torznab");
                 initConfigCheckbox("#use_kat");
                 initConfigCheckbox("#use_piratebay");
                 initConfigCheckbox("#use_oldpiratebay");
@@ -2213,6 +2390,7 @@
                 initConfigCheckbox("#use_waffles");
                 initConfigCheckbox("#use_rutracker");
                 initConfigCheckbox("#use_whatcd");
+                initConfigCheckbox("#use_strike");
                 initConfigCheckbox("#api_enabled");
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");

data/interfaces/default/css/style.css

@@ -229,6 +229,17 @@ textarea,
 button {
   font: 99%;
 }
+select {
+  -moz-border-radius: 5px;
+  -webkit-border-radius: 5px;
+  border-radius: 5px;
+  background: #4F4F4F;
+  border: 0;
+  border-bottom: 1px solid rgba(0, 0, 0, 0.25);
+  color: #fff;
+  padding: 3px 10px;
+  text-shadow: 0 -1px 1px rgba(0, 0, 0, 0.25);
+}
 textarea {
   overflow: auto;
 }
@@ -334,7 +345,7 @@ form .row label {
   font-size: 12px;
   line-height: normal;
   padding-top: 7px;
-  width: 175px;
+  width: 170px;
 }
 form .row label.inline {
   margin-right: 5px;
@@ -368,7 +379,7 @@ form .row small {
   display: block;
   font-size: 9px;
   line-height: 12px;
-  margin-left: 175px;
+  margin-left: 170px;
   margin-top: 3px;
 }
 form .left label {

data/interfaces/default/css/style.less

@@ -124,7 +124,23 @@ table {
 
 
 // Forms
-select, input, textarea, button { font: 99%;}
+select, input, textarea, button
+{
+    font: 99%;
+}
+
+select 
+{
+    .rounded(5px);
+    background: #4F4F4F;
+    border: 0;
+    border-bottom: 1px solid rgba(0, 0, 0, 0.25);
+    color: #fff;
+    padding: 3px 10px;
+    text-shadow: 0 -1px 1px rgba(0, 0, 0, 0.25);
+}
+
+
 textarea {overflow: auto;}
 input { .rounded(3px);}
 input:invalid, textarea:invalid {
@@ -190,7 +206,7 @@ form {
             font-size: 12px;
             line-height: normal;
             padding-top: 7px;
-            width: 175px;
+            width: 170px;
 
             &.inline {
                 margin-right: 5px;
@@ -216,7 +232,7 @@ form {
             display: block;
             font-size: 9px;
             line-height: 12px;
-            margin-left: 175px;
+            margin-left: 170px;
             margin-top: 3px;
         }
     }

data/interfaces/default/history.html

@@ -7,11 +7,11 @@
 <%def name="headerIncludes()">
 	<div id="subhead_container">
 		<div id="subhead_menu">
-			<a id="menu_link_delete" href="#" onclick="doAjaxCall('clearhistory?type=all',$(this),'table')" data-success="All History cleared"><i class="fa fa-trash-o"></i> Clear All History</a>
-			<a id="menu_link_delete" href="#" onclick="doAjaxCall('clearhistory?type=Processed',$(this),'table')" data-success="All Processed cleared"><i class="fa fa-trash-o"></i> Clear Processed</a>
-			<a id="menu_link_delete" href="#" onclick="doAjaxCall('clearhistory?type=Unprocessed',$(this),'table')" data-success="All Unprocessed cleared"><i class="fa fa-trash-o"></i> Clear Unprocessed</a>
-			<a id="menu_link_delete" href="#" onclick="doAjaxCall('clearhistory?type=Frozen',$(this),'table')" data-success="All Frozen cleared"><i class="fa fa-trash-o"></i> Clear Frozen</a>
-			<a id="menu_link_delete" href="#" onclick="doAjaxCall('clearhistory?type=Snatched',$(this),'table')" data-success="All Snatched cleared"><i class="fa fa-trash-o"></i> Clear Snatched</a>
+			<a id="menu_link_delete" href="javascript:void(0)" onclick="doAjaxCall('clearhistory?type=all',$(this),'table')" data-success="All History cleared"><i class="fa fa-trash-o"></i> Clear All History</a>
+			<a id="menu_link_delete" href="javascript:void(0)" onclick="doAjaxCall('clearhistory?type=Processed',$(this),'table')" data-success="All Processed cleared"><i class="fa fa-trash-o"></i> Clear Processed</a>
+			<a id="menu_link_delete" href="javascript:void(0)" onclick="doAjaxCall('clearhistory?type=Unprocessed',$(this),'table')" data-success="All Unprocessed cleared"><i class="fa fa-trash-o"></i> Clear Unprocessed</a>
+			<a id="menu_link_delete" href="javascript:void(0)" onclick="doAjaxCall('clearhistory?type=Frozen',$(this),'table')" data-success="All Frozen cleared"><i class="fa fa-trash-o"></i> Clear Frozen</a>
+			<a id="menu_link_delete" href="javascript:void(0)" onclick="doAjaxCall('clearhistory?type=Snatched',$(this),'table')" data-success="All Snatched cleared"><i class="fa fa-trash-o"></i> Clear Snatched</a>
 		</div>
 	</div>
 </%def>
@@ -50,6 +50,8 @@
                 fileid = 'nzb'
             if item['URL'].find('torrent') != -1:
                 fileid = 'torrent'
+            if item['URL'].find('magnet:') != -1:
+                fileid = 'torrent'
             if item['URL'].find('rutracker') != -1:
                 fileid = 'torrent'
             if item['URL'].find('codeshy') != -1:
@@ -63,8 +65,8 @@
 				<td id="filename">${cgi.escape(item['Title'], quote=True)} [<a href="${item['URL']}">${fileid}</a>]<a href="albumPage?AlbumID=${item['AlbumID']}">[album page]</a></td>
 				<td id="size">${helpers.bytes_to_mb(item['Size'])}</td>
 				<td title="${folder}" id="status">${item['Status']}</td>
-				<td id="action">[<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${cgi.escape(item['Title'], quote=True)}'">retry</a>][<a href="#" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${cgi.escape(item['Title'], quote=True)}'">new</a>]</td>
-				<td id="delete"><a href="#" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${cgi.escape(item['Title'], quote=True)}',$(this),'table')" data-success="${cgi.escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
+				<td id="action">[<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&redirect=history', $(this),'table')"  data-success="Retrying download of '${cgi.escape(item['Title'], quote=True)}'">retry</a>][<a href="javascript:void(0)" onclick="doAjaxCall('queueAlbum?AlbumID=${item['AlbumID']}&new=True&redirect=history',$(this),'table')"  data-success="Looking for a new version of '${cgi.escape(item['Title'], quote=True)}'">new</a>]</td>
+				<td id="delete"><a href="javascript:void(0)" onclick="doAjaxCall('clearhistory?date_added=${item['DateAdded']}&title=${cgi.escape(item['Title'], quote=True)}',$(this),'table')" data-success="${cgi.escape(item['Title'], quote=True)} cleared from history"><img src="interfaces/default/images/trashcan.png" height="18" width="18" id="trashcan" title="Clear this item from the history"></a>
 			</tr>
 		%endfor
 		</tbody>

data/interfaces/default/manage.html

@@ -6,7 +6,7 @@
 <%def name="headerIncludes()">
 	<div id="subhead_container">
 		<div id="subhead_menu">
-			<a class="menu_link_edit" id="manage_albums" href="#"><i class="fa fa-pencil"></i> Manage Albums</a>
+			<a class="menu_link_edit" id="manage_albums" href="javascript:void(0)"><i class="fa fa-pencil"></i> Manage Albums</a>
 				<div id="dialog" title="Choose Album Filter" style="display:none" class="configtable">
 					<div class="links">
 						<a href="manageAlbums?Status=Downloaded"><i class="fa fa-check fa-fw"></i> Manage Downloaded Albums</a><br>
@@ -90,7 +90,7 @@
 				%>
 				<input type="text" value="${lastfmvalue}" placeholder="Last.fm username" onfocus="if
 				(this.value==this.defaultValue) this.value='';" name="username" id="username" size="18" />
-				<a href="#" onclick="doAjaxCall('importLastFM?username=',$(this),'tabs');return false;" data-success="Last.fm username has been reset"><i class="fa fa-reply"></i> Reset username</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('importLastFM?username=',$(this),'tabs');return false;" data-success="Last.fm username has been reset"><i class="fa fa-reply"></i> Reset username</a>
 			</div>
 			</fieldset>
 			<input type="button" value="Save changes" onclick="doAjaxCall('importLastFM',$(this),'tabs',true);return false;" data-success="Last.fm artists will be imported" data-error="Fill in a last.fm username"/>
@@ -121,10 +121,10 @@
 		<fieldset>
 			<legend>Force Search</legend>
 			<div class="links">
-				<a href="#" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><i class="fa fa-search fa-fw"></i> Force Check for Wanted Albums</a>
-				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><i class="fa fa-heart fa-fw"></i> Force Update Active Artists [Fast]</a>
-				<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><i class="fa fa-refresh fa-fw"></i> Check for Headphones Updates</a>
-				<a href="#" id="delete_empty_artists"><i class="fa fa-trash-o fa-fw"></i> Delete empty Artists</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><i class="fa fa-search fa-fw"></i> Force Check for Wanted Albums</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><i class="fa fa-heart fa-fw"></i> Force Update Active Artists [Fast]</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><i class="fa fa-refresh fa-fw"></i> Check for Headphones Updates</a>
+				<a href="javascript:void(0)" id="delete_empty_artists"><i class="fa fa-trash-o fa-fw"></i> Delete empty Artists</a>
 					<div id="emptyartistdialog" title="Confirm Artist Deletion" style="display:none" class="configtable">
 					    %if emptyArtists:
 						<h3>The following artists will be deleted:</h3>
@@ -138,7 +138,7 @@
 					    %endif
 					</div>
 					<div id="post_process">
-					<a href="#" class="btnOpenDialog"><i class="fa fa-wrench fa-fw"></i> Force Post-Process Albums in Download Folder</a>
+					<a href="javascript:void(0)" class="btnOpenDialog"><i class="fa fa-wrench fa-fw"></i> Force Post-Process Albums in Download Folder</a>
 					</div>
 			</div>
 		</fieldset>
@@ -166,9 +166,9 @@
 			<legend>Force Legacy</legend>
 				<p>Please note that these functions will take a significant amount of time to complete.</p>
 			<div class="links">
-				<a href="#" onclick="doAjaxCall('forceFullUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><i class="fa fa-heart fa-fw"></i> Force Update Active Artists [Comprehensive]</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('forceFullUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><i class="fa fa-heart fa-fw"></i> Force Update Active Artists [Comprehensive]</a>
 				<BR>
-				<a href="#" onclick="doAjaxCall('forceScan',$(this))" data-success="Library scan successful" data-error="Error forcing library scan"><i class="fa fa-refresh fa-fw"></i> Force Re-scan Library [Comprehensive]</a>
+				<a href="javascript:void(0)" onclick="doAjaxCall('forceScan',$(this))" data-success="Library scan successful" data-error="Error forcing library scan"><i class="fa fa-refresh fa-fw"></i> Force Re-scan Library [Comprehensive]</a>
 				<BR>
 				<small>*Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched".</small>
 

data/interfaces/default/managemanual.html

@@ -43,9 +43,9 @@
 					<tr><td>Are you sure you want to reset Local Artist: ${album['ArtistName']} to unmatched?</td></tr>
 					<tr><td align="right"><BR>
 					%if album['AlbumStatus'] == "Ignored":
-						<button href="#" onclick="doAjaxCall('markManual?action=unignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully reset ${album['ArtistName']} to unmatched">Reset Artist</button>
+						<button href="javascript:void(0)" onclick="doAjaxCall('markManual?action=unignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully reset ${album['ArtistName']} to unmatched">Reset Artist</button>
 					%elif album['AlbumStatus'] == "Matched":
-						<button href="#" onclick="doAjaxCall('markManual?action=unmatchArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Reset Artist</button>
+						<button href="javascript:void(0)" onclick="doAjaxCall('markManual?action=unmatchArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Reset Artist</button>
 					%endif
 					</td></tr>
 					</table>
@@ -58,9 +58,9 @@
 					<tr><td>Are you sure you want to reset Local Album: ${album['AlbumTitle']} to unmatched?</td></tr>
 					<tr><td align="right"><BR>
 					%if album['AlbumStatus'] == "Ignored":
-						<button href="#" onclick="doAjaxCall('markManual?action=unignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully reset ${album['AlbumTitle']} to unmatched">Reset Album</button>
+						<button href="javascript:void(0)" onclick="doAjaxCall('markManual?action=unignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully reset ${album['AlbumTitle']} to unmatched">Reset Album</button>
 					%elif album['AlbumStatus'] == "Matched":
-						<button href="#" onclick="doAjaxCall('markManual?action=unmatchAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully reset ${album['AlbumTitle']} to unmatched">Reset Album</button>
+						<button href="javascript:void(0)" onclick="doAjaxCall('markManual?action=unmatchAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully reset ${album['AlbumTitle']} to unmatched">Reset Album</button>
 					%endif
 					</td></tr>
 					</table>

data/interfaces/default/manageunmatched.html

@@ -52,7 +52,7 @@
 					<table>
 					<tr><td>Are you sure you want to ignore Local Artist: ${album['ArtistName']} from future matching?</td></tr>
 					<tr><td align="right"><BR>
-					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully ignored ${album['ArtistName']} from future matching">Ignore Artist</button>
+					<button href="javascript:void(0)" onclick="doAjaxCall('markUnmatched?action=ignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully ignored ${album['ArtistName']} from future matching">Ignore Artist</button>
 					</td></tr>
 					</table>
 				</div>
@@ -66,7 +66,7 @@
 						</select>
 						</td></tr>
 					<tr><td></td><td align="right"><BR>
-					<button href="#" onclick="artist_matcher(${count_albums}, '${old_artist_js}')">Match Artist</button>
+					<button href="javascript:void(0)" onclick="artist_matcher(${count_albums}, '${old_artist_js}')">Match Artist</button>
 					</td></tr>
 					</table>
 				</div>
@@ -78,7 +78,7 @@
 					<table>
 					<tr><td>Are you sure you want to ignore Local Album: ${album['AlbumTitle']} from future matching?</td></tr>
 					<tr><td align="right"><BR>
-					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully ignored ${album['AlbumTitle']} from future matching">Ignore Album</button>
+					<button href="javascript:void(0)" onclick="doAjaxCall('markUnmatched?action=ignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully ignored ${album['AlbumTitle']} from future matching">Ignore Album</button>
 					</td></tr>
 					</table>
 				</div>
@@ -97,7 +97,7 @@
 					</select>
 					</td></tr>
 					<tr><td></td><td align="right"><BR>
-					<button href="#" onclick="album_matcher(${count_albums}, '${old_artist_js}', '${old_album_js}')">Match Album</button>
+					<button href="javascript:void(0)" onclick="album_matcher(${count_albums}, '${old_artist_js}', '${old_album_js}')">Match Album</button>
 					</td></tr>
 					</table>
 				</div>

data/interfaces/default/upcoming.html

@@ -3,7 +3,7 @@
 <%def name="headerIncludes()">
 	<div id="subhead_container">
 		<div id="subhead_menu">
-			<a href="#" id="menu_link_scan" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><i class="fa fa-search"></i> Force Check</a>
+			<a href="javascript:void(0)" id="menu_link_scan" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><i class="fa fa-search"></i> Force Check</a>
 		</div>
 	</div>
 </%def>

headphones/__init__.py

@@ -16,20 +16,21 @@
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of
 # XBian - XBMC on the Raspberry Pi
 
-import os
 import sys
 import subprocess
 import threading
 import webbrowser
 import sqlite3
-import cherrypy
 import datetime
 
+import os
+import cherrypy
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.interval import IntervalTrigger
-
 from headphones import versioncheck, logger
 import headphones.config
+from headphones.softchroot import SoftChroot
+import headphones.exceptions
 
 # (append new extras to the end)
 POSSIBLE_EXTRAS = [
@@ -74,6 +75,7 @@ started = False
 DATA_DIR = None
 
 CONFIG = None
+SOFT_CHROOT = None
 
 DB_FILE = None
 
@@ -94,10 +96,10 @@ UMASK = None
 
 
 def initialize(config_file):
-
     with INIT_LOCK:
 
         global CONFIG
+        global SOFT_CHROOT
         global _INITIALIZED
         global CURRENT_VERSION
         global LATEST_VERSION
@@ -131,11 +133,19 @@ def initialize(config_file):
 
                 if not QUIET:
                     sys.stderr.write("Unable to create the log directory. " \
-                        "Logging to screen only.\n")
+                                     "Logging to screen only.\n")
 
         # Start the logger, disable console if needed
         logger.initLogger(console=not QUIET, log_dir=CONFIG.LOG_DIR,
-            verbose=VERBOSE)
+                          verbose=VERBOSE)
+
+        try:
+            SOFT_CHROOT = SoftChroot(str(CONFIG.SOFT_CHROOT))
+            if SOFT_CHROOT.isEnabled():
+                logger.info("Soft-chroot enabled for dir: %s", str(CONFIG.SOFT_CHROOT))
+        except exceptions.SoftChrootError as e:
+            logger.error("SoftChroot error: %s", e)
+            raise e
 
         if not CONFIG.CACHE_DIR:
             # Put the cache dir in the data dir for now
@@ -246,7 +256,6 @@ def daemonize():
 
 
 def launch_browser(host, port, root):
-
     if host == '0.0.0.0':
         host = 'localhost'
 
@@ -287,17 +296,19 @@ def initialize_scheduler():
         hours = CONFIG.UPDATE_DB_INTERVAL
         schedule_job(updater.dbUpdate, 'MusicBrainz Update', hours=hours, minutes=0)
 
-        #Update check
+        # Update check
         if CONFIG.CHECK_GITHUB:
             if CONFIG.CHECK_GITHUB_INTERVAL:
                 minutes = CONFIG.CHECK_GITHUB_INTERVAL
             else:
                 minutes = 0
-            schedule_job(versioncheck.checkGithub, 'Check GitHub for updates', hours=0, minutes=minutes)
+            schedule_job(versioncheck.checkGithub, 'Check GitHub for updates', hours=0,
+                         minutes=minutes)
 
         # Remove Torrent + data if Post Processed and finished Seeding
         minutes = CONFIG.TORRENT_REMOVAL_INTERVAL
-        schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0, minutes=minutes)
+        schedule_job(torrentfinished.checkTorrentFinished, 'Torrent removal check', hours=0,
+                     minutes=minutes)
 
         # Start scheduler
         if start_jobs and len(SCHED.get_jobs()):
@@ -306,8 +317,8 @@ def initialize_scheduler():
             except Exception as e:
                 logger.info(e)
 
-        # Debug
-        #SCHED.print_jobs()
+                # Debug
+                # SCHED.print_jobs()
 
 
 def schedule_job(function, name, hours=0, minutes=0):
@@ -334,7 +345,6 @@ def schedule_job(function, name, hours=0, minutes=0):
 
 
 def start():
-
     global started
 
     if _INITIALIZED:
@@ -349,7 +359,6 @@ def sig_handler(signum=None, frame=None):
 
 
 def dbcheck():
-
     conn = sqlite3.connect(DB_FILE)
     c = conn.cursor()
     c.execute(
@@ -609,7 +618,6 @@ def dbcheck():
 
 
 def shutdown(restart=False, update=False):
-
     cherrypy.engine.exit()
     SCHED.shutdown(wait=False)
 

headphones/albumart_test.py

@@ -0,0 +1,14 @@
+#import unittest
+#import mock
+from headphones.unittestcompat import TestCase
+
+import headphones.albumart
+
+
+# no tests...
+class AlbumArtTest(TestCase):
+    def test_nothing(self):
+        x = 100 - 2 * 50
+        if x:
+            headphones.albumart.getAlbumArt('asdf')
+        self.assertTrue(True)

headphones/albumswitcher.py

@@ -53,7 +53,6 @@ def switch(AlbumID, ReleaseID):
     c.get_artwork_from_cache(AlbumID=AlbumID)
 
     for track in newtrackdata:
-
         controlValueDict = {"TrackID": track['TrackID'],
                             "AlbumID": AlbumID}
 
@@ -79,15 +78,18 @@ def switch(AlbumID, ReleaseID):
     have_track_count = len(myDB.select(
         'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [AlbumID]))
 
-    if oldalbumdata['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
+    if oldalbumdata['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (
+        headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
         myDB.action(
             'UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', AlbumID])
 
     # Update have track counts on index
     totaltracks = len(myDB.select(
-        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [newalbumdata['ArtistID']]))
+        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")',
+        [newalbumdata['ArtistID']]))
     havetracks = len(myDB.select(
-        'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [newalbumdata['ArtistID']]))
+        'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
+        [newalbumdata['ArtistID']]))
 
     controlValueDict = {"ArtistID": newalbumdata['ArtistID']}
 

headphones/api.py

@@ -13,21 +13,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import db, mb, updater, importer, searcher, cache, postprocessor, versioncheck, logger
-
-import headphones
 import json
 
-cmd_list = ['getIndex', 'getArtist', 'getAlbum', 'getUpcoming', 'getWanted', 'getSnatched', 'getSimilar', 'getHistory', 'getLogs',
-            'findArtist', 'findAlbum', 'addArtist', 'delArtist', 'pauseArtist', 'resumeArtist', 'refreshArtist',
-            'addAlbum', 'queueAlbum', 'unqueueAlbum', 'forceSearch', 'forceProcess', 'forceActiveArtistsUpdate', 
-            'getVersion', 'checkGithub', 'shutdown', 'restart', 'update', 'getArtistArt', 'getAlbumArt', 
+from headphones import db, mb, updater, importer, searcher, cache, postprocessor, versioncheck, \
+    logger
+import headphones
+
+cmd_list = ['getIndex', 'getArtist', 'getAlbum', 'getUpcoming', 'getWanted', 'getSnatched',
+            'getSimilar', 'getHistory', 'getLogs',
+            'findArtist', 'findAlbum', 'addArtist', 'delArtist', 'pauseArtist', 'resumeArtist',
+            'refreshArtist',
+            'addAlbum', 'queueAlbum', 'unqueueAlbum', 'forceSearch', 'forceProcess',
+            'forceActiveArtistsUpdate',
+            'getVersion', 'checkGithub', 'shutdown', 'restart', 'update', 'getArtistArt',
+            'getAlbumArt',
             'getArtistInfo', 'getAlbumInfo', 'getArtistThumb', 'getAlbumThumb', 'clearLogs',
             'choose_specific_download', 'download_specific_release']
 
 
 class Api(object):
-
     def __init__(self):
 
         self.apikey = None
@@ -170,7 +174,7 @@ class Api(object):
         self.data = self._dic_from_query(
             "SELECT * from albums WHERE Status='Snatched'")
         return
-        
+
     def _getSimilar(self, **kwargs):
         self.data = self._dic_from_query('SELECT * from lastfmcloud')
         return
@@ -328,10 +332,15 @@ class Api(object):
         searcher.searchforalbum()
 
     def _forceProcess(self, **kwargs):
-        self.dir = None
-        if 'dir' in kwargs:
+        if 'album_dir' in kwargs:
+            album_dir = kwargs['album_dir']
+            dir = None
+            postprocessor.forcePostProcess(self, dir, album_dir)
+        elif 'dir' in kwargs:
             self.dir = kwargs['dir']
-        postprocessor.forcePostProcess(self.dir)
+            postprocessor.forcePostProcess(self.dir)
+        else:
+            postprocessor.forcePostProcess()
 
     def _forceActiveArtistsUpdate(self, **kwargs):
         updater.dbUpdate()
@@ -432,7 +441,6 @@ class Api(object):
         results_as_dicts = []
 
         for result in results:
-
             result_dict = {
                 'title': result[0],
                 'size': result[1],

headphones/cache.py

@@ -14,8 +14,8 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import headphones
 
+import headphones
 from headphones import db, helpers, logger, lastfm, request
 
 LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
@@ -45,8 +45,8 @@ class Cache(object):
 
     def __init__(self):
         self.id = None
-        self.id_type = None # 'artist' or 'album' - set automatically depending on whether ArtistID or AlbumID is passed
-        self.query_type = None # 'artwork','thumb' or 'info' - set automatically
+        self.id_type = None  # 'artist' or 'album' - set automatically depending on whether ArtistID or AlbumID is passed
+        self.query_type = None  # 'artwork','thumb' or 'info' - set automatically
 
         self.artwork_files = []
         self.thumb_files = []
@@ -116,7 +116,7 @@ class Cache(object):
             return None
 
         for image in images:
-            if image['size'] == 'medium':
+            if image['size'] == 'medium' and '#text' in image:
                 thumb_url = image['#text']
                 break
 
@@ -182,13 +182,18 @@ class Cache(object):
         if ArtistID:
             self.id = ArtistID
             self.id_type = 'artist'
-            db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?', [self.id]).fetchone()
+            db_info = myDB.action(
+                'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?',
+                [self.id]).fetchone()
         else:
             self.id = AlbumID
             self.id_type = 'album'
-            db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?', [self.id]).fetchone()
+            db_info = myDB.action(
+                'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?',
+                [self.id]).fetchone()
 
-        if not db_info or not db_info['LastUpdated'] or not self._is_current(date=db_info['LastUpdated']):
+        if not db_info or not db_info['LastUpdated'] or not self._is_current(
+                date=db_info['LastUpdated']):
 
             self._update_cache()
             info_dict = {'Summary': self.info_summary, 'Content': self.info_content}
@@ -309,13 +314,19 @@ class Cache(object):
                 logger.debug('No artist thumbnail image found')
 
         else:
-            dbalbum = myDB.action('SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', [self.id]).fetchone()
+            dbalbum = myDB.action(
+                'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?',
+                [self.id]).fetchone()
             if dbalbum['ReleaseID'] != self.id:
-                data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], api_key=LASTFM_API_KEY)
+                data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'],
+                                             api_key=LASTFM_API_KEY)
                 if not data:
-                    data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
+                    data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                                 album=dbalbum['AlbumTitle'],
+                                                 api_key=LASTFM_API_KEY)
             else:
-                data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
+                data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'],
+                                             album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY)
 
             if not data:
                 return
@@ -357,7 +368,8 @@ class Cache(object):
         # Save the image URL to the database
         if image_url:
             if self.id_type == 'artist':
-                myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id])
+                myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?',
+                            [image_url, self.id])
             else:
                 myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id])
 
@@ -378,7 +390,8 @@ class Cache(object):
                 if not os.path.isdir(self.path_to_art_cache):
                     try:
                         os.makedirs(self.path_to_art_cache)
-                        os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8))
+                        os.chmod(self.path_to_art_cache,
+                                 int(headphones.CONFIG.FOLDER_PERMISSIONS, 8))
                     except OSError as e:
                         logger.error('Unable to create artwork cache dir. Error: %s', e)
                         self.artwork_errors = True
@@ -393,7 +406,8 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext)
+                artwork_path = os.path.join(self.path_to_art_cache,
+                                            self.id + '.' + helpers.today() + ext)
                 try:
                     with open(artwork_path, 'wb') as f:
                         f.write(artwork)
@@ -406,7 +420,8 @@ class Cache(object):
 
         # Grab the thumbnail as well if we're getting the full artwork (as long
         # as it's missing/outdated.
-        if thumb_url and self.query_type in ['thumb', 'artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])):
+        if thumb_url and self.query_type in ['thumb', 'artwork'] and not (
+            self.thumb_files and self._is_current(self.thumb_files[0])):
             artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
@@ -414,7 +429,8 @@ class Cache(object):
                 if not os.path.isdir(self.path_to_art_cache):
                     try:
                         os.makedirs(self.path_to_art_cache)
-                        os.chmod(self.path_to_art_cache, int(headphones.CONFIG.FOLDER_PERMISSIONS, 8))
+                        os.chmod(self.path_to_art_cache,
+                                 int(headphones.CONFIG.FOLDER_PERMISSIONS, 8))
                     except OSError as e:
                         logger.error('Unable to create artwork cache dir. Error: %s' + e)
                         self.thumb_errors = True
@@ -429,7 +445,8 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext)
+                thumb_path = os.path.join(self.path_to_art_cache,
+                                          'T_' + self.id + '.' + helpers.today() + ext)
                 try:
                     with open(thumb_path, 'wb') as f:
                         f.write(artwork)

headphones/classes.py

@@ -13,9 +13,9 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-#########################################
-## Stolen from Sick-Beard's classes.py ##
-#########################################
+#######################################
+# Stolen from Sick-Beard's classes.py #
+#######################################
 
 
 import urllib
@@ -133,4 +133,5 @@ class Proper:
         self.episode = -1
 
     def __str__(self):
-        return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str(self.episode) + " of " + str(self.tvdbid)
+        return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str(
+            self.episode) + " of " + str(self.tvdbid)

headphones/common.py

@@ -20,15 +20,16 @@ Created on Aug 1, 2011
 '''
 import platform
 import operator
+
 import os
 import re
-
 from headphones import version
 
-#Identify Our Application
+
+# Identify Our Application
 USER_AGENT = 'Headphones/-' + version.HEADPHONES_VERSION + ' (' + platform.system() + ' ' + platform.release() + ')'
 
-### Notification Types
+# Notification Types
 NOTIFY_SNATCH = 1
 NOTIFY_DOWNLOAD = 2
 
@@ -36,26 +37,25 @@ notifyStrings = {}
 notifyStrings[NOTIFY_SNATCH] = "Started Download"
 notifyStrings[NOTIFY_DOWNLOAD] = "Download Finished"
 
-### Release statuses
-UNKNOWN = -1 # should never happen
-UNAIRED = 1 # releases that haven't dropped yet
-SNATCHED = 2 # qualified with quality
-WANTED = 3 # releases we don't have but want to get
-DOWNLOADED = 4 # qualified with quality
-SKIPPED = 5 # releases we don't want
-ARCHIVED = 6 # releases that you don't have locally (counts toward download completion stats)
-IGNORED = 7 # releases that you don't want included in your download stats
-SNATCHED_PROPER = 9 # qualified with quality
+# Release statuses
+UNKNOWN = -1  # should never happen
+UNAIRED = 1  # releases that haven't dropped yet
+SNATCHED = 2  # qualified with quality
+WANTED = 3  # releases we don't have but want to get
+DOWNLOADED = 4  # qualified with quality
+SKIPPED = 5  # releases we don't want
+ARCHIVED = 6  # releases that you don't have locally (counts toward download completion stats)
+IGNORED = 7  # releases that you don't want included in your download stats
+SNATCHED_PROPER = 9  # qualified with quality
 
 
 class Quality:
-
     NONE = 0
-    B192 = 1 << 1     # 2
-    VBR = 1 << 2     # 4
-    B256 = 1 << 3     # 8
-    B320 = 1 << 4     #16
-    FLAC = 1 << 5     #32
+    B192 = 1 << 1  # 2
+    VBR = 1 << 2  # 4
+    B256 = 1 << 3  # 8
+    B320 = 1 << 4  # 16
+    FLAC = 1 << 5  # 32
 
     # put these bits at the other end of the spectrum, far enough out that they shouldn't interfere
     UNKNOWN = 1 << 15
@@ -75,7 +75,8 @@ class Quality:
     def _getStatusStrings(status):
         toReturn = {}
         for x in Quality.qualityStrings.keys():
-            toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + Quality.qualityStrings[x] + ")"
+            toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + \
+                                                           Quality.qualityStrings[x] + ")"
         return toReturn
 
     @staticmethod
@@ -103,6 +104,9 @@ class Quality:
     @staticmethod
     def nameQuality(name):
 
+        def checkName(list, func):
+            return func([re.search(x, name, re.I) for x in list])
+
         name = os.path.basename(name)
 
         # if we have our exact text then assume we put it there
@@ -115,9 +119,7 @@ class Quality:
             if regex_match:
                 return x
 
-        checkName = lambda list, func: func([re.search(x, name, re.I) for x in list])
-
-        #TODO: fix quality checking here
+        # TODO: fix quality checking here
         if checkName(["mp3", "192"], any) and not checkName(["flac"], all):
             return Quality.B192
         elif checkName(["mp3", "256"], any) and not checkName(["flac"], all):
@@ -131,7 +133,6 @@ class Quality:
 
     @staticmethod
     def assumeQuality(name):
-
         if name.lower().endswith(".mp3"):
             return Quality.MP3
         elif name.lower().endswith(".flac"):
@@ -167,13 +168,16 @@ class Quality:
     SNATCHED = None
     SNATCHED_PROPER = None
 
+
 Quality.DOWNLOADED = [Quality.compositeStatus(DOWNLOADED, x) for x in Quality.qualityStrings.keys()]
 Quality.SNATCHED = [Quality.compositeStatus(SNATCHED, x) for x in Quality.qualityStrings.keys()]
-Quality.SNATCHED_PROPER = [Quality.compositeStatus(SNATCHED_PROPER, x) for x in Quality.qualityStrings.keys()]
+Quality.SNATCHED_PROPER = [Quality.compositeStatus(SNATCHED_PROPER, x) for x in
+                           Quality.qualityStrings.keys()]
 
 MP3 = Quality.combineQualities([Quality.B192, Quality.B256, Quality.B320, Quality.VBR], [])
 LOSSLESS = Quality.combineQualities([Quality.FLAC], [])
-ANY = Quality.combineQualities([Quality.B192, Quality.B256, Quality.B320, Quality.VBR, Quality.FLAC], [])
+ANY = Quality.combineQualities(
+    [Quality.B192, Quality.B256, Quality.B320, Quality.VBR, Quality.FLAC], [])
 
 qualityPresets = (MP3, LOSSLESS, ANY)
 qualityPresetStrings = {MP3: "MP3 (All bitrates 192+)",

headphones/config.py

@@ -1,7 +1,8 @@
-import headphones.logger
 import itertools
+
 import os
 import re
+import headphones.logger
 from configobj import ConfigObj
 
 
@@ -14,6 +15,21 @@ def bool_int(value):
             value = 0
     return int(bool(value))
 
+
+class path(str):
+    """Internal 'marker' type for paths in config."""
+
+    @staticmethod
+    def __call__(val):
+        return path(val)
+
+    def __new__(cls, *args, **kw):
+        hstr = str.__new__(cls, *args, **kw)
+        return hstr
+
+    def __repr__(self):
+        return 'headphones.config.path(%s)' % self
+
 _CONFIG_DEFINITIONS = {
     'ADD_ALBUM_ART': (int, 'General', 0),
     'ADVANCEDENCODER': (str, 'General', ''),
@@ -29,11 +45,11 @@ _CONFIG_DEFINITIONS = {
     'AUTO_ADD_ARTISTS': (int, 'General', 1),
     'BITRATE': (int, 'General', 192),
     'BLACKHOLE': (int, 'General', 0),
-    'BLACKHOLE_DIR': (str, 'General', ''),
+    'BLACKHOLE_DIR': (path, 'General', ''),
     'BOXCAR_ENABLED': (int, 'Boxcar', 0),
     'BOXCAR_ONSNATCH': (int, 'Boxcar', 0),
     'BOXCAR_TOKEN': (str, 'Boxcar', ''),
-    'CACHE_DIR': (str, 'General', ''),
+    'CACHE_DIR': (path, 'General', ''),
     'CACHE_SIZEMB': (int, 'Advanced', 32),
     'CHECK_GITHUB': (int, 'General', 1),
     'CHECK_GITHUB_INTERVAL': (int, 'General', 360),
@@ -42,8 +58,8 @@ _CONFIG_DEFINITIONS = {
     'CONFIG_VERSION': (str, 'General', '0'),
     'CORRECT_METADATA': (int, 'General', 0),
     'CUE_SPLIT': (int, 'General', 1),
-    'CUE_SPLIT_FLAC_PATH': (str, 'General', ''),
-    'CUE_SPLIT_SHNTOOL_PATH': (str, 'General', ''),
+    'CUE_SPLIT_FLAC_PATH': (path, 'General', ''),
+    'CUE_SPLIT_SHNTOOL_PATH': (path, 'General', ''),
     'CUSTOMAUTH': (int, 'General', 0),
     'CUSTOMHOST': (str, 'General', 'localhost'),
     'CUSTOMPASS': (str, 'General', ''),
@@ -51,11 +67,17 @@ _CONFIG_DEFINITIONS = {
     'CUSTOMSLEEP': (int, 'General', 1),
     'CUSTOMUSER': (str, 'General', ''),
     'DELETE_LOSSLESS_FILES': (int, 'General', 1),
+    'DELUGE_HOST': (str, 'Deluge', ''),
+    'DELUGE_PASSWORD': (str, 'Deluge', ''),
+    'DELUGE_LABEL': (str, 'Deluge', ''),
+    'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''),
+    'DELUGE_PAUSED': (int, 'Deluge', 0),
     'DESTINATION_DIR': (str, 'General', ''),
     'DETECT_BITRATE': (int, 'General', 0),
-    'DOWNLOAD_DIR': (str, 'General', ''),
+    'DO_NOT_PROCESS_UNMATCHED': (int, 'General', 0),
+    'DOWNLOAD_DIR': (path, 'General', ''),
     'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5),
-    'DOWNLOAD_TORRENT_DIR': (str, 'General', ''),
+    'DOWNLOAD_TORRENT_DIR': (path, 'General', ''),
     'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0),
     'EMAIL_ENABLED': (int, 'Email', 0),
     'EMAIL_FROM': (str, 'Email', ''),
@@ -71,24 +93,27 @@ _CONFIG_DEFINITIONS = {
     'EMBED_LYRICS': (int, 'General', 0),
     'ENABLE_HTTPS': (int, 'General', 0),
     'ENCODER': (str, 'General', 'ffmpeg'),
-    'ENCODERFOLDER': (str, 'General', ''),
+    'ENCODERFOLDER': (path, 'General', ''),
     'ENCODERLOSSLESS': (int, 'General', 1),
     'ENCODEROUTPUTFORMAT': (str, 'General', 'mp3'),
     'ENCODERQUALITY': (int, 'General', 2),
     'ENCODERVBRCBR': (str, 'General', 'cbr'),
     'ENCODER_MULTICORE': (int, 'General', 0),
     'ENCODER_MULTICORE_COUNT': (int, 'General', 0),
-    'ENCODER_PATH': (str, 'General', ''),
+    'ENCODER_PATH': (path, 'General', ''),
     'EXTRAS': (str, 'General', ''),
     'EXTRA_NEWZNABS': (list, 'Newznab', ''),
+    'EXTRA_TORZNABS': (list, 'Torznab', ''),
     'FILE_FORMAT': (str, 'General', 'Track Artist - Album [Year] - Title'),
     'FILE_PERMISSIONS': (str, 'General', '0644'),
+    'FILE_PERMISSIONS_ENABLED': (bool_int, 'General', True),
     'FILE_UNDERSCORES': (int, 'General', 0),
     'FOLDER_FORMAT': (str, 'General', 'Artist/Album [Year]'),
+    'FOLDER_PERMISSIONS_ENABLED': (bool_int, 'General', True),
     'FOLDER_PERMISSIONS': (str, 'General', '0755'),
     'FREEZE_DB': (int, 'General', 0),
     'GIT_BRANCH': (str, 'General', 'master'),
-    'GIT_PATH': (str, 'General', ''),
+    'GIT_PATH': (path, 'General', ''),
     'GIT_USER': (str, 'General', 'rembo10'),
     'GROWL_ENABLED': (int, 'Growl', 0),
     'GROWL_HOST': (str, 'Growl', ''),
@@ -97,8 +122,8 @@ _CONFIG_DEFINITIONS = {
     'HEADPHONES_INDEXER': (bool_int, 'General', False),
     'HPPASS': (str, 'General', ''),
     'HPUSER': (str, 'General', ''),
-    'HTTPS_CERT': (str, 'General', ''),
-    'HTTPS_KEY': (str, 'General', ''),
+    'HTTPS_CERT': (path, 'General', ''),
+    'HTTPS_KEY': (path, 'General', ''),
     'HTTP_HOST': (str, 'General', 'localhost'),
     'HTTP_PASSWORD': (str, 'General', ''),
     'HTTP_PORT': (int, 'General', 8181),
@@ -108,8 +133,8 @@ _CONFIG_DEFINITIONS = {
     'IDTAG': (int, 'Beets', 0),
     'IGNORE_CLEAN_RELEASES': (int, 'General', 0),
     'IGNORED_WORDS': (str, 'General', ''),
-    'IGNORED_FOLDERS': (list, 'Advanced', []),
-    'IGNORED_FILES': (list, 'Advanced', []),
+    'IGNORED_FOLDERS': (list, 'Advanced', []),  # path
+    'IGNORED_FILES': (list, 'Advanced', []),    # path
     'INCLUDE_EXTRAS': (int, 'General', 0),
     'INTERFACE': (str, 'General', 'default'),
     'JOURNAL_MODE': (str, 'Advanced', 'wal'),
@@ -124,17 +149,17 @@ _CONFIG_DEFINITIONS = {
     'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
-    'LOG_DIR': (str, 'General', ''),
+    'LOG_DIR': (path, 'General', ''),
     'LOSSLESS_BITRATE_FROM': (int, 'General', 0),
     'LOSSLESS_BITRATE_TO': (int, 'General', 0),
-    'LOSSLESS_DESTINATION_DIR': (str, 'General', ''),
+    'LOSSLESS_DESTINATION_DIR': (path, 'General', ''),
     'MB_IGNORE_AGE': (int, 'General', 365),
     'MININOVA': (int, 'Mininova', 0),
     'MININOVA_RATIO': (str, 'Mininova', ''),
     'MIRROR': (str, 'General', 'musicbrainz.org'),
     'MOVE_FILES': (int, 'General', 0),
     'MPC_ENABLED': (bool_int, 'MPC', False),
-    'MUSIC_DIR': (str, 'General', ''),
+    'MUSIC_DIR': (path, 'General', ''),
     'MUSIC_ENCODER': (int, 'General', 0),
     'NEWZNAB': (int, 'Newznab', 0),
     'NEWZNAB_APIKEY': (str, 'Newznab', ''),
@@ -158,7 +183,7 @@ _CONFIG_DEFINITIONS = {
     'OMGWTFNZBS': (int, 'omgwtfnzbs', 0),
     'OMGWTFNZBS_APIKEY': (str, 'omgwtfnzbs', ''),
     'OMGWTFNZBS_UID': (str, 'omgwtfnzbs', ''),
-    'OPEN_MAGNET_LINKS': (int, 'General', 0),  # 0: Ignore, 1: Open, 2: Convert
+    'OPEN_MAGNET_LINKS': (int, 'General', 0),  # 0: Ignore, 1: Open, 2: Convert, 3: Embed (rtorrent)
     'MAGNET_LINKS': (int, 'General', 0),
     'OSX_NOTIFY_APP': (str, 'OSX_Notify', '/Applications/Headphones'),
     'OSX_NOTIFY_ENABLED': (int, 'OSX_Notify', 0),
@@ -201,6 +226,8 @@ _CONFIG_DEFINITIONS = {
     'PUSHOVER_ONSNATCH': (int, 'Pushover', 0),
     'PUSHOVER_PRIORITY': (int, 'Pushover', 0),
     'RENAME_FILES': (int, 'General', 0),
+    'RENAME_UNPROCESSED': (bool_int, 'General', 1),
+    'RENAME_FROZEN': (bool_int, 'General', 1),
     'REPLACE_EXISTING_FOLDERS': (int, 'General', 0),
     'KEEP_ORIGINAL_FOLDER': (int, 'General', 0),
     'REQUIRED_WORDS': (str, 'General', ''),
@@ -215,18 +242,29 @@ _CONFIG_DEFINITIONS = {
     'SAB_USERNAME': (str, 'SABnzbd', ''),
     'SAMPLINGFREQUENCY': (int, 'General', 44100),
     'SEARCH_INTERVAL': (int, 'General', 1440),
+    'SOFT_CHROOT': (path, 'General', ''),
     'SONGKICK_APIKEY': (str, 'Songkick', 'nd1We7dFW2RqxPw8'),
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
+    'STRIKE': (int, 'Strike', 0),
+    'STRIKE_RATIO': (str, 'Strike', ''),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
     'SUBSONIC_USERNAME': (str, 'Subsonic', ''),
     'SYNOINDEX_ENABLED': (int, 'Synoindex', 0),
+    'TELEGRAM_TOKEN': (str, 'Telegram', ''),
+    'TELEGRAM_USERID': (str, 'Telegram', ''),
+    'TELEGRAM_ENABLED': (int, 'Telegram', 0),
+    'TELEGRAM_ONSNATCH': (int, 'Telegram', 0),
     'TORRENTBLACKHOLE_DIR': (str, 'General', ''),
     'TORRENT_DOWNLOADER': (int, 'General', 0),
     'TORRENT_REMOVAL_INTERVAL': (int, 'General', 720),
+    'TORZNAB': (int, 'Torznab', 0),
+    'TORZNAB_APIKEY': (str, 'Torznab', ''),
+    'TORZNAB_ENABLED': (int, 'Torznab', 1),
+    'TORZNAB_HOST': (str, 'Torznab', ''),
     'TRANSMISSION_HOST': (str, 'Transmission', ''),
     'TRANSMISSION_PASSWORD': (str, 'Transmission', ''),
     'TRANSMISSION_USERNAME': (str, 'Transmission', ''),
@@ -242,7 +280,7 @@ _CONFIG_DEFINITIONS = {
     'UTORRENT_PASSWORD': (str, 'uTorrent', ''),
     'UTORRENT_USERNAME': (str, 'uTorrent', ''),
     'VERIFY_SSL_CERT': (bool_int, 'Advanced', 1),
-    'WAIT_UNTIL_RELEASE_DATE' : (int, 'General', 0),
+    'WAIT_UNTIL_RELEASE_DATE': (int, 'General', 0),
     'WAFFLES': (int, 'Waffles', 0),
     'WAFFLES_PASSKEY': (str, 'Waffles', ''),
     'WAFFLES_RATIO': (str, 'Waffles', ''),
@@ -260,6 +298,7 @@ _CONFIG_DEFINITIONS = {
     'XLDPROFILE': (str, 'General', '')
 }
 
+
 # pylint:disable=R0902
 # it might be nice to refactor for fewer instance variables
 class Config(object):
@@ -280,7 +319,7 @@ class Config(object):
         definition = _CONFIG_DEFINITIONS[key]
         if len(definition) == 3:
             definition_type, section, default = definition
-        else:
+        elif len(definition) == 4:
             definition_type, section, _, default = definition
         return key, definition_type, section, ini_key, default
 
@@ -336,7 +375,7 @@ class Config(object):
         """ Return the extra newznab tuples """
         extra_newznabs = list(
             itertools.izip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
-            for i in range(3)])
+                             for i in range(3)])
         )
         return extra_newznabs
 
@@ -351,6 +390,25 @@ class Config(object):
             extra_newznabs.append(item)
         self.EXTRA_NEWZNABS = extra_newznabs
 
+    def get_extra_torznabs(self):
+        """ Return the extra torznab tuples """
+        extra_torznabs = list(
+            itertools.izip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
+                             for i in range(3)])
+        )
+        return extra_torznabs
+
+    def clear_extra_torznabs(self):
+        """ Forget about the configured extra torznabs """
+        self.EXTRA_TORZNABS = []
+
+    def add_extra_torznab(self, torznab):
+        """ Add a new extra torznab """
+        extra_torznabs = self.EXTRA_TORZNABS
+        for item in torznab:
+            extra_torznabs.append(item)
+        self.EXTRA_TORZNABS = extra_torznabs
+
     def __getattr__(self, name):
         """
         Returns something from the ini unless it is a real property

headphones/config_test.py

@@ -0,0 +1,440 @@
+import mock
+from mock import MagicMock
+import headphones.config
+import re
+import unittestcompat
+from unittestcompat import TestCase, TestArgs
+
+
+class ConfigApiTest(TestCase):
+    """ Common tests for headphones.Config
+
+    Common tests for headphones.Config This test suite guarantees, that external
+    API of the Config class conforms all expectations of other modules.
+    """
+
+    def _setUpConfigMock(self, mock, sections):
+        # every constructor `xx = ConfigObj()` in headphones.config will return
+        # this mock:
+        self.config_mock = self.config_module_mock.return_value = mock
+
+        if sections:
+            mock.__contains__.side_effect = sections.__contains__
+            mock.__getitem__.side_effect = sections.__getitem__
+            mock.__setitem__.side_effect = sections.__setitem__
+            mock.items.side_effect = sections.items
+
+        return mock
+
+    def setUp(self):
+        # patch for low-level ConfigObj for entire test class
+        # result - each test_* method will get one additional
+        # argument during testing
+        self.config_module_mock_patcher = mock.patch('headphones.config.ConfigObj', name='ConfigObjModuleMock')
+        self.config_module_mock = self.config_module_mock_patcher.start()
+
+        existing_sections = {'General': {}, 'Email': {}}
+        # every constructor `xx = ConfigObj()` in headphones.config will return
+        # this mock:
+        self._setUpConfigMock(MagicMock(), existing_sections)
+
+    def tearDown(self):
+        self.config_module_mock_patcher.stop()
+
+    def test_constructor(self):
+        """ Config : creating """
+
+        cf = headphones.config.Config('/tmp/notexist')
+        self.assertIsInstance(cf, headphones.config.Config)
+
+    @TestArgs(
+        # this sections are explicitly added in the test body:
+        ('General', False),
+        ('Email', False),
+
+        # this sections will not be created nor in the test, either in the
+        # Config module
+        ('some_new_section_never_defined', True),
+        ('another_new_section_never_defined', True),
+    )
+    def test_check_section(self, section_name, expected_return):
+        """ Config : check_section """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.check_section(section_name)
+        res2 = c.check_section(section_name)
+
+        # assertions:
+        self.assertEqual(res, expected_return)
+        self.assertFalse(res2)
+
+    @TestArgs(
+        ('api_enabled', 0, int),
+        ('Api_Key', '', str),
+    )
+    def test_check_setting(self, setting_name, expected_return, expected_instance):
+        """ Config: check_setting , basic cases """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.check_setting(setting_name)
+        res2 = c.check_setting(setting_name)
+
+        # assertions:
+        self.assertIsInstance(res, expected_instance)
+        self.assertEqual(res, expected_return)
+        self.assertEqual(res, res2)
+
+    @TestArgs(
+        (''),
+        ('This_IsNew_Name'),
+    )
+    def test_check_setting_raise_on_unknown_settings(self, setting_name):
+        """ Config: check_setting should raise on unknown """
+        path = '/tmp/notexist'
+
+        exc_regex = re.compile(setting_name, re.IGNORECASE)
+
+        # call methods
+        c = headphones.config.Config(path)
+        # assertions:
+        with self.assertRaisesRegexp(KeyError, exc_regex):
+            c.check_setting(setting_name)
+        pass
+
+    @TestArgs(
+        (None)
+    )
+    def test_check_setting_raise_on_none(self, setting_name):
+        """ Config: check_setting shoud raise on None name """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        # assertions:
+        with self.assertRaises(AttributeError):
+            c.check_setting(setting_name)
+        pass
+
+    def test_write(self):
+        """ Config : write """
+        path = '/tmp/notexist'
+
+        # overload mocks, defined in setUp:
+        old_conf_mock = self._setUpConfigMock(MagicMock(), {'a': {}})
+
+        option_name_not_from_definitions = 'some_invalid_option_with_super_uniq1_name'
+        option_name_not_from_definitions_value = 1
+        old_conf_mock['asdf'] = {option_name_not_from_definitions: option_name_not_from_definitions_value}
+
+        # call methods
+        cf = headphones.config.Config(path)
+
+        # overload mock-patching for NEW CONFIG
+        new_patcher = mock.patch('headphones.config.ConfigObj', name='NEW_ConfigObjModuleMock_FOR_WRITE')
+
+        new_conf_module_mock = new_patcher.start()
+        new_conf_mock = \
+            new_conf_module_mock.return_value = \
+            MagicMock()
+        cf.write()
+        new_patcher.stop()
+
+        # assertions:
+        self.assertFalse(old_conf_mock.write.called, 'write not called for old config')
+        self.assertTrue(new_conf_mock.write.called, 'write called for new config')
+        self.assertEqual(new_conf_mock.filename, path)
+
+        new_conf_mock['General'].__setitem__.assert_any_call('download_dir', '')
+        # from 3.5... new_conf_mock['asdf'].__setitem__.assert_not_called('download_dir', '')
+        new_conf_mock['asdf'].__setitem__.assert_any_call(option_name_not_from_definitions, option_name_not_from_definitions_value)
+
+    @unittestcompat.skip("process_kwargs should be removed")
+    def test_process_kwargs(self):
+        self.assertTrue(True)
+
+    # ===========================================================
+    #   GET ATTR
+    # ===========================================================
+
+    @TestArgs(
+        ('ADD_ALBUM_ART', True),
+        ('ALBUM_ART_FORMAT', 'shmolder'),
+        ('API_ENABLED', 1),
+        ('API_KEY', 'Hello'),
+    )
+    def test__getattr__ConfValues(self, name, value):
+        """ Config: __getattr__ with setting value explicit """
+        path = '/tmp/notexist'
+
+        self.config_mock["General"] = {name.lower(): value}
+
+        # call methods
+        c = headphones.config.Config(path)
+        act = c.__getattr__(name)
+
+        # assertions:
+        self.assertEqual(act, value)
+
+    @TestArgs(
+        ('ADD_ALBUM_ART', 0),
+        ('ALBUM_ART_FORMAT', 'folder'),
+        ('API_ENABLED', 0),
+        ('API_KEY', ''),
+    )
+    def test__getattr__ConfValuesDefault(self, name, value):
+        """ Config: __getattr__ from config(by braces), default values """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.__getattr__(name)
+
+        # assertions:
+        self.assertEqual(res, value)
+
+    def test__getattr__ConfValuesDefaultUsingDotNotation(self):
+        """ Config: __getattr__ from config (by dot), default values """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+
+        # assertions:
+        self.assertEqual(c.ALBUM_ART_FORMAT, 'folder')
+        self.assertEqual(c.API_ENABLED, 0)
+        self.assertEqual(c.API_KEY, '')
+
+    def test__getattr__OwnAttributes(self):
+        """ Config: __getattr__ access own attrs """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+
+        # assertions:
+        self.assertIsNotNone(c)
+        self.assertIn('<headphones.config.Config', c.__str__())
+
+    # ===========================================================
+    #   SET ATTR
+    # ===========================================================
+
+    @TestArgs(
+        ('ADD_ALBUM_ART', True),
+        ('ALBUM_ART_FORMAT', 'shmolder'),
+        ('API_ENABLED', 1),
+        ('API_KEY', 'Hello'),
+    )
+    def test__setattr__ConfValuesDefault(self, name, value):
+        """ Config: __setattr__ with setting value explicit """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        act = c.__setattr__(name, value)
+
+        # assertions:
+        self.assertEqual(self.config_mock["General"][name.lower()], value)
+        self.assertEqual(act, value)
+
+    def test__setattr__ExplicitSetUsingDotNotation(self):
+        """ Config: __setattr__ with setting values using dot notation """
+        path = '/tmp/notexist'
+
+        # call methods
+        c = headphones.config.Config(path)
+        act1 = c.ALBUM_ART_FORMAT = 'Apple'
+        act2 = c.API_ENABLED = True
+        act3 = c.API_KEY = 123
+
+        # assertions:
+        self.assertEqual(self.config_mock["General"]['album_art_format'], 'Apple')
+        self.assertEqual(self.config_mock["General"]['api_enabled'], 1)
+        self.assertEqual(self.config_mock["General"]['api_key'], '123')
+
+        self.assertEqual(act1, 'Apple')
+        self.assertEqual(act2, 1)
+
+        # TODO : check this trange behaviour. I have expected to see here '123', not 123.
+        self.assertEqual(act3, 123)
+
+    # ===========================================================
+    #   NEWZNABS
+    #
+
+    @TestArgs(
+        ('', []),
+        ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
+        (['ABC', 'DEF'], []),
+        ([1], []),
+        ([1, 2], []),
+        ([1, 2, 3], [(1, 2, 3)]),
+
+        ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
+        ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
+        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    )
+    def test_get_extra_newznabs(self, conf_value, expected):
+        """ Config: get_extra_newznabs """
+        path = '/tmp/notexist'
+
+        #itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
+        # set up mocks:
+        # 'EXTRA_NEWZNABS': (list, 'Newznab', ''),
+        # 'EXTRA_TORZNABS': (list, 'Torznab', ''),
+        self.config_mock["Newznab"] = {"extra_newznabs": conf_value}
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.get_extra_newznabs()
+
+        # assertions:
+        self.assertEqual(res, expected)
+
+    def test_clear_extra_newznabs(self):
+        """ Config: clear_extra_newznabs """
+        path = '/tmp/notexist'
+
+        random_value = 1827746
+        self.config_mock["Newznab"] = {"extra_newznabs": [1, 2, 3]}
+        self.config_mock["Newznab"] = {"do_not_touch": random_value}
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.clear_extra_newznabs()
+
+        # assertions:
+        self.assertIsNone(res)
+        self.assertEqual(self.config_mock["Newznab"]["extra_newznabs"], [])
+        self.assertEqual(self.config_mock["Newznab"]["do_not_touch"], random_value)
+
+    @TestArgs(
+        ([], [''], ['']),
+        ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
+
+        ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
+    )
+    def test_add_extra_newznab(self, initial, added, expected):
+        """ Config: add_extra_newznab """
+        path = '/tmp/notexist'
+
+        self.config_mock["Newznab"] = {"extra_newznabs": initial}
+
+        # call methods
+        c = headphones.config.Config(path)
+        c.add_extra_newznab(added)
+        act = self.config_mock["Newznab"]["extra_newznabs"]
+
+        # assertions:
+        self.assertEqual(act, expected)
+
+    @TestArgs(
+        (None),
+        ([]),
+        ([1, 2, 3]),
+        ([True]),
+    )
+    def test_add_extra_newznab_raise_on_none(self, initial):
+        """ Config: add_extra_newznab should raise on None adding"""
+        path = '/tmp/notexist'
+
+        self.config_mock["Newznab"] = {"extra_newznabs": initial}
+
+        # call methods
+        c = headphones.config.Config(path)
+        with self.assertRaises(TypeError):
+            c.add_extra_newznab(None)
+        pass
+
+    # ===========================================================
+    #   TORZNABS
+    # TODO : here is copypaste from of NEZNABS tests. Make tests better, plz refactor them
+    #
+    @TestArgs(
+        ('', []),
+        ('ABCDEF', [('A', 'B', 'C'), ('D', 'E', 'F')]),
+        (['ABC', 'DEF'], []),
+        ([1], []),
+        ([1, 2], []),
+        ([1, 2, 3], [(1, 2, 3)]),
+
+        ([1, 2, 3, 'Aaa'], [(1, 2, 3)]),
+        ([1, 2, 3, 'Aaa', 'Bbba'], [(1, 2, 3)]),
+        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+        ([1, 2, 3, 'Aaa', 'Bbba', 'Ccccc', 'Ddddda'], [(1, 2, 3), ('Aaa', 'Bbba', 'Ccccc')]),
+    )
+    def test_get_extra_torznabs(self, conf_value, expected):
+        """ Config: get_extra_torznabs """
+        path = '/tmp/notexist'
+
+        #itertools.izip(*[itertools.islice('', i, None, 3) for i in range(3)])
+        # set up mocks:
+        # 'EXTRA_TORZNABS': (list, '', ''),
+        self.config_mock["Torznab"] = {"extra_torznabs": conf_value}
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.get_extra_torznabs()
+
+        # assertions:
+        self.assertEqual(res, expected)
+
+    def test_clear_extra_torznabs(self):
+        """ Config: clear_extra_torznabs """
+        path = '/tmp/notexist'
+
+        random_value = -1292721
+        self.config_mock["Torznab"] = {"extra_torznabs": [1, 2, 3]}
+        self.config_mock["Torznab"] = {"do_not_touch": random_value}
+
+        # call methods
+        c = headphones.config.Config(path)
+        res = c.clear_extra_torznabs()
+
+        # assertions:
+        self.assertIsNone(res)
+        self.assertEqual(self.config_mock["Torznab"]["extra_torznabs"], [])
+        self.assertEqual(self.config_mock["Torznab"]["do_not_touch"], random_value)
+
+    @TestArgs(
+        ([], [''], ['']),
+        ([], 'ABCDEF', ['A', 'B', 'C', 'D', 'E', 'F']),
+
+        ([1, 2, [False, True]], ['3', [0, 0]], [1, 2, [False, True], '3', [0, 0]]),
+    )
+    def test_add_extra_torznab(self, initial, added, expected):
+        """ Config: add_extra_torznab """
+        path = '/tmp/notexist'
+
+        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+
+        # call methods
+        c = headphones.config.Config(path)
+        c.add_extra_torznab(added)
+        act = self.config_mock["Torznab"]["extra_torznabs"]
+
+        # assertions:
+        self.assertEqual(act, expected)
+
+    @TestArgs(
+        (None),
+        ([]),
+        ([1, 2, 3]),
+        ([True]),
+    )
+    def test_add_extra_torznab_raise_on_none(self, initial):
+        """ Config: add_extra_torznab should raise on None adding"""
+        path = '/tmp/notexist'
+
+        self.config_mock["Torznab"] = {"extra_torznabs": initial}
+
+        # call methods
+        c = headphones.config.Config(path)
+        with self.assertRaises(TypeError):
+            c.add_extra_torznab(None)
+        pass

headphones/cuesplit.py

@@ -15,13 +15,13 @@
 
 # Most of this lifted from here: https://github.com/SzieberthAdam/gneposis-cdgrab
 
-import os
 import sys
-import re
 import subprocess
 import copy
 import glob
 
+import os
+import re
 import headphones
 from headphones import logger
 from mutagen.flac import FLAC
@@ -62,7 +62,7 @@ WAVE_FILE_TYPE_BY_EXTENSION = {
     '.flac': 'Free Lossless Audio Codec'
 }
 
-#SHNTOOL_COMPATIBLE = ("Free Lossless Audio Codec", "Waveform Audio", "Monkey's Audio")
+# SHNTOOL_COMPATIBLE = ("Free Lossless Audio Codec", "Waveform Audio", "Monkey's Audio")
 
 # TODO: Make this better!
 # this module-level variable is bad. :(
@@ -288,7 +288,7 @@ class CueFile(File):
             global line_content
             c = self.content.splitlines()
             header_dict = {}
-            #remaining_headers = CUE_HEADER
+            # remaining_headers = CUE_HEADER
             remaining_headers = copy.copy(CUE_HEADER)
             line_index = 0
             match = True
@@ -314,7 +314,8 @@ class CueFile(File):
             line_content = c[line_index]
             search_result = re.search(CUE_TRACK, line_content, re.I)
             if not search_result:
-                raise ValueError('inconsistent CUE sheet, TRACK expected at line {0}'.format(line_index + 1))
+                raise ValueError(
+                    'inconsistent CUE sheet, TRACK expected at line {0}'.format(line_index + 1))
             track_nr = int(search_result.group(1))
             line_index += 1
             next_track = False
@@ -353,7 +354,8 @@ class CueFile(File):
                         track_meta['dcpflag'] = True
                         line_index += 1
                     else:
-                        raise ValueError('unknown entry in track error, line {0}'.format(line_index + 1))
+                        raise ValueError(
+                            'unknown entry in track error, line {0}'.format(line_index + 1))
                 else:
                     next_track = True
 
@@ -371,8 +373,8 @@ class CueFile(File):
 
         if not self.content:
             try:
-                 with open(self.name, encoding="cp1252") as cue_file:
-                     self.content = cue_file.read()
+                with open(self.name, encoding="cp1252") as cue_file:
+                    self.content = cue_file.read()
             except:
                 raise ValueError('Cant encode CUE Sheet.')
 
@@ -406,9 +408,11 @@ class CueFile(File):
         for i in range(len(self.tracks)):
             if self.tracks[i]:
                 if self.tracks[i].get('artist'):
-                    content += 'track' + int_to_str(i) + 'artist' + '\t' + self.tracks[i].get('artist') + '\n'
+                    content += 'track' + int_to_str(i) + 'artist' + '\t' + self.tracks[i].get(
+                        'artist') + '\n'
                 if self.tracks[i].get('title'):
-                    content += 'track' + int_to_str(i) + 'title' + '\t' + self.tracks[i].get('title') + '\n'
+                    content += 'track' + int_to_str(i) + 'title' + '\t' + self.tracks[i].get(
+                        'title') + '\n'
         return content
 
     def htoa(self):
@@ -449,7 +453,8 @@ class MetaFile(File):
                         raise ValueError('Syntax error in album meta file')
                     if not content['tracks'][int(parsed_track.group(1))]:
                         content['tracks'][int(parsed_track.group(1))] = dict()
-                    content['tracks'][int(parsed_track.group(1))][parsed_track.group(2)] = parsed_line.group(2)
+                    content['tracks'][int(parsed_track.group(1))][
+                        parsed_track.group(2)] = parsed_line.group(2)
                 else:
                     content[parsed_line.group(1)] = parsed_line.group(2)
 
@@ -472,15 +477,16 @@ class MetaFile(File):
         if 'genre' in CUE_META.content:
             common_tags['genre'] = CUE_META.content['genre']
 
-        #freeform tags
-        #freeform_tags['country'] = self.content['country']
-        #freeform_tags['releasedate'] = self.content['releasedate']
+        # freeform tags
+        # freeform_tags['country'] = self.content['country']
+        # freeform_tags['releasedate'] = self.content['releasedate']
 
         return common_tags, freeform_tags
 
     def folders(self):
         artist = self.content['artist']
-        album = self.content['date'] + ' - ' + self.content['title'] + ' (' + self.content['label'] + ' - ' + self.content['catalog'] + ')'
+        album = self.content['date'] + ' - ' + self.content['title'] + ' (' + self.content[
+            'label'] + ' - ' + self.content['catalog'] + ')'
         return artist, album
 
     def complete(self):
@@ -535,6 +541,7 @@ class WaveFile(File):
         if self.type == 'Free Lossless Audio Codec':
             return FLAC(self.name)
 
+
 def split(albumpath):
     global CUE_META
     os.chdir(albumpath)
@@ -577,7 +584,8 @@ def split(albumpath):
         import getXldProfile
         xldprofile, xldformat, _ = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE)
         if not xldformat:
-            raise ValueError('Details for xld profile "%s" not found, cannot split cue' % (xldprofile))
+            raise ValueError(
+                'Details for xld profile "%s" not found, cannot split cue' % (xldprofile))
         else:
             if headphones.CONFIG.ENCODERFOLDER:
                 splitter = os.path.join(headphones.CONFIG.ENCODERFOLDER, 'xld')
@@ -590,7 +598,7 @@ def split(albumpath):
             splitter = 'shntool'
 
     if splitter == 'shntool' and not check_splitter(splitter):
-            raise ValueError('Command not found, ensure shntool or xld installed')
+        raise ValueError('Command not found, ensure shntool or xld installed')
 
     # Determine if file can be split
     if wave.name_ext not in WAVE_FILE_TYPE_BY_EXTENSION.keys():

headphones/db.py

@@ -13,44 +13,41 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-#####################################
-## Stolen from Sick-Beard's db.py  ##
-#####################################
+###################################
+# Stolen from Sick-Beard's db.py  #
+###################################
 
 from __future__ import with_statement
 
-import os
 import sqlite3
 
+import os
 import headphones
-
 from headphones import logger
 
 
 def dbFilename(filename="headphones.db"):
-
     return os.path.join(headphones.DATA_DIR, filename)
 
 
 def getCacheSize():
-    #this will protect against typecasting problems produced by empty string and None settings
+    # this will protect against typecasting problems produced by empty string and None settings
     if not headphones.CONFIG.CACHE_SIZEMB:
-        #sqlite will work with this (very slowly)
+        # sqlite will work with this (very slowly)
         return 0
     return int(headphones.CONFIG.CACHE_SIZEMB)
 
 
 class DBConnection:
-
     def __init__(self, filename="headphones.db"):
 
         self.filename = filename
         self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
-        #don't wait for the disk to finish writing
+        # don't wait for the disk to finish writing
         self.connection.execute("PRAGMA synchronous = OFF")
-        #journal disabled since we never do rollbacks
+        # journal disabled since we never do rollbacks
         self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE)
-        #64mb of cache memory,probably need to make it user configurable
+        # 64mb of cache memory,probably need to make it user configurable
         self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024))
         self.connection.row_factory = sqlite3.Row
 
@@ -92,17 +89,20 @@ class DBConnection:
 
     def upsert(self, tableName, valueDict, keyDict):
 
+        def genParams(myDict):
+            return [x + " = ?" for x in myDict.keys()]
+
         changesBefore = self.connection.total_changes
 
-        genParams = lambda myDict: [x + " = ?" for x in myDict.keys()]
-
-        update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
+        update_query = "UPDATE " + tableName + " SET " + ", ".join(
+            genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
 
         self.action(update_query, valueDict.values() + keyDict.values())
 
         if self.connection.total_changes == changesBefore:
             insert_query = (
-                "INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" +
+                "INSERT INTO " + tableName + " (" + ", ".join(
+                    valueDict.keys() + keyDict.keys()) + ")" +
                 " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
             )
             try:

headphones/deluge.py

@@ -0,0 +1,460 @@
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
+
+# Parts of this file are a part of SickRage.
+# Author: Mr_Orange <mr_orange@hotmail.it>
+# URL: http://code.google.com/p/sickbeard/
+# Adapted for Headphones by <noamgit@gmail.com>
+# URL: https://github.com/noam09
+#
+# SickRage is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# SickRage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with SickRage. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+from headphones import logger
+#from headphones import request
+
+import time
+import re
+import os
+import json
+import headphones
+import requests
+from base64 import b64encode
+import traceback
+
+delugeweb_auth = {}
+delugeweb_url = ''
+
+
+def addTorrent(link, data=None):
+    try:
+        result = {}
+        retid = False
+
+        if link.startswith('magnet:'):
+            logger.debug('Deluge: Got a magnet link: %s' % link)
+            result = {'type': 'magnet',
+                        'url': link}
+            retid = _add_torrent_magnet(result)
+
+        elif link.startswith('http://') or link.startswith('https://'):
+            logger.debug('Deluge: Got a URL: %s' % link)
+            user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
+            headers = {'User-Agent': user_agent}
+            torrentfile = ''
+            logger.debug('Deluge: Trying to download (GET)')
+            try:
+                r = requests.get(link, headers=headers)
+                if r.status_code == 200:
+                    logger.debug('Deluge: 200 OK')
+                    torrentfile = r.text
+                    #for chunk in r.iter_content(chunk_size=1024):
+                    #    if chunk: # filter out keep-alive new chunks
+                    #        torrentfile = torrentfile + chunk
+                else:
+                    logger.debug('Deluge: Trying to GET %s returned status %d' % (link, r.status_code))
+                    return False
+            except Exception as e:
+                logger.debug('Deluge: Download failed: %s' % str(e))
+            if 'announce' not in torrentfile[:40]:
+                logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % link)
+                return False
+            # Extract torrent name from .torrent
+            try:
+                logger.debug('Deluge: Getting torrent name length')
+                name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0])
+                logger.debug('Deluge: Getting torrent name')
+                name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length]
+            except Exception as e:
+                logger.debug('Deluge: Could not get torrent name, getting file name')
+                # get last part of link/path (name only)
+                name = link.split('\\')[-1].split('/')[-1]
+                # remove '.torrent' suffix
+                if name[-len('.torrent'):] == '.torrent':
+                    name = name[:-len('.torrent')]
+            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40]))
+            result = {'type': 'torrent',
+                        'name': name,
+                        'content': torrentfile}
+            retid = _add_torrent_file(result)
+
+        # elif link.endswith('.torrent') or data:
+        elif not (link.startswith('http://') or link.startswith('https://')):
+            if data:
+                logger.debug('Deluge: Getting .torrent data')
+                torrentfile = data
+            else:
+                logger.debug('Deluge: Getting .torrent file')
+                with open(link, 'rb') as f:
+                    torrentfile = f.read()
+            # Extract torrent name from .torrent
+            try:
+                logger.debug('Deluge: Getting torrent name length')
+                name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0])
+                logger.debug('Deluge: Getting torrent name')
+                name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length]
+            except Exception as e:
+                logger.debug('Deluge: Could not get torrent name, getting file name')
+                # get last part of link/path (name only)
+                name = link.split('\\')[-1].split('/')[-1]
+                # remove '.torrent' suffix
+                if name[-len('.torrent'):] == '.torrent':
+                    name = name[:-len('.torrent')]
+            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40]))
+            result = {'type': 'torrent',
+                        'name': name,
+                        'content': torrentfile}
+            retid = _add_torrent_file(result)
+
+        else:
+            logger.error('Deluge: Unknown file type: %s' % link)
+
+        if retid:
+            logger.info('Deluge: Torrent sent to Deluge successfully  (%s)' % retid)
+            return retid
+        else:
+            logger.info('Deluge returned status %s' % retid)
+            return False
+
+    except Exception as e:
+        logger.error(str(e))
+        formatted_lines = traceback.format_exc().splitlines()
+        logger.error('; '.join(formatted_lines))
+
+
+def getTorrentFolder(result):
+    logger.debug('Deluge: Get torrent folder name')
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    try:
+        post_data = json.dumps({"method": "web.get_torrent_status",
+                                "params": [
+                                    result['hash'],
+                                    ["total_done"]
+                                ],
+                                "id": 22})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        result['total_done'] = json.loads(response.text)['result']['total_done']
+
+        tries = 0
+        while result['total_done'] == 0 and tries < 10:
+            tries += 1
+            time.sleep(5)
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+            result['total_done'] = json.loads(response.text)['result']['total_done']
+
+        post_data = json.dumps({"method": "web.get_torrent_status",
+                                "params": [
+                                    result['hash'],
+                                    [
+                                        "name",
+                                        "save_path",
+                                        "total_size",
+                                        "num_files",
+                                        "message",
+                                        "tracker",
+                                        "comment"
+                                    ]
+                                ],
+                                "id": 23})
+
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+
+        result['save_path'] = json.loads(response.text)['result']['save_path']
+        result['name'] = json.loads(response.text)['result']['name']
+
+        return json.loads(response.text)['result']['name']
+    except Exception as e:
+        logger.debug('Deluge: Could not get torrent folder name: %s' % str(e))
+
+
+def removeTorrent(torrentid, remove_data=False):
+
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    result = False
+    post_data = json.dumps({"method": "core.remove_torrent",
+                            "params": [
+                                torrentid,
+                                remove_data
+                                ],
+                            "id": 25})
+    response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+    result = json.loads(response.text)['result']
+
+    return result
+
+
+def _get_auth():
+    logger.debug('Deluge: Authenticating...')
+    global delugeweb_auth, delugeweb_url
+    delugeweb_auth = {}
+
+    delugeweb_host = headphones.CONFIG.DELUGE_HOST
+    delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD
+
+    if not delugeweb_host.startswith('http'):
+        delugeweb_host = 'http://%s' % delugeweb_host
+
+    if delugeweb_host.endswith('/'):
+        delugeweb_host = delugeweb_host[:-1]
+
+    delugeweb_url = delugeweb_host + '/json'
+
+    post_data = json.dumps({"method": "auth.login",
+                            "params": [delugeweb_password],
+                            "id": 1})
+    try:
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        #                                  , verify=TORRENT_VERIFY_CERT)
+    except Exception:
+        return None
+
+    auth = json.loads(response.text)["result"]
+    delugeweb_auth = response.cookies
+
+    post_data = json.dumps({"method": "web.connected",
+                            "params": [],
+                            "id": 10})
+    try:
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        #                                  , verify=TORRENT_VERIFY_CERT)
+    except Exception:
+        return None
+
+    connected = json.loads(response.text)['result']
+
+    if not connected:
+        post_data = json.dumps({"method": "web.get_hosts",
+                                "params": [],
+                                "id": 11})
+        try:
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+            #                                  , verify=TORRENT_VERIFY_CERT)
+        except Exception:
+            return None
+
+        delugeweb_hosts = json.loads(response.text)['result']
+        if len(delugeweb_hosts) == 0:
+            logger.error('Deluge: WebUI does not contain daemons')
+            return None
+
+        post_data = json.dumps({"method": "web.connect",
+                                "params": [delugeweb_hosts[0][0]],
+                                "id": 11})
+
+        try:
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+            #                                  , verify=TORRENT_VERIFY_CERT)
+        except Exception:
+            return None
+
+        post_data = json.dumps({"method": "web.connected",
+                                "params": [],
+                                "id": 10})
+
+        try:
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+            #                                  , verify=TORRENT_VERIFY_CERT)
+        except Exception:
+            return None
+
+        connected = json.loads(response.text)['result']
+
+        if not connected:
+            logger.error('Deluge: WebUI could not connect to daemon')
+            return None
+
+    return auth
+
+
+def _add_torrent_magnet(result):
+    logger.debug('Deluge: Adding magnet')
+    if not any(delugeweb_auth):
+        _get_auth()
+    try:
+        post_data = json.dumps({"method": "core.add_torrent_magnet",
+                                "params": [result['url'], {}],
+                                "id": 2})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        result['hash'] = json.loads(response.text)['result']
+        logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result']))
+        return json.loads(response.text)['result']
+    except Exception as e:
+        logger.error('Deluge: Adding torrent magnet failed: %s' % str(e))
+
+'''
+def _add_torrent_url(result):
+    logger.debug('Deluge: Adding URL')
+    if not any(delugeweb_auth):
+        _get_auth()
+    try:
+        post_data = json.dumps({"method": "web.download_torrent_from_url",
+                                "params": [result['url'], {}],
+                                "id": 2})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        result['hash'] = json.loads(response.text)['result']
+        logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result']))
+        return json.loads(response.text)['result']
+    except Exception as e:
+        logger.error('Deluge: Adding torrent URL failed: %s' % str(e))
+'''
+
+
+def _add_torrent_file(result):
+    logger.debug('Deluge: Adding file')
+    if not any(delugeweb_auth):
+        _get_auth()
+    try:
+        # content is torrent file contents that needs to be encoded to base64
+        post_data = json.dumps({"method": "core.add_torrent_file",
+                                "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}],
+                                "id": 2})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        result['hash'] = json.loads(response.text)['result']
+        logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result']))
+        return json.loads(response.text)['result']
+    except Exception as e:
+        logger.error('Deluge: Adding torrent file failed: %s' % str(e))
+        formatted_lines = traceback.format_exc().splitlines()
+        logger.error('; '.join(formatted_lines))
+
+
+def setTorrentLabel(result):
+    logger.debug('Deluge: Setting label')
+    label = headphones.CONFIG.DELUGE_LABEL
+
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    if ' ' in label:
+        logger.error('Deluge: Invalid label. Label can\'t contain spaces - replacing with underscores')
+        label = label.replace(' ', '_')
+    if label:
+        # check if label already exists and create it if not
+        post_data = json.dumps({"method": 'label.get_labels',
+                                "params": [],
+                                "id": 3})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        labels = json.loads(response.text)['result']
+
+        if labels is not None:
+            if label not in labels:
+                try:
+                    logger.debug('Deluge: %s label doesn\'t exist in Deluge, let\'s add it' % label)
+                    post_data = json.dumps({"method": 'label.add',
+                                            "params": [label],
+                                            "id": 4})
+                    response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+                    logger.debug('Deluge: %s label added to Deluge' % label)
+                except Exception as e:
+                    logger.error('Deluge: Setting label failed: %s' % str(e))
+                    formatted_lines = traceback.format_exc().splitlines()
+                    logger.error('; '.join(formatted_lines))
+
+            # add label to torrent
+            post_data = json.dumps({"method": 'label.set_torrent',
+                                    "params": [result['hash'], label],
+                                    "id": 5})
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+            logger.debug('Deluge: %s label added to torrent' % label)
+        else:
+            logger.debug('Deluge: Label plugin not detected')
+            return False
+
+    return not json.loads(response.text)['error']
+
+
+def setSeedRatio(result):
+    logger.debug('Deluge: Setting seed ratio')
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    ratio = None
+    if result['ratio']:
+        ratio = result['ratio']
+
+    if ratio:
+        post_data = json.dumps({"method": "core.set_torrent_stop_at_ratio",
+                                "params": [result['hash'], True],
+                                "id": 5})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+        post_data = json.dumps({"method": "core.set_torrent_stop_ratio",
+                                "params": [result['hash'], float(ratio)],
+                                "id": 6})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+
+        return not json.loads(response.text)['error']
+
+    return True
+
+
+def setTorrentPath(result):
+    logger.debug('Deluge: Setting download path')
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
+        post_data = json.dumps({"method": "core.set_torrent_move_completed",
+                                "params": [result['hash'], True],
+                                "id": 7})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+
+        if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
+            move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
+        else:
+            move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
+
+        if not os.path.exists(move_to):
+            logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to)
+            os.makedirs(move_to)
+        post_data = json.dumps({"method": "core.set_torrent_move_completed_path",
+                                "params": [result['hash'], move_to],
+                                "id": 8})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+
+        return not json.loads(response.text)['error']
+
+    return True
+
+
+def setTorrentPause(result):
+    logger.debug('Deluge: Pausing torrent')
+    if not any(delugeweb_auth):
+        _get_auth()
+
+    if headphones.CONFIG.DELUGE_PAUSED:
+        post_data = json.dumps({"method": "core.pause_torrent",
+                                "params": [[result['hash']]],
+                                "id": 9})
+        response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth)
+
+        return not json.loads(response.text)['error']
+
+    return True

headphones/exceptions.py

@@ -24,3 +24,10 @@ class NewzbinAPIThrottled(HeadphonesException):
     """
     Newzbin has throttled us, deal with it
     """
+
+
+class SoftChrootError(HeadphonesException):
+    """
+    Fatal errors in SoftChroot module
+    """
+    pass

headphones/getXldProfile.py

@@ -1,29 +1,29 @@
 import os.path
+
 import biplist
 from headphones import logger
 
 
 def getXldProfile(xldProfile):
-
     xldProfileNotFound = xldProfile
 
     expanded = os.path.expanduser('~/Library/Preferences/jp.tmkk.XLD.plist')
     if not os.path.isfile(expanded):
         logger.warn("Could not find xld preferences at: %s", expanded)
-        return(xldProfileNotFound, None, None)
+        return (xldProfileNotFound, None, None)
 
     # Get xld preferences plist
     try:
         preferences = biplist.readPlist(expanded)
     except (biplist.InvalidPlistException, biplist.NotBinaryPlistException), e:
         logger.error("Error reading xld preferences plist: %s", e)
-        return(xldProfileNotFound, None, None)
+        return (xldProfileNotFound, None, None)
 
     if not isinstance(preferences, dict):
         logger.error("Error reading xld preferences plist, not a dict: %r", preferences)
-        return(xldProfileNotFound, None, None)
+        return (xldProfileNotFound, None, None)
 
-    profiles = preferences.get('Profiles', []) # pylint:disable=E1103
+    profiles = preferences.get('Profiles', [])  # pylint:disable=E1103
 
     xldProfile = xldProfile.lower()
     for profile in profiles:
@@ -175,6 +175,6 @@ def getXldProfile(xldProfile):
             if xldFormat and not xldBitrate:
                 xldBitrate = 400
 
-            return(xldProfileForCmd, xldFormat, xldBitrate)
+            return (xldProfileForCmd, xldFormat, xldBitrate)
 
-    return(xldProfileNotFound, None, None)
+    return (xldProfileNotFound, None, None)

headphones/helpers.py

@@ -13,19 +13,19 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
-
 from operator import itemgetter
-
 import unicodedata
-import headphones
 import datetime
-import fnmatch
 import shutil
 import time
 import sys
+
+import fnmatch
 import re
 import os
+from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+import headphones
+
 
 # Modified from https://github.com/Verrus/beets-plugin-featInTitle
 RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.")
@@ -35,7 +35,9 @@ RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
 
 
 def multikeysort(items, columns):
-    comparers = [((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns]
+    comparers = [
+        ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1))
+        for col in columns]
 
     def comparer(left, right):
         for fn, mult in comparers:
@@ -56,7 +58,6 @@ def checked(variable):
 
 
 def radio(variable, pos):
-
     if variable == pos:
         return 'Checked'
     else:
@@ -70,7 +71,7 @@ def latinToAscii(unicrap):
     xlate = {
         0xc0: 'A', 0xc1: 'A', 0xc2: 'A', 0xc3: 'A', 0xc4: 'A', 0xc5: 'A',
         0xc6: 'Ae', 0xc7: 'C',
-        0xc8: 'E', 0xc9: 'E', 0xca: 'E', 0xcb: 'E', 0x86: 'e',
+        0xc8: 'E', 0xc9: 'E', 0xca: 'E', 0xcb: 'E', 0x86: 'e', 0x39e: 'E',
         0xcc: 'I', 0xcd: 'I', 0xce: 'I', 0xcf: 'I',
         0xd0: 'Th', 0xd1: 'N',
         0xd2: 'O', 0xd3: 'O', 0xd4: 'O', 0xd5: 'O', 0xd6: 'O', 0xd8: 'O',
@@ -107,7 +108,6 @@ def latinToAscii(unicrap):
 
 
 def convert_milliseconds(ms):
-
     seconds = ms / 1000
     gmtime = time.gmtime(seconds)
     if seconds > 3600:
@@ -119,7 +119,6 @@ def convert_milliseconds(ms):
 
 
 def convert_seconds(s):
-
     gmtime = time.gmtime(s)
     if s > 3600:
         minutes = time.strftime("%H:%M:%S", gmtime)
@@ -141,7 +140,6 @@ def now():
 
 
 def get_age(date):
-
     try:
         split_date = date.split('-')
     except:
@@ -149,14 +147,13 @@ def get_age(date):
 
     try:
         days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2])
-    except (IndexError,ValueError):
+    except (IndexError, ValueError):
         days_old = False
 
     return days_old
 
 
 def bytes_to_mb(bytes):
-
     mb = int(bytes) / 1048576
     size = '%.1f MB' % mb
     return size
@@ -172,7 +169,7 @@ def piratesize(size):
     split = size.split(" ")
     factor = float(split[0])
     unit = split[1].upper()
-    
+
     if unit == 'MIB':
         size = factor * 1048576
     elif unit == 'MB':
@@ -194,12 +191,13 @@ def piratesize(size):
 
 
 def replace_all(text, dic, normalize=False):
-
+    from headphones import pathrender
     if not text:
         return ''
 
-    for i, j in dic.iteritems():
-        if normalize:
+    if normalize:
+        new_dic = {}
+        for i, j in dic.iteritems():
             try:
                 if sys.platform == 'darwin':
                     j = unicodedata.normalize('NFD', j)
@@ -207,8 +205,9 @@ def replace_all(text, dic, normalize=False):
                     j = unicodedata.normalize('NFC', j)
             except TypeError:
                 j = unicodedata.normalize('NFC', j.decode(headphones.SYS_ENCODING, 'replace'))
-        text = text.replace(i, j)
-    return text
+            new_dic[i] = j
+        dic = new_dic
+    return pathrender.render(text, dic)[0]
 
 
 def replace_illegal_chars(string, type="file"):
@@ -221,15 +220,14 @@ def replace_illegal_chars(string, type="file"):
 
 
 def cleanName(string):
-
     pass1 = latinToAscii(string).lower()
-    out_string = re.sub('[\.\-\/\!\@\#\$\%\^\&\*\(\)\+\-\"\'\,\;\:\[\]\{\}\<\>\=\_]', '', pass1).encode('utf-8')
+    out_string = re.sub('[\.\-\/\!\@\#\$\%\^\&\*\(\)\+\-\"\'\,\;\:\[\]\{\}\<\>\=\_]', '',
+                        pass1).encode('utf-8')
 
     return out_string
 
 
 def cleanTitle(title):
-
     title = re.sub('[\.\-\/\_]', ' ', title).lower()
 
     # Strip out extra whitespace
@@ -312,16 +310,22 @@ def expand_subfolders(f):
     difference = max(path_depths) - min(path_depths)
 
     if difference > 0:
-        logger.info("Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.", len(media_folders), difference)
+        logger.info(
+            "Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.",
+            len(media_folders), difference)
 
         # While already failed, advice the user what he could try. We assume the
         # directory may contain separate CD's and maybe some extra's. The
         # structure may look like X albums at same depth, and (one or more)
         # extra folders with a higher depth.
-        extra_media_folders = [media_folder[:min(path_depths)] for media_folder in media_folders if len(media_folder) > min(path_depths)]
-        extra_media_folders = list(set([os.path.join(*media_folder) for media_folder in extra_media_folders]))
+        extra_media_folders = [media_folder[:min(path_depths)] for media_folder in media_folders if
+                               len(media_folder) > min(path_depths)]
+        extra_media_folders = list(
+            set([os.path.join(*media_folder) for media_folder in extra_media_folders]))
 
-        logger.info("Please look at the following folder(s), since they cause the depth difference: %s", extra_media_folders)
+        logger.info(
+            "Please look at the following folder(s), since they cause the depth difference: %s",
+            extra_media_folders)
         return
 
     # Convert back to paths and remove duplicates, which may be there after
@@ -368,7 +372,7 @@ def path_filter_patterns(paths, patterns, root=None):
     for path in paths[:]:
         if path_match_patterns(path, patterns):
             logger.debug("Path ignored by pattern: %s",
-                os.path.join(root or "", path))
+                         os.path.join(root or "", path))
 
             ignored += 1
             paths.remove(path)
@@ -378,11 +382,11 @@ def path_filter_patterns(paths, patterns, root=None):
 
 
 def extract_data(s):
-
     s = s.replace('_', ' ')
 
-    #headphones default format
-    pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s[\[\(](?P<year>.*?)[\]\)]', re.VERBOSE)
+    # headphones default format
+    pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s[\[\(](?P<year>.*?)[\]\)]',
+                         re.VERBOSE)
     match = pattern.match(s)
 
     if match:
@@ -391,7 +395,7 @@ def extract_data(s):
         year = match.group("year")
         return (name, album, year)
 
-    #Gonna take a guess on this one - might be enough to search on mb
+    # Gonna take a guess on this one - might be enough to search on mb
     pat = re.compile(r"(?P<name>.*?)\s*-\s*(?P<album>[^\[(-]*)")
 
     match = pat.match(s)
@@ -468,7 +472,8 @@ def extract_metadata(f):
                 old_album = new_albums[index]
                 new_albums[index] = RE_CD_ALBUM.sub("", album).strip()
 
-                logger.debug("Stripped albumd number identifier: %s -> %s", old_album, new_albums[index])
+                logger.debug("Stripped albumd number identifier: %s -> %s", old_album,
+                             new_albums[index])
 
         # Remove duplicates
         new_albums = list(set(new_albums))
@@ -498,7 +503,8 @@ def extract_metadata(f):
             return (artist, albums[0], years[0])
 
     # Not sure what to do here.
-    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists), len(albums), len(years))
+    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists),
+                len(albums), len(years))
     logger.debug("Artists: %s, Albums: %s, Years: %s", artists, albums, years)
 
     return (None, None, None)
@@ -524,8 +530,10 @@ def preserve_torrent_directory(albumpath):
     Copy torrent directory to headphones-modified to keep files for seeding.
     """
     from headphones import logger
-    new_folder = os.path.join(albumpath, 'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
-    logger.info("Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
+    new_folder = os.path.join(albumpath,
+                              'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
+    logger.info(
+        "Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
     try:
         shutil.copytree(albumpath, new_folder)
         return new_folder
@@ -578,7 +586,9 @@ def cue_split(albumpath):
 
 def extract_logline(s):
     # Default log format
-    pattern = re.compile(r'(?P<timestamp>.*?)\s\-\s(?P<level>.*?)\s*\:\:\s(?P<thread>.*?)\s\:\s(?P<message>.*)', re.VERBOSE)
+    pattern = re.compile(
+        r'(?P<timestamp>.*?)\s\-\s(?P<level>.*?)\s*\:\:\s(?P<thread>.*?)\s\:\s(?P<message>.*)',
+        re.VERBOSE)
     match = pattern.match(s)
     if match:
         timestamp = match.group("timestamp")
@@ -593,7 +603,7 @@ def extract_logline(s):
 def extract_song_data(s):
     from headphones import logger
 
-    #headphones default format
+    # headphones default format
     pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s\[(?P<year>.*?)\]', re.VERBOSE)
     match = pattern.match(s)
 
@@ -605,7 +615,7 @@ def extract_song_data(s):
     else:
         logger.info("Couldn't parse %s into a valid default format", s)
 
-    #newzbin default format
+    # newzbin default format
     pattern = re.compile(r'(?P<name>.*?)\s\-\s(?P<album>.*?)\s\((?P<year>\d+?\))', re.VERBOSE)
     match = pattern.match(s)
     if match:
@@ -619,7 +629,6 @@ def extract_song_data(s):
 
 
 def smartMove(src, dest, delete=True):
-
     from headphones import logger
 
     source_dir = os.path.dirname(src)
@@ -640,7 +649,8 @@ def smartMove(src, dest, delete=True):
                     os.rename(src, os.path.join(source_dir, newfile))
                     filename = newfile
                 except Exception as e:
-                    logger.warn('Error renaming %s: %s', src.decode(headphones.SYS_ENCODING, 'replace'), e)
+                    logger.warn('Error renaming %s: %s',
+                                src.decode(headphones.SYS_ENCODING, 'replace'), e)
                 break
 
     try:
@@ -650,7 +660,9 @@ def smartMove(src, dest, delete=True):
             shutil.copy(os.path.join(source_dir, filename), os.path.join(dest, filename))
             return True
     except Exception as e:
-        logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'), e)
+        logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'),
+                    e)
+
 
 def walk_directory(basedir, followlinks=True):
     """
@@ -672,8 +684,8 @@ def walk_directory(basedir, followlinks=True):
                 real_path = os.path.abspath(os.readlink(path))
 
                 if real_path in traversed:
-                    logger.debug("Skipping '%s' since it is a symlink to "\
-                        "'%s', which is already visited.", path, real_path)
+                    logger.debug("Skipping '%s' since it is a symlink to " \
+                                 "'%s', which is already visited.", path, real_path)
                 else:
                     traversed.append(real_path)
 
@@ -689,8 +701,9 @@ def walk_directory(basedir, followlinks=True):
         for result in _inner(*args):
             yield result
 
+
 #########################
-#Sab renaming functions #
+# Sab renaming functions #
 #########################
 
 # TODO: Grab config values from sab to know when these options are checked. For now we'll just iterate through all combinations
@@ -739,18 +752,20 @@ def sab_sanitize_foldername(name):
     if not name:
         name = 'unknown'
 
-    #maxlen = cfg.folder_max_length()
-    #if len(name) > maxlen:
+    # maxlen = cfg.folder_max_length()
+    # if len(name) > maxlen:
     #    name = name[:maxlen]
 
     return name
 
+
 def split_string(mystring, splitvar=','):
     mylist = []
     for each_word in mystring.split(splitvar):
         mylist.append(each_word.strip())
     return mylist
 
+
 def create_https_certificates(ssl_cert, ssl_key):
     """
     Create a pair of self-signed HTTPS certificares and store in them in
@@ -768,11 +783,13 @@ def create_https_certificates(ssl_cert, ssl_key):
     # Create the CA Certificate
     cakey = createKeyPair(TYPE_RSA, 2048)
     careq = createCertRequest(cakey, CN="Certificate Authority")
-    cacert = createCertificate(careq, (careq, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years
+    cacert = createCertificate(careq, (careq, cakey), serial,
+                               (0, 60 * 60 * 24 * 365 * 10))  # ten years
 
     pkey = createKeyPair(TYPE_RSA, 2048)
     req = createCertRequest(pkey, CN="Headphones")
-    cert = createCertificate(req, (cacert, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years
+    cert = createCertificate(req, (cacert, cakey), serial,
+                             (0, 60 * 60 * 24 * 365 * 10))  # ten years
 
     # Save the key and certificate to disk
     try:

headphones/importer.py

@@ -13,39 +13,39 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import logger, helpers, db, mb, lastfm, metacritic
-
-from beets.mediafile import MediaFile
-
 import time
+
+from headphones import logger, helpers, db, mb, lastfm, metacritic
+from beets.mediafile import MediaFile
 import headphones
 
 blacklisted_special_artist_names = ['[anonymous]', '[data]', '[no artist]',
-    '[traditional]', '[unknown]', 'Various Artists']
+                                    '[traditional]', '[unknown]', 'Various Artists']
 blacklisted_special_artists = ['f731ccc4-e22a-43af-a747-64213329e088',
-    '33cf029c-63b0-41a0-9855-be2a3665fb3b',
-    '314e1c25-dde7-4e4d-b2f4-0a7b9f7c56dc',
-    'eec63d3c-3b81-4ad4-b1e4-7c147d4d2b61',
-    '9be7f096-97ec-4615-8957-8d40b5dcbc41',
-    '125ec42a-7229-4250-afc5-e057484327fe',
-    '89ad4ac3-39f7-470e-963a-56509c546377']
+                               '33cf029c-63b0-41a0-9855-be2a3665fb3b',
+                               '314e1c25-dde7-4e4d-b2f4-0a7b9f7c56dc',
+                               'eec63d3c-3b81-4ad4-b1e4-7c147d4d2b61',
+                               '9be7f096-97ec-4615-8957-8d40b5dcbc41',
+                               '125ec42a-7229-4250-afc5-e057484327fe',
+                               '89ad4ac3-39f7-470e-963a-56509c546377']
 
 
 def is_exists(artistid):
     myDB = db.DBConnection()
 
     # See if the artist is already in the database
-    artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid])
+    artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?',
+                             [artistid])
 
     if any(artistid in x for x in artistlist):
-        logger.info(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information")
+        logger.info(artistlist[0][
+                        1] + u" is already in the database. Updating 'have tracks', but not artist information")
         return True
     else:
         return False
 
 
 def artistlist_to_mbids(artistlist, forced=False):
-
     for artist in artistlist:
 
         if not artist and artist != ' ':
@@ -77,9 +77,12 @@ def artistlist_to_mbids(artistlist, forced=False):
         myDB = db.DBConnection()
 
         if not forced:
-            bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?', [artistid]).fetchone()
+            bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?',
+                                    [artistid]).fetchone()
             if bl_artist or artistid in blacklisted_special_artists:
-                logger.info("Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must do it manually (Artist ID: %s)" % (artist, artistid))
+                logger.info(
+                    "Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must do it manually (Artist ID: %s)" % (
+                    artist, artistid))
                 continue
 
         # Add to database if it doesn't exist
@@ -88,7 +91,9 @@ def artistlist_to_mbids(artistlist, forced=False):
 
         # Just update the tracks if it does
         else:
-            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
+            havetracks = len(
+                myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
+                myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
             myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
 
         # Delete it from the New Artists if the request came from there
@@ -112,7 +117,6 @@ def addArtistIDListToDB(artistidlist):
 
 
 def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
-
     # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
     from headphones import cache
 
@@ -142,7 +146,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                         "Status": "Loading",
                         "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS,
                         "Extras": headphones.CONFIG.EXTRAS}
-        if type=="series":
+        if type == "series":
             newValueDict['Type'] = "series"
     else:
         newValueDict = {"Status": "Loading"}
@@ -151,7 +155,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
     myDB.upsert("artists", newValueDict, controlValueDict)
 
-    if type=="series":
+    if type == "series":
         artist = mb.getSeries(artistid)
     else:
         artist = mb.getArtist(artistid, extrasonly)
@@ -159,7 +163,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     if artist and artist.get('artist_name') in blacklisted_special_artist_names:
         logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
         myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
-        #in case it's already in the db
+        # in case it's already in the db
         myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
         myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
         return
@@ -168,7 +172,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         logger.warn("Error fetching artist info. ID: " + artistid)
         if dbartist is None:
             newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid),
-                    "Status": "Active"}
+                            "Status": "Active"}
         else:
             newValueDict = {"Status": "Active"}
         myDB.upsert("artists", newValueDict, controlValueDict)
@@ -191,7 +195,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     # See if we need to grab extras. Artist specific extras take precedence
     # over global option. Global options are set when adding a new artist
     try:
-        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
+        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?',
+                                [artistid]).fetchone()
         includeExtras = db_artist['IncludeExtras']
     except IndexError:
         includeExtras = False
@@ -206,9 +211,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         for groups in artist['releasegroups']:
             group_list.append(groups['id'])
         if not extrasonly:
-            remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
+            remove_missing_groups_from_albums = myDB.select(
+                "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
         else:
-            remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid])
+            remove_missing_groups_from_albums = myDB.select(
+                'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"',
+                [artistid])
         for items in remove_missing_groups_from_albums:
             if items['AlbumID'] not in group_list:
                 # Remove all from albums/tracks that aren't in release groups
@@ -217,12 +225,16 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
                 myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
                 myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']])
-                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID']))
+                logger.info(
+                    "[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (
+                    artist['artist_name'], items['AlbumID']))
                 if not extrasonly:
                     force_repackage = 1
     else:
         if not extrasonly:
-            logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name'])
+            logger.info(
+                "[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" %
+                artist['artist_name'])
 
     # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
     album_searches = []
@@ -232,7 +244,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         today = helpers.today()
         rgid = rg['id']
         skip_log = 0
-        #Make a user configurable variable to skip update of albums with release dates older than this date (in days)
+        # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
         pause_delta = headphones.CONFIG.MB_IGNORE_AGE
 
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
@@ -247,12 +259,14 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 new_release_group = True
 
             if new_release_group:
-                logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
+                logger.info("[%s] Now adding: %s (New Release Group)" % (
+                artist['artist_name'], rg['title']))
                 new_releases = mb.get_new_releases(rgid, includeExtras)
 
             else:
                 if check_release_date is None or check_release_date == u"None":
-                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                    logger.info("[%s] Now updating: %s (No Release Date)" % (
+                    artist['artist_name'], rg['title']))
                     new_releases = mb.get_new_releases(rgid, includeExtras, True)
                 else:
                     if len(check_release_date) == 10:
@@ -264,20 +278,24 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     else:
                         release_date = today
                     if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
-                        logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta)
+                        logger.info("[%s] Now updating: %s (Release Date <%s Days)",
+                                    artist['artist_name'], rg['title'], pause_delta)
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
-                        logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta)
+                        logger.info("[%s] Skipping: %s (Release Date >%s Days)",
+                                    artist['artist_name'], rg['title'], pause_delta)
                         skip_log = 1
                         new_releases = 0
 
             if force_repackage == 1:
                 new_releases = -1
-                logger.info('[%s] Forcing repackage of %s (Release Group Removed)', artist['artist_name'], al_title)
+                logger.info('[%s] Forcing repackage of %s (Release Group Removed)',
+                            artist['artist_name'], al_title)
             else:
                 new_releases = new_releases
         else:
-            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title'])
+            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'],
+                        rg['title'])
             new_releases = mb.get_new_releases(rgid, includeExtras, forcefull)
 
         if new_releases != 0:
@@ -291,23 +309,26 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
             # This will be used later to build a hybrid release
             fullreleaselist = []
             # Search for releases within a release group
-            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
+            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
+                                               [rg['id']])
 
             # Build the dictionary for the fullreleaselist
             for items in find_hybrid_releases:
-                if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
+                if items['ReleaseID'] != rg[
+                    'id']:  # don't include hybrid information, since that's what we're replacing
                     hybrid_release_id = items['ReleaseID']
                     newValueDict = {"ArtistID": items['ArtistID'],
-                        "ArtistName": items['ArtistName'],
-                        "AlbumTitle": items['AlbumTitle'],
-                        "AlbumID": items['AlbumID'],
-                        "AlbumASIN": items['AlbumASIN'],
-                        "ReleaseDate": items['ReleaseDate'],
-                        "Type": items['Type'],
-                        "ReleaseCountry": items['ReleaseCountry'],
-                        "ReleaseFormat": items['ReleaseFormat']
-                    }
-                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
+                                    "ArtistName": items['ArtistName'],
+                                    "AlbumTitle": items['AlbumTitle'],
+                                    "AlbumID": items['AlbumID'],
+                                    "AlbumASIN": items['AlbumASIN'],
+                                    "ReleaseDate": items['ReleaseDate'],
+                                    "Type": items['Type'],
+                                    "ReleaseCountry": items['ReleaseCountry'],
+                                    "ReleaseFormat": items['ReleaseFormat']
+                                    }
+                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?",
+                                                     [hybrid_release_id])
                     totalTracks = 1
                     hybrid_track_array = []
                     for hybrid_tracks in find_hybrid_tracks:
@@ -315,9 +336,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                             'number': hybrid_tracks['TrackNumber'],
                             'title': hybrid_tracks['TrackTitle'],
                             'id': hybrid_tracks['TrackID'],
-                            #'url':           hybrid_tracks['TrackURL'],
+                            # 'url':           hybrid_tracks['TrackURL'],
                             'duration': hybrid_tracks['TrackDuration']
-                            })
+                        })
                         totalTracks += 1
                     newValueDict['ReleaseID'] = hybrid_release_id
                     newValueDict['Tracks'] = hybrid_track_array
@@ -327,10 +348,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
             # This may end up being called with an empty fullreleaselist
             try:
                 hybridrelease = getHybridRelease(fullreleaselist)
-                logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
+                logger.info('[%s] Packaging %s releases into hybrid title' % (
+                artist['artist_name'], rg['title']))
             except Exception as e:
                 errors = True
-                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e))
+                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (
+                artist['artist_name'], rg['title'], e))
                 continue
 
             # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
@@ -345,13 +368,14 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                             "AlbumASIN": hybridrelease['AlbumASIN'],
                             "ReleaseDate": hybridrelease['ReleaseDate'],
                             "Type": rg['type']
-                        }
+                            }
 
             myDB.upsert("allalbums", newValueDict, controlValueDict)
 
             for track in hybridrelease['Tracks']:
 
-                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
+                cleanname = helpers.cleanName(
+                    artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
 
                 controlValueDict = {"TrackID": track['id'],
                                     "ReleaseID": rg['id']}
@@ -365,25 +389,29 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                                 "TrackDuration": track['duration'],
                                 "TrackNumber": track['number'],
                                 "CleanName": cleanname
-                            }
+                                }
 
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
+                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?',
+                                    [cleanname]).fetchone()
 
                 if not match:
-                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
-                #if not match:
-                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
+                    match = myDB.action(
+                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                        [artist['artist_name'], rg['title'], track['title']]).fetchone()
+                    # if not match:
+                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                 if match:
                     newValueDict['Location'] = match['Location']
                     newValueDict['BitRate'] = match['BitRate']
                     newValueDict['Format'] = match['Format']
-                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
-                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))
+                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
+                                (rg['id'], match['Location']))
 
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
 
             # Delete matched tracks from the have table
-            #myDB.action('DELETE from have WHERE Matched="True"')
+            # myDB.action('DELETE from have WHERE Matched="True"')
 
             # If there's no release in the main albums tables, add the default (hybrid)
             # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
@@ -408,7 +436,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                             "Type": album['Type'],
                             "ReleaseCountry": album['ReleaseCountry'],
                             "ReleaseFormat": album['ReleaseFormat']
-                        }
+                            }
 
             if rg_exists:
                 newValueDict['DateAdded'] = rg_exists['DateAdded']
@@ -425,14 +453,17 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     newValueDict['Status'] = "Wanted"
                 # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
                 # The first test just makes sure we have year-month-day
-                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
+                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
+                        today) - helpers.get_age(
+                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
                     newValueDict['Status'] = "Wanted"
                 else:
                     newValueDict['Status'] = "Skipped"
 
             myDB.upsert("albums", newValueDict, controlValueDict)
 
-            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
+            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?',
+                                 [releaseid]).fetchall()
 
             # This is used to see how many tracks you have from an album - to
             # mark it as downloaded. Default is 80%, can be set in config as
@@ -441,7 +472,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
             if total_track_count == 0:
                 logger.warning("Total track count is zero for Release ID " +
-                    "'%s', skipping.", releaseid)
+                               "'%s', skipping.", releaseid)
                 continue
 
             for track in tracks:
@@ -449,35 +480,43 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                                     "AlbumID": rg['id']}
 
                 newValueDict = {"ArtistID": track['ArtistID'],
-                            "ArtistName": track['ArtistName'],
-                            "AlbumTitle": track['AlbumTitle'],
-                            "AlbumASIN": track['AlbumASIN'],
-                            "ReleaseID": track['ReleaseID'],
-                            "TrackTitle": track['TrackTitle'],
-                            "TrackDuration": track['TrackDuration'],
-                            "TrackNumber": track['TrackNumber'],
-                            "CleanName": track['CleanName'],
-                            "Location": track['Location'],
-                            "Format": track['Format'],
-                            "BitRate": track['BitRate']
-                            }
+                                "ArtistName": track['ArtistName'],
+                                "AlbumTitle": track['AlbumTitle'],
+                                "AlbumASIN": track['AlbumASIN'],
+                                "ReleaseID": track['ReleaseID'],
+                                "TrackTitle": track['TrackTitle'],
+                                "TrackDuration": track['TrackDuration'],
+                                "TrackNumber": track['TrackNumber'],
+                                "CleanName": track['CleanName'],
+                                "Location": track['Location'],
+                                "Format": track['Format'],
+                                "BitRate": track['BitRate']
+                                }
 
                 myDB.upsert("tracks", newValueDict, controlValueDict)
 
             # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
-            have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
+            have_track_count = len(
+                myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL',
+                            [rg['id']]))
             marked_as_downloaded = False
 
             if rg_exists:
-                if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
-                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+                if rg_exists['Status'] == 'Skipped' and (
+                    (have_track_count / float(total_track_count)) >= (
+                    headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)):
+                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
+                                ['Downloaded', rg['id']])
                     marked_as_downloaded = True
             else:
-                if (have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0):
-                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+                if (have_track_count / float(total_track_count)) >= (
+                    headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0):
+                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?',
+                                ['Downloaded', rg['id']])
                     marked_as_downloaded = True
 
-            logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+            logger.info(
+                u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
             cache.getThumb(AlbumID=rg['id'])
 
             # Start a search for the album if it's new, hasn't been marked as
@@ -487,7 +526,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                 album_searches.append(rg['id'])
         else:
             if skip_log == 0:
-                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
+                logger.info(u"[%s] No new releases, so no changes made to %s" % (
+                artist['artist_name'], rg['title']))
 
     time.sleep(3)
     finalize_update(artistid, artist['artist_name'], errors)
@@ -499,7 +539,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
     metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
 
     if errors:
-        logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name']))
+        logger.info(
+            "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (
+            artist['artist_name'], artist['artist_name']))
     else:
         myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
         logger.info(u"Updating complete for: %s" % artist['artist_name'])
@@ -518,10 +560,18 @@ def finalize_update(artistid, artistname, errors=False):
 
     myDB = db.DBConnection()
 
-    latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone()
-    totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [artistid]))
-    #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
-    havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artistname]))
+    latestalbum = myDB.action(
+        'SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC',
+        [artistid]).fetchone()
+    totaltracks = len(myDB.select(
+        'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")',
+        [artistid]))
+    # havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
+    havetracks = len(
+        myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
+                    [artistid])) + len(
+        myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                    [artistname]))
 
     controlValueDict = {"ArtistID": artistid}
 
@@ -544,7 +594,6 @@ def finalize_update(artistid, artistname, errors=False):
 
 
 def addReleaseById(rid, rgid=None):
-
     myDB = db.DBConnection()
 
     # Create minimum info upfront if added from searchresults
@@ -563,14 +612,18 @@ def addReleaseById(rid, rgid=None):
     rgid = None
     artistid = None
     release_dict = None
-    results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid])
+    results = myDB.select(
+        "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1",
+        [rid])
     for result in results:
         rgid = result['ReleaseGroupID']
         artistid = result['ArtistID']
-        logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid)
+        logger.debug(
+            "Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid)
     if not rgid:
-        #didn't find it in the cache, get the information from MB
-        logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID")
+        # didn't find it in the cache, get the information from MB
+        logger.debug(
+            "Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID")
         try:
             release_dict = mb.getRelease(rid)
         except Exception as e:
@@ -587,10 +640,10 @@ def addReleaseById(rid, rgid=None):
         rgid = release_dict['rgid']
         artistid = release_dict['artist_id']
 
-    #we don't want to make more calls to MB here unless we have to, could be happening quite a lot
+    # we don't want to make more calls to MB here unless we have to, could be happening quite a lot
     rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid])
 
-    #make sure the artist exists since I don't know what happens later if it doesn't
+    # make sure the artist exists since I don't know what happens later if it doesn't
     artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid])
 
     if not artist_exists and release_dict:
@@ -599,7 +652,8 @@ def addReleaseById(rid, rgid=None):
         else:
             sortname = release_dict['artist_name']
 
-        logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
+        logger.info(
+            u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused")
         controlValueDict = {"ArtistID": release_dict['artist_id']}
         newValueDict = {"ArtistName": release_dict['artist_name'],
                         "ArtistSortName": sortname,
@@ -624,13 +678,14 @@ def addReleaseById(rid, rgid=None):
         myDB.upsert("artists", newValueDict, controlValueDict)
 
     elif not artist_exists and not release_dict:
-        logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.")
+        logger.error(
+            "Artist does not exist in the database and did not get a valid response from MB. Skipping release.")
         if status == 'Loading':
             myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
         return
 
-    if not rg_exists and release_dict or status == 'Loading' and release_dict:  #it should never be the case that we have an rg and not the artist
-                                                                                #but if it is this will fail
+    if not rg_exists and release_dict or status == 'Loading' and release_dict:  # it should never be the case that we have an rg and not the artist
+        # but if it is this will fail
         logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid)
         controlValueDict = {"AlbumID": rgid}
         if status != 'Loading':
@@ -639,7 +694,8 @@ def addReleaseById(rid, rgid=None):
         newValueDict = {"ArtistID": release_dict['artist_id'],
                         "ReleaseID": rgid,
                         "ArtistName": release_dict['artist_name'],
-                        "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'],
+                        "AlbumTitle": release_dict['title'] if 'title' in release_dict else
+                        release_dict['rg_title'],
                         "AlbumASIN": release_dict['asin'],
                         "ReleaseDate": release_dict['date'],
                         "DateAdded": helpers.today(),
@@ -650,41 +706,48 @@ def addReleaseById(rid, rgid=None):
 
         myDB.upsert("albums", newValueDict, controlValueDict)
 
-        #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
+        # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB
         myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']])
 
         for track in release_dict['tracks']:
-            cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])
+            cleanname = helpers.cleanName(
+                release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title'])
 
             controlValueDict = {"TrackID": track['id'],
                                 "AlbumID": rgid}
             newValueDict = {"ArtistID": release_dict['artist_id'],
-                        "ArtistName": release_dict['artist_name'],
-                        "AlbumTitle": release_dict['rg_title'],
-                        "AlbumASIN": release_dict['asin'],
-                        "TrackTitle": track['title'],
-                        "TrackDuration": track['duration'],
-                        "TrackNumber": track['number'],
-                        "CleanName": cleanname
-                        }
+                            "ArtistName": release_dict['artist_name'],
+                            "AlbumTitle": release_dict['rg_title'],
+                            "AlbumASIN": release_dict['asin'],
+                            "TrackTitle": track['title'],
+                            "TrackDuration": track['duration'],
+                            "TrackNumber": track['number'],
+                            "CleanName": cleanname
+                            }
 
-            match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone()
+            match = myDB.action(
+                'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?',
+                [cleanname]).fetchone()
 
             if not match:
-                match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone()
+                match = myDB.action(
+                    'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                    [release_dict['artist_name'], release_dict['rg_title'],
+                     track['title']]).fetchone()
 
-            #if not match:
-                #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
+                # if not match:
+                # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
 
             if match:
                 newValueDict['Location'] = match['Location']
                 newValueDict['BitRate'] = match['BitRate']
                 newValueDict['Format'] = match['Format']
-                #myDB.action('DELETE from have WHERE Location=?', [match['Location']])
+                # myDB.action('DELETE from have WHERE Location=?', [match['Location']])
 
                 # If the album has been scanned before adding the release it will be unmatched, update to matched
                 if match['Matched'] == 'Failed':
-                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release_dict['rgid'], match['Location']))
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
+                                (release_dict['rgid'], match['Location']))
 
             myDB.upsert("tracks", newValueDict, controlValueDict)
 
@@ -703,7 +766,8 @@ def addReleaseById(rid, rgid=None):
             searcher.searchforalbum(rgid, False)
 
     elif not rg_exists and not release_dict:
-        logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.")
+        logger.error(
+            "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.")
         if status == 'Loading':
             myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid])
         return
@@ -811,11 +875,13 @@ def getHybridRelease(fullreleaselist):
 
     sortable_release_list.sort(key=lambda x: getSortableReleaseDate(x['releasedate']))
 
-    average_tracks = sum(x['trackscount'] for x in sortable_release_list) / float(len(sortable_release_list))
+    average_tracks = sum(x['trackscount'] for x in sortable_release_list) / float(
+        len(sortable_release_list))
     for item in sortable_release_list:
         item['trackscount_delta'] = abs(average_tracks - item['trackscount'])
 
-    a = helpers.multikeysort(sortable_release_list, ['-hasasin', 'country', 'format', 'trackscount_delta'])
+    a = helpers.multikeysort(sortable_release_list,
+                             ['-hasasin', 'country', 'format', 'trackscount_delta'])
 
     release_dict = {'ReleaseDate': sortable_release_list[0]['releasedate'],
                     'Tracks': a[0]['tracks'],

headphones/lastfm.py

@@ -14,15 +14,14 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import random
-import headphones
-import headphones.lock
-
-from headphones import db, logger, request
-
 from collections import defaultdict
 
-TIMEOUT = 60.0 # seconds
-REQUEST_LIMIT = 1.0 / 5 # seconds
+import headphones
+import headphones.lock
+from headphones import db, logger, request
+
+TIMEOUT = 60.0  # seconds
+REQUEST_LIMIT = 1.0 / 5  # seconds
 ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
 API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
@@ -79,7 +78,7 @@ def getSimilar():
                 try:
                     artist_mbid = artist["mbid"]
                     artist_name = artist["name"]
-                except TypeError:
+                except KeyError:
                     continue
 
                 if not any(artist_mbid in x for x in results):
@@ -116,7 +115,7 @@ def getArtists():
         return
 
     logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME)
-    data = request_lastfm("library.getartists", limit=10000, user=headphones.CONFIG.LASTFM_USERNAME)
+    data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME)
 
     if data and "artists" in data:
         artistlist = []

headphones/librarysync.py

@@ -14,17 +14,17 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import math
+
 import headphones
-
 from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
-
 from headphones import db, logger, helpers, importer, lastfm
 
+
 # You can scan a single directory and append it to the current library by
 # specifying append=True, ArtistID and ArtistName.
 def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
-    cron=False, artistScan=False):
-
+                cron=False, artistScan=False):
     if cron and not headphones.CONFIG.LIBRARYSCAN:
         return
 
@@ -40,7 +40,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         dir = dir.encode(headphones.SYS_ENCODING)
 
     if not os.path.isdir(dir):
-        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING, 'replace'))
+        logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(headphones.SYS_ENCODING,
+                                                                           'replace'))
         return
 
     myDB = db.DBConnection()
@@ -50,13 +51,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     if not append:
         # Clean up bad filepaths
-        tracks = myDB.select('SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
+        tracks = myDB.select(
+            'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
 
         for track in tracks:
-            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
+            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
             if not os.path.isfile(encoded_track_string):
-                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
-                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
+                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
+                            [None, None, None, track['Location']])
+                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
+                            [None, None, None, track['Location']])
 
         del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
 
@@ -67,7 +71,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                     # Make sure deleted files get accounted for when updating artist track counts
                     new_artists.append(track['ArtistName'])
                 myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
-                logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace'))
+                logger.info(
+                    'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
+                        headphones.SYS_ENCODING, 'replace'))
 
     bitrates = []
     song_list = []
@@ -89,9 +95,14 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 latest_subdirectory.append(subdirectory)
 
                 if file_count == 0 and r.replace(dir, '') != '':
-                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
-                elif latest_subdirectory[file_count] != latest_subdirectory[file_count - 1] and file_count != 0:
-                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+                    logger.info("[%s] Now scanning subdirectory %s" % (
+                        dir.decode(headphones.SYS_ENCODING, 'replace'),
+                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+                elif latest_subdirectory[file_count] != latest_subdirectory[
+                            file_count - 1] and file_count != 0:
+                    logger.info("[%s] Now scanning subdirectory %s" % (
+                        dir.decode(headphones.SYS_ENCODING, 'replace'),
+                        subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
 
                 song = os.path.join(r, files)
 
@@ -102,10 +113,13 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 try:
                     f = MediaFile(song)
                 except (FileTypeError, UnreadableFileError):
-                    logger.warning("Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path)
+                    logger.warning(
+                        "Cannot read media file '%s', skipping. It may be corrupted or not a media file.",
+                        unicode_song_path)
                     continue
                 except IOError:
-                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path)
+                    logger.warning("Cannnot read media file '%s', skipping. Does the file exists?",
+                                   unicode_song_path)
                     continue
 
                 # Grab the bitrates for the auto detect bit rate option
@@ -131,45 +145,52 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 controlValueDict = {'Location': unicode_song_path}
 
                 newValueDict = {'TrackID': f.mb_trackid,
-                              #'ReleaseID' : f.mb_albumid,
-                              'ArtistName': f_artist,
-                              'AlbumTitle': f.album,
-                              'TrackNumber': f.track,
-                              'TrackLength': f.length,
-                              'Genre': f.genre,
-                              'Date': f.date,
-                              'TrackTitle': f.title,
-                              'BitRate': f.bitrate,
-                              'Format': f.format,
-                              'CleanName': CleanName
-                              }
+                                # 'ReleaseID' : f.mb_albumid,
+                                'ArtistName': f_artist,
+                                'AlbumTitle': f.album,
+                                'TrackNumber': f.track,
+                                'TrackLength': f.length,
+                                'Genre': f.genre,
+                                'Date': f.date,
+                                'TrackTitle': f.title,
+                                'BitRate': f.bitrate,
+                                'Format': f.format,
+                                'CleanName': CleanName
+                                }
 
-                #song_list.append(song_dict)
-                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone()
-                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
+                # song_list.append(song_dict)
+                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?",
+                                               [unicode_song_path]).fetchone()
+                # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
                 if not check_exist_song:
-                    #This is a new track
+                    # This is a new track
                     if f_artist:
                         new_artists.append(f_artist)
                     myDB.upsert("have", newValueDict, controlValueDict)
                     new_song_count += 1
                 else:
-                    if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
-                        #Important track metadata has been modified, need to run matcher again
+                    if check_exist_song['ArtistName'] != f_artist or check_exist_song[
+                        'AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                        # Important track metadata has been modified, need to run matcher again
                         if f_artist and f_artist != check_exist_song['ArtistName']:
                             new_artists.append(f_artist)
-                        elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored":
+                        elif f_artist and f_artist == check_exist_song['ArtistName'] and \
+                                        check_exist_song['Matched'] != "Ignored":
                             new_artists.append(f_artist)
                         else:
                             continue
 
                         newValueDict['Matched'] = None
                         myDB.upsert("have", newValueDict, controlValueDict)
-                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
-                        myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
+                        myDB.action(
+                            'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
+                            [None, None, None, unicode_song_path])
+                        myDB.action(
+                            'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
+                            [None, None, None, unicode_song_path])
                         new_song_count += 1
                     else:
-                        #This track information hasn't changed
+                        # This track information hasn't changed
                         if f_artist and check_exist_song['Matched'] != "Ignored":
                             new_artists.append(f_artist)
 
@@ -177,9 +198,13 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     # Now we start track matching
     logger.info("%s new/modified songs found and added to the database" % new_song_count)
-    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
-    total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
-    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+                            [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"])
+    total_number_of_songs = \
+        myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+                    [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0]
+    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(
+        headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
 
     # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
     # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
@@ -190,6 +215,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
     # We'll use this to give a % completion, just because the track matching might take a while
     song_count = 0
     latest_artist = []
+    last_completion_percentage = 0
 
     for song in song_list:
 
@@ -200,27 +226,30 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
             logger.info("Now matching songs by %s" % song['ArtistName'])
 
         song_count += 1
-        completion_percentage = float(song_count) / total_number_of_songs * 100
+        completion_percentage = math.floor(float(song_count) / total_number_of_songs * 1000) / 10
 
-        if completion_percentage % 10 == 0:
+        if completion_percentage >= (last_completion_percentage + 10):
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
+            last_completion_percentage = completion_percentage
 
-        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
-        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
-        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
-        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
+        # THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
+        # EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
+        # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
+        # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
         if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
 
-            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            track = myDB.action(
+                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
             have_updated = False
             if track:
                 controlValueDict = {'ArtistName': track['ArtistName'],
-                                     'AlbumTitle': track['AlbumTitle'],
-                                     'TrackTitle': track['TrackTitle']}
+                                    'AlbumTitle': track['AlbumTitle'],
+                                    'TrackTitle': track['TrackTitle']}
                 newValueDict = {'Location': song['Location'],
-                                 'BitRate': song['BitRate'],
-                                 'Format': song['Format']}
+                                'BitRate': song['BitRate'],
+                                'Format': song['Format']}
                 myDB.upsert("tracks", newValueDict, controlValueDict)
 
                 controlValueDict2 = {'Location': song['Location']}
@@ -228,12 +257,13 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 myDB.upsert("have", newValueDict2, controlValueDict2)
                 have_updated = True
             else:
-                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
+                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
+                                    [song['CleanName']]).fetchone()
                 if track:
                     controlValueDict = {'CleanName': track['CleanName']}
                     newValueDict = {'Location': song['Location'],
-                                     'BitRate': song['BitRate'],
-                                     'Format': song['Format']}
+                                    'BitRate': song['BitRate'],
+                                    'Format': song['Format']}
                     myDB.upsert("tracks", newValueDict, controlValueDict)
 
                     controlValueDict2 = {'Location': song['Location']}
@@ -246,26 +276,30 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                     myDB.upsert("have", newValueDict2, controlValueDict2)
                     have_updated = True
 
-            alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            alltrack = myDB.action(
+                'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
             if alltrack:
                 controlValueDict = {'ArtistName': alltrack['ArtistName'],
-                                     'AlbumTitle': alltrack['AlbumTitle'],
-                                     'TrackTitle': alltrack['TrackTitle']}
+                                    'AlbumTitle': alltrack['AlbumTitle'],
+                                    'TrackTitle': alltrack['TrackTitle']}
                 newValueDict = {'Location': song['Location'],
-                                 'BitRate': song['BitRate'],
-                                 'Format': song['Format']}
+                                'BitRate': song['BitRate'],
+                                'Format': song['Format']}
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
 
                 controlValueDict2 = {'Location': song['Location']}
                 newValueDict2 = {'Matched': alltrack['AlbumID']}
                 myDB.upsert("have", newValueDict2, controlValueDict2)
             else:
-                alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
+                alltrack = myDB.action(
+                    'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
+                    [song['CleanName']]).fetchone()
                 if alltrack:
                     controlValueDict = {'CleanName': alltrack['CleanName']}
                     newValueDict = {'Location': song['Location'],
-                                     'BitRate': song['BitRate'],
-                                     'Format': song['Format']}
+                                    'BitRate': song['BitRate'],
+                                    'Format': song['Format']}
                     myDB.upsert("alltracks", newValueDict, controlValueDict)
 
                     controlValueDict2 = {'Location': song['Location']}
@@ -283,9 +317,10 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
             newValueDict2 = {'Matched': "Failed"}
             myDB.upsert("have", newValueDict2, controlValueDict2)
 
-        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+            #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
 
-    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
+    logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
+                                                                            'replace'))
 
     if not append or artistScan:
         logger.info('Updating scanned artist track counts')
@@ -294,29 +329,32 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         unique_artists = {}.fromkeys(new_artists).keys()
         current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
 
-        #There was a bug where artists with special characters (-,') would show up in new artists.
+        # There was a bug where artists with special characters (-,') would show up in new artists.
         artist_list = [
             x for x in unique_artists
             if helpers.cleanName(x).lower() not in [
                 helpers.cleanName(y[0]).lower()
                 for y in current_artists
+                ]
             ]
-        ]
         artists_checked = [
             x for x in unique_artists
             if helpers.cleanName(x).lower() in [
                 helpers.cleanName(y[0]).lower()
                 for y in current_artists
+                ]
             ]
-        ]
 
         # Update track counts
         for artist in artists_checked:
             # Have tracks are selected from tracks table and not all tracks because of duplicates
             # We update the track count upon an album switch to compliment this
             havetracks = (
-                len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist]))
-                + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))
+                len(myDB.select(
+                    'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
+                    [artist])) + len(myDB.select(
+                    'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+                    [artist]))
             )
             # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
             # (can fix by getting rid of second len statement)
@@ -330,7 +368,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 importer.artistlist_to_mbids(artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
-                #myDB.action('DELETE from newartists')
+                # myDB.action('DELETE from newartists')
                 for artist in artist_list:
                     myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
 
@@ -341,7 +379,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
 
-        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName]))
+        havetracks = len(
+            myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
+                        [ArtistID])) + len(myDB.select(
+            'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
+            [ArtistName]))
         myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
 
     if not append:
@@ -352,18 +394,21 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     logger.info('Library scan complete')
 
-#ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
+
+# ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
 
 
 def update_album_status(AlbumID=None):
     myDB = db.DBConnection()
     logger.info('Counting matched tracks to mark albums as skipped/downloaded')
     if AlbumID:
-        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+        album_status_updater = myDB.action(
+            'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
     else:
         album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
     for album in album_status_updater:
-        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']])
+        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?',
+                                    [album['AlbumID']])
         total_tracks = 0
         have_tracks = 0
         for track in track_counter:
@@ -383,7 +428,7 @@ def update_album_status(AlbumID=None):
         # I think we can only automatically change Skipped->Downloaded when updating
         # There was a bug report where this was causing infinite downloads if the album was
         # recent, but matched to less than 80%. It would go Downloaded->Skipped->Wanted->Downloaded->Skipped->Wanted->etc....
-        #else:
+        # else:
         #    if album['Status'] == "Skipped" or album['Status'] == "Downloaded":
         #        new_album_status = "Skipped"
         #    else:

headphones/lock.py

@@ -2,11 +2,12 @@
 Locking-related classes
 """
 
-import headphones.logger
 import time
 import threading
 import Queue
 
+import headphones.logger
+
 
 class TimedLock(object):
     """

headphones/logger.py

@@ -13,24 +13,24 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import helpers
-
-from logutils.queue import QueueHandler, QueueListener
 from logging import handlers
-
 import multiprocessing
 import contextlib
-import headphones
 import threading
 import traceback
 import logging
 import errno
 import sys
+
 import os
+from headphones import helpers
+from logutils.queue import QueueHandler, QueueListener
+import headphones
+
 
 # These settings are for file logging only
 FILENAME = "headphones.log"
-MAX_SIZE = 1000000 # 1 MB
+MAX_SIZE = 1000000  # 1 MB
 MAX_FILES = 5
 
 # Headphones logger
@@ -39,6 +39,7 @@ logger = logging.getLogger("headphones")
 # Global queue for multiprocessing logging
 queue = None
 
+
 class LogListHandler(logging.Handler):
     """
     Log handler for Web UI.
@@ -71,8 +72,8 @@ def listener():
             # http://stackoverflow.com/questions/2009278 for more information.
             if e.errno == errno.EACCES:
                 logger.warning("Multiprocess logging disabled, because "
-                    "current user cannot map shared memory. You won't see any" \
-                    "logging generated by the worker processed.")
+                               "current user cannot map shared memory. You won't see any" \
+                               "logging generated by the worker processed.")
 
     # Multiprocess logging may be disabled.
     if not queue:
@@ -149,8 +150,10 @@ def initLogger(console=False, log_dir=False, verbose=False):
     if log_dir:
         filename = os.path.join(log_dir, FILENAME)
 
-        file_formatter = logging.Formatter('%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
-        file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE, backupCount=MAX_FILES)
+        file_formatter = logging.Formatter(
+            '%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
+        file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE,
+                                                    backupCount=MAX_FILES)
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(file_formatter)
 
@@ -158,7 +161,8 @@ def initLogger(console=False, log_dir=False, verbose=False):
 
     # Setup console logger
     if console:
-        console_formatter = logging.Formatter('%(asctime)s - %(levelname)s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
+        console_formatter = logging.Formatter(
+            '%(asctime)s - %(levelname)s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S')
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(console_formatter)
         console_handler.setLevel(logging.DEBUG)
@@ -212,11 +216,13 @@ def initHooks(global_exceptions=True, thread_exceptions=True, pass_original=True
                     raise
                 except:
                     excepthook(*sys.exc_info())
+
             self.run = new_run
 
         # Monkey patch the run() by monkey patching the __init__ method
         threading.Thread.__init__ = new_init
 
+
 # Expose logger methods
 info = logger.info
 warn = logger.warn

headphones/lyrics.py

@@ -13,18 +13,17 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
 import htmlentitydefs
 
+import re
 from headphones import logger, request
 
 
 def getLyrics(artist, song):
-
     params = {"artist": artist.encode('utf-8'),
-                "song": song.encode('utf-8'),
-                "fmt": 'xml'
-                }
+              "song": song.encode('utf-8'),
+              "fmt": 'xml'
+              }
 
     url = 'http://lyrics.wikia.com/api.php'
     data = request.request_minidom(url, params=params)
@@ -46,10 +45,13 @@ def getLyrics(artist, song):
         logger.warn('Error fetching lyrics from: %s' % lyricsurl)
         return
 
-    m = re.compile('''<div class='lyricbox'><div class='rtMatcher'>.*?</div>(.*?)<!--''').search(lyricspage)
+    m = re.compile('''<div class='lyricbox'><div class='rtMatcher'>.*?</div>(.*?)<!--''').search(
+        lyricspage)
 
     if not m:
-        m = re.compile('''<div class='lyricbox'><span style="padding:1em"><a href="/Category:Instrumental" title="Instrumental">''').search(lyricspage)
+        m = re.compile(
+            '''<div class='lyricbox'><span style="padding:1em"><a href="/Category:Instrumental" title="Instrumental">''').search(
+            lyricspage)
         if m:
             return u'(Instrumental)'
         else:
@@ -67,12 +69,12 @@ def convert_html_entities(s):
     if len(matches) > 0:
         hits = set(matches)
         for hit in hits:
-                name = hit[2:-1]
-                try:
-                        entnum = int(name)
-                        s = s.replace(hit, unichr(entnum))
-                except ValueError:
-                        pass
+            name = hit[2:-1]
+            try:
+                entnum = int(name)
+                s = s.replace(hit, unichr(entnum))
+            except ValueError:
+                pass
 
     matches = re.findall("&\w+;", s)
     hits = set(matches)

headphones/mb.py

@@ -31,12 +31,12 @@ except ImportError:
 
 mb_lock = headphones.lock.TimedLock(0)
 
+
 # Quick fix to add mirror switching on the fly. Need to probably return the mbhost & mbport that's
 # being used, so we can send those values to the log
 
 
 def startmb():
-
     mbuser = None
     mbpass = None
 
@@ -51,7 +51,7 @@ def startmb():
         mbpass = headphones.CONFIG.CUSTOMPASS
         sleepytime = int(headphones.CONFIG.CUSTOMSLEEP)
     elif headphones.CONFIG.MIRROR == "headphones":
-        mbhost = "144.76.94.239"
+        mbhost = "codeshy.com"
         mbport = 8181
         mbuser = headphones.CONFIG.HPUSER
         mbpass = headphones.CONFIG.HPPASS
@@ -66,7 +66,7 @@ def startmb():
     if sleepytime == 0:
         musicbrainzngs.set_rate_limit(False)
     else:
-        #calling it with an it ends up blocking all requests after the first
+        # calling it with an it ends up blocking all requests after the first
         musicbrainzngs.set_rate_limit(limit_or_interval=float(sleepytime))
         mb_lock.minimum_delta = sleepytime
 
@@ -81,59 +81,72 @@ def startmb():
     if not headphones.CONFIG.CUSTOMAUTH and headphones.CONFIG.MIRROR == "custom":
         musicbrainzngs.disable_hpauth()
 
-    logger.debug('Using the following server values: MBHost: %s, MBPort: %i, Sleep Interval: %i', mbhost, mbport, sleepytime)
+    logger.debug('Using the following server values: MBHost: %s, MBPort: %i, Sleep Interval: %i',
+                 mbhost, mbport, sleepytime)
 
     return True
 
 
 def findArtist(name, limit=1):
-     artistlist = []
-     artistResults = None
+    artistlist = []
+    artistResults = None
 
-     chars = set('!?*-')
-     if any((c in chars) for c in name):
-         name = '"' + name + '"'
+    chars = set('!?*-')
+    if any((c in chars) for c in name):
+        name = '"' + name + '"'
 
-     criteria = {'artist': name.lower()}
+    criteria = {'artist': name.lower()}
 
-     with mb_lock:
-         try:
-             artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
-         except musicbrainzngs.WebServiceError as e:
-             logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
-             mb_lock.snooze(5)
+    with mb_lock:
+        try:
+            artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
+        except ValueError as e:
+            if "at least one query term is required" in e.message:
+                logger.error(
+                    "Tried to search without a term, or an empty one. Provided artist (probably emtpy): %s",
+                    name)
+                return False
+            else:
+                raise
+        except musicbrainzngs.WebServiceError as e:
+            logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
+            mb_lock.snooze(5)
+
+    if not artistResults:
+        return False
+    for result in artistResults:
+        if 'disambiguation' in result:
+            uniquename = unicode(result['sort-name'] + " (" + result['disambiguation'] + ")")
+        else:
+            uniquename = unicode(result['sort-name'])
+        if result['name'] != uniquename and limit == 1:
+            logger.info(
+                'Found an artist with a disambiguation: %s - doing an album based search' % name)
+            artistdict = findArtistbyAlbum(name)
+            if not artistdict:
+                logger.info(
+                    'Cannot determine the best match from an artist/album search. Using top match instead')
+                artistlist.append({
+                    # Just need the artist id if the limit is 1
+                    #    'name':             unicode(result['sort-name']),
+                    #    'uniquename':        uniquename,
+                    'id': unicode(result['id']),
+                    #    'url':                 unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed
+                    #    'score':            int(result['ext:score'])
+                })
+            else:
+                artistlist.append(artistdict)
+        else:
+            artistlist.append({
+                'name': unicode(result['sort-name']),
+                'uniquename': uniquename,
+                'id': unicode(result['id']),
+                'url': unicode("http://musicbrainz.org/artist/" + result['id']),
+                # probably needs to be changed
+                'score': int(result['ext:score'])
+            })
+    return artistlist
 
-     if not artistResults:
-         return False
-     for result in artistResults:
-         if 'disambiguation' in result:
-             uniquename = unicode(result['sort-name'] + " (" + result['disambiguation'] + ")")
-         else:
-             uniquename = unicode(result['sort-name'])
-         if result['name'] != uniquename and limit == 1:
-             logger.info('Found an artist with a disambiguation: %s - doing an album based search' % name)
-             artistdict = findArtistbyAlbum(name)
-             if not artistdict:
-                 logger.info('Cannot determine the best match from an artist/album search. Using top match instead')
-                 artistlist.append({
-                 # Just need the artist id if the limit is 1
-                 #    'name':             unicode(result['sort-name']),
-                 #    'uniquename':        uniquename,
-                     'id': unicode(result['id']),
-                 #    'url':                 unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed
-                 #    'score':            int(result['ext:score'])
-                     })
-             else:
-                 artistlist.append(artistdict)
-         else:
-             artistlist.append({
-                     'name': unicode(result['sort-name']),
-                     'uniquename': uniquename,
-                     'id': unicode(result['id']),
-                     'url': unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed
-                     'score': int(result['ext:score'])
-                     })
-     return artistlist
 
 def findRelease(name, limit=1, artist=None):
     releaselist = []
@@ -151,8 +164,9 @@ def findRelease(name, limit=1, artist=None):
 
     with mb_lock:
         try:
-            releaseResults = musicbrainzngs.search_releases(query=name, limit=limit, artist=artist)['release-list']
-        except musicbrainzngs.WebServiceError as e: #need to update exceptions
+            releaseResults = musicbrainzngs.search_releases(query=name, limit=limit, artist=artist)[
+                'release-list']
+        except musicbrainzngs.WebServiceError as e:  # need to update exceptions
             logger.warn('Attempt to query MusicBrainz for "%s" failed: %s' % (name, str(e)))
             mb_lock.snooze(5)
 
@@ -196,55 +210,61 @@ def findRelease(name, limit=1, artist=None):
                     rg_type = secondary_type
 
         releaselist.append({
-                    'uniquename': unicode(result['artist-credit'][0]['artist']['name']),
-                    'title': unicode(title),
-                    'id': unicode(result['artist-credit'][0]['artist']['id']),
-                    'albumid': unicode(result['id']),
-                    'url': unicode("http://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),#probably needs to be changed
-                    'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),#probably needs to be changed
-                    'score': int(result['ext:score']),
-                    'date': unicode(result['date']) if 'date' in result else '',
-                    'country': unicode(result['country']) if 'country' in result else '',
-                    'formats': unicode(formats),
-                    'tracks': unicode(tracks),
-                    'rgid': unicode(result['release-group']['id']),
-                    'rgtype': unicode(rg_type)
-                    })
+            'uniquename': unicode(result['artist-credit'][0]['artist']['name']),
+            'title': unicode(title),
+            'id': unicode(result['artist-credit'][0]['artist']['id']),
+            'albumid': unicode(result['id']),
+            'url': unicode(
+                "http://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),
+            # probably needs to be changed
+            'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),
+            # probably needs to be changed
+            'score': int(result['ext:score']),
+            'date': unicode(result['date']) if 'date' in result else '',
+            'country': unicode(result['country']) if 'country' in result else '',
+            'formats': unicode(formats),
+            'tracks': unicode(tracks),
+            'rgid': unicode(result['release-group']['id']),
+            'rgtype': unicode(rg_type)
+        })
     return releaselist
 
+
 def findSeries(name, limit=1):
-     serieslist = []
-     seriesResults = None
+    serieslist = []
+    seriesResults = None
 
-     chars = set('!?*-')
-     if any((c in chars) for c in name):
-         name = '"' + name + '"'
+    chars = set('!?*-')
+    if any((c in chars) for c in name):
+        name = '"' + name + '"'
 
-     criteria = {'series': name.lower()}
+    criteria = {'series': name.lower()}
 
-     with mb_lock:
-         try:
-             seriesResults = musicbrainzngs.search_series(limit=limit, **criteria)['series-list']
-         except musicbrainzngs.WebServiceError as e:
-             logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
-             mb_lock.snooze(5)
+    with mb_lock:
+        try:
+            seriesResults = musicbrainzngs.search_series(limit=limit, **criteria)['series-list']
+        except musicbrainzngs.WebServiceError as e:
+            logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
+            mb_lock.snooze(5)
+
+    if not seriesResults:
+        return False
+    for result in seriesResults:
+        if 'disambiguation' in result:
+            uniquename = unicode(result['name'] + " (" + result['disambiguation'] + ")")
+        else:
+            uniquename = unicode(result['name'])
+        serieslist.append({
+            'uniquename': uniquename,
+            'name': unicode(result['name']),
+            'type': unicode(result['type']),
+            'id': unicode(result['id']),
+            'url': unicode("http://musicbrainz.org/series/" + result['id']),
+            # probably needs to be changed
+            'score': int(result['ext:score'])
+        })
+    return serieslist
 
-     if not seriesResults:
-         return False
-     for result in seriesResults:
-         if 'disambiguation' in result:
-             uniquename = unicode(result['name'] + " (" + result['disambiguation'] + ")")
-         else:
-             uniquename = unicode(result['name'])
-         serieslist.append({
-                     'uniquename': uniquename,
-                     'name': unicode(result['name']),
-                     'type': unicode(result['type']),
-                     'id': unicode(result['id']),
-                     'url': unicode("http://musicbrainz.org/series/" + result['id']),#probably needs to be changed
-                     'score': int(result['ext:score'])
-                     })
-     return serieslist
 
 def getArtist(artistid, extrasonly=False):
     artist_dict = {}
@@ -265,7 +285,9 @@ def getArtist(artistid, extrasonly=False):
             newRgs = newRgs['release-group-list']
             artist['release-group-list'] += newRgs
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e)))
+        logger.warn(
+            'Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (
+            artistid, str(e)))
         mb_lock.snooze(5)
     except Exception as e:
         pass
@@ -279,7 +301,7 @@ def getArtist(artistid, extrasonly=False):
 
     if not extrasonly:
         for rg in artist['release-group-list']:
-            if "secondary-type-list" in rg.keys(): #only add releases without a secondary type
+            if "secondary-type-list" in rg.keys():  # only add releases without a secondary type
                 continue
             releasegroups.append({
                 'title': unicode(rg['title']),
@@ -293,7 +315,8 @@ def getArtist(artistid, extrasonly=False):
     myDB = db.DBConnection()
 
     try:
-        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
+        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?',
+                                [artistid]).fetchone()
         includeExtras = db_artist['IncludeExtras']
     except IndexError:
         includeExtras = False
@@ -329,7 +352,9 @@ def getArtist(artistid, extrasonly=False):
                     newRgs = newRgs['release-group-list']
                     mb_extras_list += newRgs
             except musicbrainzngs.WebServiceError as e:
-                logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e)))
+                logger.warn(
+                    'Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (
+                    artistid, str(e)))
                 mb_lock.snooze(5)
 
             for rg in mb_extras_list:
@@ -348,23 +373,28 @@ def getArtist(artistid, extrasonly=False):
     artist_dict['releasegroups'] = releasegroups
     return artist_dict
 
+
 def getSeries(seriesid):
     series_dict = {}
     series = None
     try:
         with mb_lock:
-            series = musicbrainzngs.get_series_by_id(seriesid,includes=['release-group-rels'])['series']
+            series = musicbrainzngs.get_series_by_id(seriesid, includes=['release-group-rels'])[
+                'series']
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to retrieve series information from MusicBrainz failed for seriesid: %s (%s)' % (seriesid, str(e)))
+        logger.warn(
+            'Attempt to retrieve series information from MusicBrainz failed for seriesid: %s (%s)' % (
+            seriesid, str(e)))
         mb_lock.snooze(5)
     except Exception as e:
         pass
 
     if not series:
         return False
-    
+
     if 'disambiguation' in series:
-        series_dict['artist_name'] = unicode(series['name'] + " (" + unicode(series['disambiguation']) + ")")
+        series_dict['artist_name'] = unicode(
+            series['name'] + " (" + unicode(series['disambiguation']) + ")")
     else:
         series_dict['artist_name'] = unicode(series['name'])
 
@@ -373,14 +403,15 @@ def getSeries(seriesid):
     for rg in series['release_group-relation-list']:
         releasegroup = rg['release-group']
         releasegroups.append({
-            'title':releasegroup['title'],
-            'date':releasegroup['first-release-date'],
-            'id':releasegroup['id'],
-            'type':rg['type']
-            })
+            'title': releasegroup['title'],
+            'date': releasegroup['first-release-date'],
+            'id': releasegroup['id'],
+            'type': rg['type']
+        })
     series_dict['releasegroups'] = releasegroups
     return series_dict
 
+
 def getReleaseGroup(rgid):
     """
     Returns a list of releases in a release group
@@ -392,7 +423,9 @@ def getReleaseGroup(rgid):
                 rgid, ["artists", "releases", "media", "discids", ])
             releaseGroup = releaseGroup['release-group']
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
+        logger.warn(
+            'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (
+            rgid, str(e)))
         mb_lock.snooze(5)
 
     if not releaseGroup:
@@ -411,11 +444,16 @@ def getRelease(releaseid, include_artist_info=True):
     try:
         with mb_lock:
             if include_artist_info:
-                results = musicbrainzngs.get_release_by_id(releaseid, ["artists", "release-groups", "media", "recordings"]).get('release')
+                results = musicbrainzngs.get_release_by_id(releaseid,
+                                                           ["artists", "release-groups", "media",
+                                                            "recordings"]).get('release')
             else:
-                results = musicbrainzngs.get_release_by_id(releaseid, ["media", "recordings"]).get('release')
+                results = musicbrainzngs.get_release_by_id(releaseid, ["media", "recordings"]).get(
+                    'release')
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to retrieve information from MusicBrainz for release "%s" failed (%s)' % (releaseid, str(e)))
+        logger.warn(
+            'Attempt to retrieve information from MusicBrainz for release "%s" failed (%s)' % (
+            releaseid, str(e)))
         mb_lock.snooze(5)
 
     if not results:
@@ -443,7 +481,8 @@ def getRelease(releaseid, include_artist_info=True):
             try:
                 release['rg_type'] = unicode(results['release-group']['type'])
 
-                if release['rg_type'] == 'Album' and 'secondary-type-list' in results['release-group']:
+                if release['rg_type'] == 'Album' and 'secondary-type-list' in results[
+                    'release-group']:
                     secondary_type = unicode(results['release-group']['secondary-type-list'][0])
                     if secondary_type != release['rg_type']:
                         release['rg_type'] = secondary_type
@@ -463,7 +502,6 @@ def getRelease(releaseid, include_artist_info=True):
 
 
 def get_new_releases(rgid, includeExtras=False, forcefull=False):
-
     myDB = db.DBConnection()
     results = []
 
@@ -479,30 +517,33 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                 newResults = musicbrainzngs.browse_releases(
                     release_group=rgid,
                     includes=['artist-credits', 'labels', 'recordings', 'release-groups', 'media'],
-                    release_status = release_status,
+                    release_status=release_status,
                     limit=limit,
                     offset=len(results))
             if 'release-list' not in newResults:
-                break #may want to raise an exception here instead ?
+                break  # may want to raise an exception here instead ?
             newResults = newResults['release-list']
             results += newResults
 
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
+        logger.warn(
+            'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (
+            rgid, str(e)))
         mb_lock.snooze(5)
         return False
 
     if not results or len(results) == 0:
         return False
 
-    #Clean all references to releases in dB that are no longer referenced in musicbrainz
+    # Clean all references to releases in dB that are no longer referenced in musicbrainz
     release_list = []
     force_repackage1 = 0
     if len(results) != 0:
         for release_mark in results:
             release_list.append(unicode(release_mark['id']))
             release_title = release_mark['title']
-        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
+        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?",
+                                              [rgid])
         if remove_missing_releases:
             for items in remove_missing_releases:
                 if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
@@ -511,10 +552,13 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                     myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
                     myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
                     myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
-                    logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
+                    logger.info(
+                        "Removing all references to release %s to reflect MusicBrainz" % items[
+                            'ReleaseID'])
                     force_repackage1 = 1
     else:
-        logger.info("There was either an error pulling data from MusicBrainz or there might not be any releases for this category")
+        logger.info(
+            "There was either an error pulling data from MusicBrainz or there might not be any releases for this category")
 
     num_new_releases = 0
 
@@ -522,9 +566,10 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
 
         release = {}
         rel_id_check = releasedata['id']
-        album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
+        album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?',
+                                    [rel_id_check]).fetchone()
         if not album_checker or forcefull:
-            #DELETE all references to this release since we're updating it anyway.
+            # DELETE all references to this release since we're updating it anyway.
             myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
             myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
             release['AlbumTitle'] = unicode(releasedata['title'])
@@ -533,7 +578,8 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
             release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
             release['ReleaseID'] = releasedata['id']
             if 'release-group' not in releasedata:
-                raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
+                raise Exception('No release group associated with release id ' + releasedata[
+                    'id'] + ' album id' + rgid)
             release['Type'] = unicode(releasedata['release-group']['type'])
 
             if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']:
@@ -541,7 +587,7 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                 if secondary_type != release['Type']:
                     release['Type'] = secondary_type
 
-            #making the assumption that the most important artist will be first in the list
+            # making the assumption that the most important artist will be first in the list
             if 'artist-credit' in releasedata:
                 release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
                 release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
@@ -549,8 +595,9 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                 logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
                 return False
 
-            release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
-            #assuming that the list will contain media and that the format will be consistent
+            release['ReleaseCountry'] = unicode(
+                releasedata['country']) if 'country' in releasedata else u'Unknown'
+            # assuming that the list will contain media and that the format will be consistent
             try:
                 additional_medium = ''
                 for position in releasedata['medium-list']:
@@ -562,16 +609,17 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                     disc_number = ''
                 else:
                     disc_number = str(medium_count) + 'x'
-                packaged_medium = disc_number + releasedata['medium-list'][0]['format'] + additional_medium
+                packaged_medium = disc_number + releasedata['medium-list'][0][
+                    'format'] + additional_medium
                 release['ReleaseFormat'] = unicode(packaged_medium)
             except:
                 release['ReleaseFormat'] = u'Unknown'
 
             release['Tracks'] = getTracksFromRelease(releasedata)
 
-        # What we're doing here now is first updating the allalbums & alltracks table to the most
-        # current info, then moving the appropriate release into the album table and its associated
-        # tracks into the tracks table
+            # What we're doing here now is first updating the allalbums & alltracks table to the most
+            # current info, then moving the appropriate release into the album table and its associated
+            # tracks into the tracks table
             controlValueDict = {"ReleaseID": release['ReleaseID']}
 
             newValueDict = {"ArtistID": release['ArtistID'],
@@ -583,13 +631,14 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                             "Type": release['Type'],
                             "ReleaseCountry": release['ReleaseCountry'],
                             "ReleaseFormat": release['ReleaseFormat']
-                        }
+                            }
 
             myDB.upsert("allalbums", newValueDict, controlValueDict)
 
             for track in release['Tracks']:
 
-                cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
+                cleanname = helpers.cleanName(
+                    release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
 
                 controlValueDict = {"TrackID": track['id'],
                                     "ReleaseID": release['ReleaseID']}
@@ -603,30 +652,37 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False):
                                 "TrackDuration": track['duration'],
                                 "TrackNumber": track['number'],
                                 "CleanName": cleanname
-                            }
+                                }
 
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
+                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?',
+                                    [cleanname]).fetchone()
 
                 if not match:
-                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
-                #if not match:
-                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
+                    match = myDB.action(
+                        'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
+                        [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
+                    # if not match:
+                    # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                 if match:
                     newValueDict['Location'] = match['Location']
                     newValueDict['BitRate'] = match['BitRate']
                     newValueDict['Format'] = match['Format']
-                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
-                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location']))
+                    # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?',
+                                (release['AlbumID'], match['Location']))
 
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
             num_new_releases = num_new_releases + 1
             if album_checker:
-                logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+                logger.info('[%s] Existing release %s (%s) updated' % (
+                release['ArtistName'], release['AlbumTitle'], rel_id_check))
             else:
-                logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+                logger.info('[%s] New release %s (%s) added' % (
+                release['ArtistName'], release['AlbumTitle'], rel_id_check))
         if force_repackage1 == 1:
             num_new_releases = -1
-            logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title))
+            logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (
+            release['ArtistName'], release_title))
         else:
             num_new_releases = num_new_releases
 
@@ -643,23 +699,25 @@ def getTracksFromRelease(release):
             except:
                 track_title = unicode(track['recording']['title'])
             tracks.append({
-                    'number': totalTracks,
-                    'title': track_title,
-                    'id': unicode(track['recording']['id']),
-                    'url': u"http://musicbrainz.org/track/" + track['recording']['id'],
-                    'duration': int(track['length']) if 'length' in track else 0
-                    })
+                'number': totalTracks,
+                'title': track_title,
+                'id': unicode(track['recording']['id']),
+                'url': u"http://musicbrainz.org/track/" + track['recording']['id'],
+                'duration': int(track['length']) if 'length' in track else 0
+            })
             totalTracks += 1
     return tracks
 
+
 # Used when there is a disambiguation
 
 
 def findArtistbyAlbum(name):
-
     myDB = db.DBConnection()
 
-    artist = myDB.action('SELECT AlbumTitle from have WHERE ArtistName=? AND AlbumTitle IS NOT NULL ORDER BY RANDOM()', [name]).fetchone()
+    artist = myDB.action(
+        'SELECT AlbumTitle from have WHERE ArtistName=? AND AlbumTitle IS NOT NULL ORDER BY RANDOM()',
+        [name]).fetchone()
 
     if not artist:
         return False
@@ -686,21 +744,20 @@ def findArtistbyAlbum(name):
     for releaseGroup in results:
         newArtist = releaseGroup['artist-credit'][0]['artist']
         # Only need the artist ID if we're doing an artist+album lookup
-        #if 'disambiguation' in newArtist:
+        # if 'disambiguation' in newArtist:
         #    uniquename = unicode(newArtist['sort-name'] + " (" + newArtist['disambiguation'] + ")")
-        #else:
+        # else:
         #    uniquename = unicode(newArtist['sort-name'])
-        #artist_dict['name'] = unicode(newArtist['sort-name'])
-        #artist_dict['uniquename'] = uniquename
+        # artist_dict['name'] = unicode(newArtist['sort-name'])
+        # artist_dict['uniquename'] = uniquename
         artist_dict['id'] = unicode(newArtist['id'])
-        #artist_dict['url'] = u'http://musicbrainz.org/artist/' + newArtist['id']
-        #artist_dict['score'] = int(releaseGroup['ext:score'])
+        # artist_dict['url'] = u'http://musicbrainz.org/artist/' + newArtist['id']
+        # artist_dict['score'] = int(releaseGroup['ext:score'])
 
     return artist_dict
 
 
 def findAlbumID(artist=None, album=None):
-
     results = None
     chars = set('!?*-')
 
@@ -717,9 +774,11 @@ def findAlbumID(artist=None, album=None):
                 album = '"' + album + '"'
             criteria = {'release': album.lower()}
         with mb_lock:
-            results = musicbrainzngs.search_release_groups(limit=1, **criteria).get('release-group-list')
+            results = musicbrainzngs.search_release_groups(limit=1, **criteria).get(
+                'release-group-list')
     except musicbrainzngs.WebServiceError as e:
-        logger.warn('Attempt to query MusicBrainz for %s - %s failed (%s)' % (artist, album, str(e)))
+        logger.warn(
+            'Attempt to query MusicBrainz for %s - %s failed (%s)' % (artist, album, str(e)))
         mb_lock.snooze(5)
 
     if not results:

headphones/metacritic.py

@@ -13,14 +13,12 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
 import json
-import headphones
 
 from headphones import db, helpers, logger, request
-from headphones.common import USER_AGENT
 
-def update(artistid, artist_name,release_groups):
+
+def update(artistid, artist_name, release_groups):
     """ Pretty simple and crude function to find the artist page on metacritic,
     then parse that page to get critic & user scores for albums"""
 
@@ -28,12 +26,13 @@ def update(artistid, artist_name,release_groups):
     # We could just do a search, then take the top result, but at least this will
     # cut down on api calls. If it's ineffective then we'll switch to search
 
-    replacements = {" & " : " ", "." : ""}
-    mc_artist_name = helpers.replace_all(artist_name.lower(),replacements)
+    replacements = {" & ": " ", ".": ""}
+    mc_artist_name = helpers.replace_all(artist_name.lower(), replacements)
 
-    mc_artist_name = mc_artist_name.replace(" ","-")
+    mc_artist_name = mc_artist_name.replace(" ", "-")
 
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'}
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'}
 
     url = "http://www.metacritic.com/person/" + mc_artist_name + "?filter-options=music&sort_options=date&num_items=100"
 
@@ -67,12 +66,12 @@ def update(artistid, artist_name,release_groups):
             scores = row.find_all("span")
             critic_score = scores[0].string
             user_score = scores[1].string
-            score_dict = {'title':title,'critic_score':critic_score,'user_score':user_score}
+            score_dict = {'title': title, 'critic_score': critic_score, 'user_score': user_score}
             score_list.append(score_dict)
 
         # Save scores to the database
         controlValueDict = {"ArtistID": artistid}
-        newValueDict = {'MetaCritic':json.dumps(score_list)}
+        newValueDict = {'MetaCritic': json.dumps(score_list)}
         myDB.upsert("artists", newValueDict, controlValueDict)
 
     for score in score_list:
@@ -84,5 +83,5 @@ def update(artistid, artist_name,release_groups):
                 critic_score = score['critic_score']
                 user_score = score['user_score']
                 controlValueDict = {"AlbumID": rg['id']}
-                newValueDict = {'CriticScore':critic_score,'UserScore':user_score}
+                newValueDict = {'CriticScore': critic_score, 'UserScore': user_score}
                 myDB.upsert("albums", newValueDict, controlValueDict)

headphones/music_encoder.py

@@ -13,16 +13,17 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import time
 import shutil
 import subprocess
-import headphones
 import multiprocessing
 
+import os
+import headphones
 from headphones import logger
 from beets.mediafile import MediaFile
 
+
 # xld
 import getXldProfile
 
@@ -32,9 +33,11 @@ def encode(albumPath):
 
     # Return if xld details not found
     if use_xld:
-        (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE)
+        (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(
+            headphones.CONFIG.XLDPROFILE)
         if not xldFormat:
-            logger.error('Details for xld profile \'%s\' not found, files will not be re-encoded', xldProfile)
+            logger.error('Details for xld profile \'%s\' not found, files will not be re-encoded',
+                         xldProfile)
             return None
     else:
         xldProfile = None
@@ -60,7 +63,8 @@ def encode(albumPath):
         for music in f:
             if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
                 if not use_xld:
-                    encoderFormat = headphones.CONFIG.ENCODEROUTPUTFORMAT.encode(headphones.SYS_ENCODING)
+                    encoderFormat = headphones.CONFIG.ENCODEROUTPUTFORMAT.encode(
+                        headphones.SYS_ENCODING)
                 else:
                     xldMusicFile = os.path.join(r, music)
                     xldInfoMusic = MediaFile(xldMusicFile)
@@ -68,9 +72,11 @@ def encode(albumPath):
 
                 if headphones.CONFIG.ENCODERLOSSLESS:
                     ext = os.path.normpath(os.path.splitext(music)[1].lstrip(".")).lower()
-                    if not use_xld and ext == 'flac' or use_xld and (ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)):
+                    if not use_xld and ext == 'flac' or use_xld and (
+                            ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)):
                         musicFiles.append(os.path.join(r, music))
-                        musicTemp = os.path.normpath(os.path.splitext(music)[0] + '.' + encoderFormat)
+                        musicTemp = os.path.normpath(
+                            os.path.splitext(music)[0] + '.' + encoderFormat)
                         musicTempFiles.append(os.path.join(tempDirEncode, musicTemp))
                     else:
                         logger.debug('%s is already encoded', music)
@@ -86,7 +92,7 @@ def encode(albumPath):
             encoder = os.path.join('/Applications', 'xld')
         elif headphones.CONFIG.ENCODER == 'lame':
             if headphones.SYS_PLATFORM == "win32":
-                ## NEED THE DEFAULT LAME INSTALL ON WIN!
+                # NEED THE DEFAULT LAME INSTALL ON WIN!
                 encoder = "C:/Program Files/lame/lame.exe"
             else:
                 encoder = "lame"
@@ -111,24 +117,31 @@ def encode(albumPath):
 
         if use_xld:
             if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate):
-                logger.info('%s has bitrate <= %skb, will not be re-encoded', music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)
+                logger.info('%s has bitrate <= %skb, will not be re-encoded',
+                            music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)
             else:
                 encode = True
         elif headphones.CONFIG.ENCODER == 'lame':
-            if not any(music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x in ["mp3", "wav"]):
-                logger.warn('Lame cannot encode %s format for %s, use ffmpeg', os.path.splitext(music)[1], music)
+            if not any(
+                    music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x
+                    in ["mp3", "wav"]):
+                logger.warn('Lame cannot encode %s format for %s, use ffmpeg',
+                            os.path.splitext(music)[1], music)
             else:
-                if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
-                    logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.CONFIG.BITRATE)
+                if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (
+                    int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
+                    logger.info('%s has bitrate <= %skb, will not be re-encoded', music,
+                                headphones.CONFIG.BITRATE)
                 else:
                     encode = True
         else:
             if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg':
                 if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'):
-                    logger.warn('Cannot re-encode .ogg %s', music.decode(headphones.SYS_ENCODING, 'replace'))
+                    logger.warn('Cannot re-encode .ogg %s',
+                                music.decode(headphones.SYS_ENCODING, 'replace'))
                 else:
                     encode = True
-            elif headphones.CONFIG.ENCODEROUTPUTFORMAT == 'mp3' or headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
+            else:
                 if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + headphones.CONFIG.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE):
                     logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.CONFIG.BITRATE)
                 else:
@@ -155,7 +168,7 @@ def encode(albumPath):
                 processes = headphones.CONFIG.ENCODER_MULTICORE_COUNT
 
             logger.debug("Multi-core encoding enabled, spawning %d processes",
-                processes)
+                         processes)
 
         # Use multiprocessing only if it's worth the overhead. and if it is
         # enabled. If not, then use the old fashioned way.
@@ -185,7 +198,8 @@ def encode(albumPath):
         for dest in musicTempFiles:
             if not os.path.exists(dest):
                 encoder_failed = True
-                logger.error("Encoded file '%s' does not exist in the destination temp directory", dest)
+                logger.error("Encoded file '%s' does not exist in the destination temp directory",
+                             dest)
 
     # No errors, move from temp to parent
     if not encoder_failed and musicTempFiles:
@@ -211,7 +225,9 @@ def encode(albumPath):
 
     # Return with error if any encoding errors
     if encoder_failed:
-        logger.error("One or more files failed to encode. Ensure you have the latest version of %s installed.", headphones.CONFIG.ENCODER)
+        logger.error(
+            "One or more files failed to encode. Ensure you have the latest version of %s installed.",
+            headphones.CONFIG.ENCODER)
         return None
 
     time.sleep(1)
@@ -269,7 +285,8 @@ def command(encoder, musicSource, musicDest, albumPath, xldProfile):
         if not headphones.CONFIG.ADVANCEDENCODER:
             opts.extend(['-h'])
             if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
-                opts.extend(['--resample', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-b', str(headphones.CONFIG.BITRATE)])
+                opts.extend(['--resample', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-b',
+                             str(headphones.CONFIG.BITRATE)])
             elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                 opts.extend(['-v', str(headphones.CONFIG.ENCODERQUALITY)])
         else:
@@ -290,7 +307,8 @@ def command(encoder, musicSource, musicDest, albumPath, xldProfile):
             if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
                 opts.extend(['-strict', 'experimental'])
             if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
-                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab', str(headphones.CONFIG.BITRATE) + 'k'])
+                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
+                             str(headphones.CONFIG.BITRATE) + 'k'])
             elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                 opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
             opts.extend(['-y', '-ac', '2', '-vn'])
@@ -311,7 +329,8 @@ def command(encoder, musicSource, musicDest, albumPath, xldProfile):
             if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a':
                 opts.extend(['-strict', 'experimental'])
             if headphones.CONFIG.ENCODERVBRCBR == 'cbr':
-                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab', str(headphones.CONFIG.BITRATE) + 'k'])
+                opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab',
+                             str(headphones.CONFIG.BITRATE) + 'k'])
             elif headphones.CONFIG.ENCODERVBRCBR == 'vbr':
                 opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)])
             opts.extend(['-y', '-ac', '2', '-vn'])
@@ -337,13 +356,14 @@ def command(encoder, musicSource, musicDest, albumPath, xldProfile):
     logger.debug(subprocess.list2cmdline(cmd))
 
     process = subprocess.Popen(cmd, startupinfo=startupinfo,
-        stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE)
+                               stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
     stdout, stderr = process.communicate(headphones.CONFIG.ENCODER)
 
     # Error if return code not zero
     if process.returncode:
-        logger.error('Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
+        logger.error(
+            'Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
         out = stdout if stdout else stderr
         out = out.decode(headphones.SYS_ENCODING, 'replace')
         outlast2lines = '\n'.join(out.splitlines()[-2:])

headphones/notifiers.py

@@ -13,30 +13,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import logger, helpers, common, request
-
-from xml.dom import minidom
-from httplib import HTTPSConnection
-from urlparse import parse_qsl
 from urllib import urlencode
-from pynma import pynma
-
-import base64
-import cherrypy
 import urllib
-import urllib2
-import headphones
-import os.path
 import subprocess
-import gntp.notifier
 import json
-
-import oauth2 as oauth
-import pythontwitter as twitter
-
 from email.mime.text import MIMEText
 import smtplib
 import email.utils
+from httplib import HTTPSConnection
+from urlparse import parse_qsl
+import urllib2
+
+import os.path
+from headphones import logger, helpers, common, request
+from pynma import pynma
+import cherrypy
+import headphones
+import gntp.notifier
+import oauth2 as oauth
+import pythontwitter as twitter
 
 
 class GROWL(object):
@@ -95,7 +90,7 @@ class GROWL(object):
 
         # Send it, including an image
         image_file = os.path.join(str(headphones.PROG_DIR),
-            "data/images/headphoneslogo.png")
+                                  "data/images/headphoneslogo.png")
 
         with open(image_file, 'rb') as f:
             image = f.read()
@@ -114,7 +109,7 @@ class GROWL(object):
         logger.info(u"Growl notifications sent.")
 
     def updateLibrary(self):
-        #For uniformity reasons not removed
+        # For uniformity reasons not removed
         return
 
     def test(self, host, password):
@@ -151,24 +146,24 @@ class PROWL(object):
                 'priority': headphones.CONFIG.PROWL_PRIORITY}
 
         http_handler.request("POST",
-                                "/publicapi/add",
-                                headers={'Content-type': "application/x-www-form-urlencoded"},
-                                body=urlencode(data))
+                             "/publicapi/add",
+                             headers={'Content-type': "application/x-www-form-urlencoded"},
+                             body=urlencode(data))
         response = http_handler.getresponse()
         request_status = response.status
 
         if request_status == 200:
-                logger.info(u"Prowl notifications sent.")
-                return True
+            logger.info(u"Prowl notifications sent.")
+            return True
         elif request_status == 401:
-                logger.info(u"Prowl auth failed: %s" % response.reason)
-                return False
+            logger.info(u"Prowl auth failed: %s" % response.reason)
+            return False
         else:
-                logger.info(u"Prowl notification failed.")
-                return False
+            logger.info(u"Prowl notification failed.")
+            return False
 
     def updateLibrary(self):
-        #For uniformity reasons not removed
+        # For uniformity reasons not removed
         return
 
     def test(self, keys, priority):
@@ -185,7 +180,6 @@ class MPC(object):
     """
 
     def __init__(self):
-
         pass
 
     def notify(self):
@@ -218,9 +212,11 @@ class XBMC(object):
         url = host + '/jsonrpc'
 
         if self.password:
-            response = request.request_json(url, method="post", data=json.dumps(data), headers=headers, auth=(self.username, self.password))
+            response = request.request_json(url, method="post", data=json.dumps(data),
+                                            headers=headers, auth=(self.username, self.password))
         else:
-            response = request.request_json(url, method="post", data=json.dumps(data), headers=headers)
+            response = request.request_json(url, method="post", data=json.dumps(data),
+                                            headers=headers)
 
         if response:
             return response[0]['result']
@@ -244,20 +240,24 @@ class XBMC(object):
 
         header = "Headphones"
         message = "%s - %s added to your library" % (artist, album)
-        time = "3000" # in ms
+        time = "3000"  # in ms
 
         for host in hosts:
             logger.info('Sending notification command to XMBC @ ' + host)
             try:
-                version = self._sendjson(host, 'Application.GetProperties', {'properties': ['version']})['version']['major']
+                version = \
+                self._sendjson(host, 'Application.GetProperties', {'properties': ['version']})[
+                    'version']['major']
 
-                if version < 12: #Eden
+                if version < 12:  # Eden
                     notification = header + "," + message + "," + time + "," + albumartpath
-                    notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification(' + notification + ')'}
+                    notifycommand = {'command': 'ExecBuiltIn',
+                                     'parameter': 'Notification(' + notification + ')'}
                     request = self._sendhttp(host, notifycommand)
 
-                else: #Frodo
-                    params = {'title': header, 'message': message, 'displaytime': int(time), 'image': albumartpath}
+                else:  # Frodo
+                    params = {'title': header, 'message': message, 'displaytime': int(time),
+                              'image': albumartpath}
                     request = self._sendjson(host, 'GUI.ShowNotification', params)
 
                 if not request:
@@ -335,9 +335,11 @@ class Plex(object):
         url = host + '/jsonrpc'
 
         if self.password:
-            response = request.request_json(url, method="post", data=json.dumps(data), headers=headers, auth=(self.username, self.password))
+            response = request.request_json(url, method="post", data=json.dumps(data),
+                                            headers=headers, auth=(self.username, self.password))
         else:
-            response = request.request_json(url, method="post", data=json.dumps(data), headers=headers)
+            response = request.request_json(url, method="post", data=json.dumps(data),
+                                            headers=headers)
 
         if response:
             return response[0]['result']
@@ -376,20 +378,24 @@ class Plex(object):
 
         header = "Headphones"
         message = "%s - %s added to your library" % (artist, album)
-        time = "3000" # in ms
+        time = "3000"  # in ms
 
         for host in hosts:
             logger.info('Sending notification command to Plex client @ ' + host)
             try:
-                version = self._sendjson(host, 'Application.GetProperties', {'properties': ['version']})['version']['major']
+                version = \
+                self._sendjson(host, 'Application.GetProperties', {'properties': ['version']})[
+                    'version']['major']
 
-                if version < 12: #Eden
+                if version < 12:  # Eden
                     notification = header + "," + message + "," + time + "," + albumartpath
-                    notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification(' + notification + ')'}
+                    notifycommand = {'command': 'ExecBuiltIn',
+                                     'parameter': 'Notification(' + notification + ')'}
                     request = self._sendhttp(host, notifycommand)
 
-                else: #Frodo
-                    params = {'title': header, 'message': message, 'displaytime': int(time), 'image': albumartpath}
+                else:  # Frodo
+                    params = {'title': header, 'message': message, 'displaytime': int(time),
+                              'image': albumartpath}
                     request = self._sendjson(host, 'GUI.ShowNotification', params)
 
                 if not request:
@@ -438,12 +444,11 @@ class NMA(object):
 
 
 class PUSHBULLET(object):
-
     def __init__(self):
         self.apikey = headphones.CONFIG.PUSHBULLET_APIKEY
         self.deviceid = headphones.CONFIG.PUSHBULLET_DEVICEID
 
-    def notify(self, message):
+    def notify(self, message, status):
         if not headphones.CONFIG.PUSHBULLET_ENABLED:
             return
 
@@ -451,13 +456,13 @@ class PUSHBULLET(object):
 
         data = {'type': "note",
                 'title': "Headphones",
-                'body': message}
+                'body': message + ': ' + status}
 
         if self.deviceid:
             data['device_iden'] = self.deviceid
 
-        headers={'Content-type': "application/json",
-                 'Authorization': 'Bearer ' + headphones.CONFIG.PUSHBULLET_APIKEY}
+        headers = {'Content-type': "application/json",
+                   'Authorization': 'Bearer ' + headphones.CONFIG.PUSHBULLET_APIKEY}
 
         response = request.request_json(url, method="post", headers=headers, data=json.dumps(data))
 
@@ -468,8 +473,8 @@ class PUSHBULLET(object):
             logger.info(u"PushBullet notification failed.")
             return False
 
-class PUSHALOT(object):
 
+class PUSHALOT(object):
     def notify(self, message, event):
         if not headphones.CONFIG.PUSHALOT_ENABLED:
             return
@@ -487,9 +492,9 @@ class PUSHALOT(object):
                 'Body': message.encode("utf-8")}
 
         http_handler.request("POST",
-                                "/api/sendmessage",
-                                headers={'Content-type': "application/x-www-form-urlencoded"},
-                                body=urlencode(data))
+                             "/api/sendmessage",
+                             headers={'Content-type': "application/x-www-form-urlencoded"},
+                             body=urlencode(data))
         response = http_handler.getresponse()
         request_status = response.status
 
@@ -498,14 +503,14 @@ class PUSHALOT(object):
         logger.debug(u"Pushalot response body: %r" % response.read())
 
         if request_status == 200:
-                logger.info(u"Pushalot notifications sent.")
-                return True
+            logger.info(u"Pushalot notifications sent.")
+            return True
         elif request_status == 410:
-                logger.info(u"Pushalot auth failed: %s" % response.reason)
-                return False
+            logger.info(u"Pushalot auth failed: %s" % response.reason)
+            return False
         else:
-                logger.info(u"Pushalot notification failed.")
-                return False
+            logger.info(u"Pushalot notification failed.")
+            return False
 
 
 class Synoindex(object):
@@ -519,7 +524,8 @@ class Synoindex(object):
         path = os.path.abspath(path)
 
         if not self.util_exists():
-            logger.warn("Error sending notification: synoindex utility not found at %s" % self.util_loc)
+            logger.warn(
+                "Error sending notification: synoindex utility not found at %s" % self.util_loc)
             return
 
         if os.path.isfile(path):
@@ -527,15 +533,17 @@ class Synoindex(object):
         elif os.path.isdir(path):
             cmd_arg = '-A'
         else:
-            logger.warn("Error sending notification: Path passed to synoindex was not a file or folder.")
+            logger.warn(
+                "Error sending notification: Path passed to synoindex was not a file or folder.")
             return
 
         cmd = [self.util_loc, cmd_arg, path]
         logger.info("Calling synoindex command: %s" % str(cmd))
         try:
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=headphones.PROG_DIR)
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                 cwd=headphones.PROG_DIR)
             out, error = p.communicate()
-            #synoindex never returns any codes other than '0', highly irritating
+            # synoindex never returns any codes other than '0', highly irritating
         except OSError, e:
             logger.warn("Error sending notification: %s" % str(e))
 
@@ -546,7 +554,6 @@ class Synoindex(object):
 
 
 class PUSHOVER(object):
-
     def __init__(self):
         self.enabled = headphones.CONFIG.PUSHOVER_ENABLED
         self.keys = headphones.CONFIG.PUSHOVER_KEYS
@@ -584,7 +591,7 @@ class PUSHOVER(object):
             return False
 
     def updateLibrary(self):
-        #For uniformity reasons not removed
+        # For uniformity reasons not removed
         return
 
     def test(self, keys, priority):
@@ -596,7 +603,6 @@ class PUSHOVER(object):
 
 
 class TwitterNotifier(object):
-
     REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth/request_token'
     ACCESS_TOKEN_URL = 'https://api.twitter.com/oauth/access_token'
     AUTHORIZATION_URL = 'https://api.twitter.com/oauth/authorize'
@@ -608,14 +614,17 @@ class TwitterNotifier(object):
 
     def notify_snatch(self, title):
         if headphones.CONFIG.TWITTER_ONSNATCH:
-            self._notifyTwitter(common.notifyStrings[common.NOTIFY_SNATCH] + ': ' + title + ' at ' + helpers.now())
+            self._notifyTwitter(
+                common.notifyStrings[common.NOTIFY_SNATCH] + ': ' + title + ' at ' + helpers.now())
 
     def notify_download(self, title):
         if headphones.CONFIG.TWITTER_ENABLED:
-            self._notifyTwitter(common.notifyStrings[common.NOTIFY_DOWNLOAD] + ': ' + title + ' at ' + helpers.now())
+            self._notifyTwitter(common.notifyStrings[
+                                    common.NOTIFY_DOWNLOAD] + ': ' + title + ' at ' + helpers.now())
 
     def test_notify(self):
-        return self._notifyTwitter("This is a test notification from Headphones at " + helpers.now(), force=True)
+        return self._notifyTwitter(
+            "This is a test notification from Headphones at " + helpers.now(), force=True)
 
     def _get_authorization(self):
 
@@ -652,7 +661,8 @@ class TwitterNotifier(object):
         logger.info('oauth_consumer: ' + str(oauth_consumer))
         oauth_client = oauth.Client(oauth_consumer, token)
         logger.info('oauth_client: ' + str(oauth_client))
-        resp, content = oauth_client.request(self.ACCESS_TOKEN_URL, method='POST', body='oauth_verifier=%s' % key)
+        resp, content = oauth_client.request(self.ACCESS_TOKEN_URL, method='POST',
+                                             body='oauth_verifier=%s' % key)
         logger.info('resp, content: ' + str(resp) + ',' + str(content))
 
         access_token = dict(parse_qsl(content))
@@ -660,7 +670,8 @@ class TwitterNotifier(object):
 
         logger.info('resp[status] = ' + str(resp['status']))
         if resp['status'] != '200':
-            logger.info('The request for a token with did not succeed: ' + str(resp['status']), logger.ERROR)
+            logger.info('The request for a token with did not succeed: ' + str(resp['status']),
+                        logger.ERROR)
             return False
         else:
             logger.info('Your Twitter Access Token key: %s' % access_token['oauth_token'])
@@ -698,7 +709,6 @@ class TwitterNotifier(object):
 
 
 class OSX_NOTIFY(object):
-
     def __init__(self):
         try:
             self.objc = __import__("objc")
@@ -751,7 +761,7 @@ class OSX_NOTIFY(object):
             if image:
                 source_img = self.AppKit.NSImage.alloc().initByReferencingFile_(image)
                 notification.setContentImage_(source_img)
-                #notification.set_identityImage_(source_img)
+                # notification.set_identityImage_(source_img)
             notification.setHasActionButton_(False)
 
             notification_center = NSUserNotificationCenter.defaultUserNotificationCenter()
@@ -769,7 +779,6 @@ class OSX_NOTIFY(object):
 
 
 class BOXCAR(object):
-
     def __init__(self):
         self.url = 'https://new.boxcar.io/api/notifications'
 
@@ -783,7 +792,7 @@ class BOXCAR(object):
                 'notification[title]': title.encode('utf-8'),
                 'notification[long_message]': message.encode('utf-8'),
                 'notification[sound]': "done"
-                })
+            })
 
             req = urllib2.Request(self.url)
             handle = urllib2.urlopen(req, data)
@@ -796,7 +805,6 @@ class BOXCAR(object):
 
 
 class SubSonicNotifier(object):
-
     def __init__(self):
         self.host = headphones.CONFIG.SUBSONIC_HOST
         self.username = headphones.CONFIG.SUBSONIC_USERNAME
@@ -812,10 +820,10 @@ class SubSonicNotifier(object):
 
         # Invoke request
         request.request_response(self.host + "musicFolderSettings.view?scanNow",
-            auth=(self.username, self.password))
+                                 auth=(self.username, self.password))
+
 
 class Email(object):
-
     def notify(self, subject, message):
 
         message = MIMEText(message, 'plain', "utf-8")
@@ -824,23 +832,60 @@ class Email(object):
         message['To'] = headphones.CONFIG.EMAIL_TO
 
         try:
-            if (headphones.CONFIG.EMAIL_SSL):
-                mailserver = smtplib.SMTP_SSL(headphones.CONFIG.EMAIL_SMTP_SERVER, headphones.CONFIG.EMAIL_SMTP_PORT)
+            if headphones.CONFIG.EMAIL_SSL:
+                mailserver = smtplib.SMTP_SSL(headphones.CONFIG.EMAIL_SMTP_SERVER,
+                                              headphones.CONFIG.EMAIL_SMTP_PORT)
             else:
-                mailserver = smtplib.SMTP(headphones.CONFIG.EMAIL_SMTP_SERVER, headphones.CONFIG.EMAIL_SMTP_PORT)
+                mailserver = smtplib.SMTP(headphones.CONFIG.EMAIL_SMTP_SERVER,
+                                          headphones.CONFIG.EMAIL_SMTP_PORT)
 
-            if (headphones.CONFIG.EMAIL_TLS):
+            if headphones.CONFIG.EMAIL_TLS:
                 mailserver.starttls()
 
             mailserver.ehlo()
 
             if headphones.CONFIG.EMAIL_SMTP_USER:
-                mailserver.login(headphones.CONFIG.EMAIL_SMTP_USER, headphones.CONFIG.EMAIL_SMTP_PASSWORD)
+                mailserver.login(headphones.CONFIG.EMAIL_SMTP_USER,
+                                 headphones.CONFIG.EMAIL_SMTP_PASSWORD)
 
-            mailserver.sendmail(headphones.CONFIG.EMAIL_FROM, headphones.CONFIG.EMAIL_TO, message.as_string())
+            mailserver.sendmail(headphones.CONFIG.EMAIL_FROM, headphones.CONFIG.EMAIL_TO,
+                                message.as_string())
             mailserver.quit()
             return True
 
         except Exception, e:
             logger.warn('Error sending Email: %s' % e)
             return False
+
+
+class TELEGRAM(object):
+
+    def notify(self, message, status):
+        if not headphones.CONFIG.TELEGRAM_ENABLED:
+            return
+
+        import requests
+
+        TELEGRAM_API = "https://api.telegram.org/bot%s/%s"
+
+        # Get configuration data
+        token = headphones.CONFIG.TELEGRAM_TOKEN
+        userid = headphones.CONFIG.TELEGRAM_USERID
+
+        # Construct message
+        payload = {'chat_id': userid, 'text': status + ': ' + message}
+
+        # Send message to user using Telegram's Bot API
+        try:
+            response = requests.post(TELEGRAM_API % (token, "sendMessage"), data=payload)
+        except Exception, e:
+            logger.info(u'Telegram notify failed: ' + str(e))
+
+        # Error logging
+        sent_successfuly = True
+        if not response.status_code == 200:
+            logger.info(u'Could not send notification to TelegramBot (token=%s). Response: [%s]', (token, response.text))
+            sent_successfuly = False
+
+        logger.info(u"Telegram notifications sent.")
+        return sent_successfuly

headphones/nzbget.py

@@ -19,18 +19,15 @@
 # along with Sick Beard.  If not, see <http://www.gnu.org/licenses/>.
 
 
-import httplib
-
-import headphones
-
 from base64 import standard_b64encode
+import httplib
 import xmlrpclib
 
+import headphones
 from headphones import logger
 
 
 def sendNZB(nzb):
-
     addToTop = False
     nzbgetXMLrpc = "%(protocol)s://%(username)s:%(password)s@%(host)s/xmlrpc"
 
@@ -45,17 +42,22 @@ def sendNZB(nzb):
         protocol = 'http'
         host = headphones.CONFIG.NZBGET_HOST.replace('http://', '', 1)
 
-    url = nzbgetXMLrpc % {"protocol": protocol, "host": host, "username": headphones.CONFIG.NZBGET_USERNAME, "password": headphones.CONFIG.NZBGET_PASSWORD}
+    url = nzbgetXMLrpc % {"protocol": protocol, "host": host,
+                          "username": headphones.CONFIG.NZBGET_USERNAME,
+                          "password": headphones.CONFIG.NZBGET_PASSWORD}
 
     nzbGetRPC = xmlrpclib.ServerProxy(url)
     try:
-        if nzbGetRPC.writelog("INFO", "headphones connected to drop of %s any moment now." % (nzb.name + ".nzb")):
+        if nzbGetRPC.writelog("INFO", "headphones connected to drop of %s any moment now." % (
+            nzb.name + ".nzb")):
             logger.debug(u"Successfully connected to NZBget")
         else:
-            logger.info(u"Successfully connected to NZBget, but unable to send a message" % (nzb.name + ".nzb"))
+            logger.info(u"Successfully connected to NZBget, but unable to send a message" % (
+            nzb.name + ".nzb"))
 
     except httplib.socket.error:
-        logger.error(u"Please check your NZBget host and port (if it is running). NZBget is not responding to this combination")
+        logger.error(
+            u"Please check your NZBget host and port (if it is running). NZBget is not responding to this combination")
         return False
 
     except xmlrpclib.ProtocolError, e:
@@ -82,7 +84,9 @@ def sendNZB(nzb):
         nzbget_version = int(nzbget_version_str[:nzbget_version_str.find(".")])
         if nzbget_version == 0:
             if nzbcontent64 is not None:
-                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, addToTop, nzbcontent64)
+                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb",
+                                                 headphones.CONFIG.NZBGET_CATEGORY, addToTop,
+                                                 nzbcontent64)
             else:
                 # from headphones.common.providers.generic import GenericProvider
                 # if nzb.resultType == "nzb":
@@ -95,24 +99,35 @@ def sendNZB(nzb):
                 return False
         elif nzbget_version == 12:
             if nzbcontent64 is not None:
-                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False,
+                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb",
+                                                 headphones.CONFIG.NZBGET_CATEGORY,
+                                                 headphones.CONFIG.NZBGET_PRIORITY, False,
                                                  nzbcontent64, False, dupekey, dupescore, "score")
             else:
-                nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False,
+                nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb",
+                                                    headphones.CONFIG.NZBGET_CATEGORY,
+                                                    headphones.CONFIG.NZBGET_PRIORITY, False,
                                                     nzb.url, False, dupekey, dupescore, "score")
         # v13+ has a new combined append method that accepts both (url and content)
         # also the return value has changed from boolean to integer
         # (Positive number representing NZBID of the queue item. 0 and negative numbers represent error codes.)
         elif nzbget_version >= 13:
-            nzbget_result = True if nzbGetRPC.append(nzb.name + ".nzb", nzbcontent64 if nzbcontent64 is not None else nzb.url,
-                                                     headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, False, dupekey, dupescore,
+            nzbget_result = True if nzbGetRPC.append(nzb.name + ".nzb",
+                                                     nzbcontent64 if nzbcontent64 is not None else nzb.url,
+                                                     headphones.CONFIG.NZBGET_CATEGORY,
+                                                     headphones.CONFIG.NZBGET_PRIORITY, False,
+                                                     False, dupekey, dupescore,
                                                      "score") > 0 else False
         else:
             if nzbcontent64 is not None:
-                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False,
+                nzbget_result = nzbGetRPC.append(nzb.name + ".nzb",
+                                                 headphones.CONFIG.NZBGET_CATEGORY,
+                                                 headphones.CONFIG.NZBGET_PRIORITY, False,
                                                  nzbcontent64)
             else:
-                nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False,
+                nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb",
+                                                    headphones.CONFIG.NZBGET_CATEGORY,
+                                                    headphones.CONFIG.NZBGET_PRIORITY, False,
                                                     nzb.url)
 
         if nzbget_result:
@@ -122,5 +137,6 @@ def sendNZB(nzb):
             logger.error(u"NZBget could not add %s to the queue" % (nzb.name + ".nzb"))
             return False
     except:
-        logger.error(u"Connect Error to NZBget: could not add %s to the queue" % (nzb.name + ".nzb"))
+        logger.error(
+            u"Connect Error to NZBget: could not add %s to the queue" % (nzb.name + ".nzb"))
         return False

headphones/pathrender.py

@@ -0,0 +1,237 @@
+# encoding=utf8
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
+'''Path pattern substitution module, see details below for syntax.
+
+   The pattern matching is loosely based on foobar2000 pattern syntax,
+   i.e. the notion of escaping characters with \' and optional elements
+   enclosed in square brackets [] is taken from there while the
+   substitution variable names are Perl-ish or sh-ish. The following
+   syntax elements are supported:
+    * escaped literal strings, that is everything that is enclosed
+      within single quotes (like \'this\');
+    * substitution variables, which start with dollar sign ($) and
+      extend until next non-alphanumeric+underscore character
+      (like $This and $5_that).
+    * optional elements enclosed in curly braces, which render
+      nonempty value only if any variable or optional inside returned
+      nonempty value, ignoring literals (like {\'[\'$That\']\'}).
+'''
+from __future__ import print_function
+from enum import Enum
+
+__author__ = "Andrzej Ciarkowski <andrzej.ciarkowski@gmail.com>"
+
+
+class _PatternElement(object):
+    '''ABC for hierarchy of path name renderer pattern elements.'''
+    def render(self, replacement):
+        # type: (Mapping[str,str]) -> str
+        '''Format this _PatternElement into string using provided substitution dictionary.'''
+        raise NotImplementedError()
+
+
+class _Generator(_PatternElement):
+    # pylint: disable=abstract-method
+    '''Tagging interface for "content-generating" elements like replacement or optional block.'''
+    pass
+
+
+class _Replacement(_Generator):
+    '''Replacement variable, eg. $title.'''
+    def __init__(self, pattern):
+        # type: (str)
+        self._pattern = pattern
+
+    def render(self, replacement):
+        # type: (Mapping[str,str]) -> str
+        return replacement.get(self._pattern, self._pattern)
+
+    def __str__(self):
+        return self._pattern
+
+
+class _LiteralText(_PatternElement):
+    '''Just a plain piece of text to be rendered "as is".'''
+    def __init__(self, text):
+        # type: (str)
+        self._text = text
+
+    def render(self, replacement):
+        # type: (Mapping[str,str]) -> str
+        return self._text
+
+    def __str__(self):
+        return self._text
+
+
+class _OptionalBlock(_Generator):
+    '''Optional block will render its contents only if any _Generator in its scope did return non-empty result.'''
+
+    def __init__(self, scope):
+        # type: ([_PatternElement])
+        self._scope = scope
+
+    def render(self, replacement):
+        # type: (Mapping[str,str]) -> str
+        res = [(isinstance(x, _Generator), x.render(replacement)) for x in self._scope]
+        if any((t[0] and len(t[1]) != 0) for t in res):
+            return u"".join(t[1] for t in res)
+        else:
+            return u""
+
+
+_OPTIONAL_START = u'{'
+_OPTIONAL_END = u'}'
+_ESCAPE_CHAR = u'\''
+_REPLACEMENT_START = u'$'
+
+
+def _is_replacement_valid(c):
+    # type: (str) -> bool
+    return c.isalnum() or c == u'_'
+
+
+class _State(Enum):
+    LITERAL = 0
+    ESCAPE = 1
+    REPLACEMENT = 2
+
+
+def _append_literal(scope, text):
+    # type: ([_PatternElement], str) -> None
+    '''Append literal text to the scope BUT ONLY if it's not an empty string.'''
+    if len(text) == 0:
+        return
+    scope.append(_LiteralText(text))
+
+
+class Warnings(Enum):
+    '''Pattern parsing warnings, as stored withing warnings property of Pattern object after parsing.'''
+    UNCLOSED_ESCAPE = 'Warnings.UNCLOSED_ESCAPE'
+    UNCLOSED_OPTIONAL = 'Warnings.UNCLOSED_OPTIONAL'
+
+
+def _parse_pattern(pattern, warnings):
+    # type: (str,MutableSet[Warnings]) -> [_PatternElement]
+    '''Parse path pattern text into list of _PatternElements, put warnings into the provided set.'''
+    start = 0                   # index of current state start char
+    root_scope = []             # here our _PatternElements will reside
+    scope_stack = [root_scope]  # stack so that we can return to the outer scope
+    scope = root_scope          # pointer to the current list for _OptionalBlock
+    inside_optional = 0         # nesting level of _OptionalBlocks
+    state = _State.LITERAL      # current state
+    for i, c in enumerate(pattern):
+        if state is _State.ESCAPE:
+            if c != _ESCAPE_CHAR:
+                # only escape char can get us out of _State.ESCAPE
+                continue
+            _append_literal(scope, pattern[start + 1:i])
+            state = _State.LITERAL
+            start = i + 1
+            # after exiting _State.ESCAPE on escape char no more processing of c
+            continue
+        if state is _State.REPLACEMENT:
+            if _is_replacement_valid(c):
+                # only replacement invalid can get us out _State.REPLACEMENT
+                continue
+            scope.append(_Replacement(pattern[start:i]))
+            state = _State.LITERAL
+            start = i
+            # intentional fall-through to _State.LITERAL
+        assert state is _State.LITERAL
+        if c == _ESCAPE_CHAR:
+            _append_literal(scope, pattern[start:i])
+            state = _State.ESCAPE
+            start = i
+            # no more processing to escape char c
+            continue
+        if c == _REPLACEMENT_START:
+            _append_literal(scope, pattern[start:i])
+            state = _State.REPLACEMENT
+            start = i
+            # no more processing to replacement char c
+            continue
+        if c == _OPTIONAL_START:
+            _append_literal(scope, pattern[start:i])
+            inside_optional += 1
+            new_scope = []
+            scope_stack.append(new_scope)
+            scope = new_scope
+            start = i + 1
+            continue
+        if c == _OPTIONAL_END:
+            if inside_optional == 0:
+                # no optional block to end, just treat as literal text
+                continue
+            inside_optional -= 1
+            _append_literal(scope, pattern[start:i])
+            scope_stack.pop()
+            prev_scope = scope_stack[-1]
+            prev_scope.append(_OptionalBlock(scope))
+            scope = prev_scope
+            start = i + 1
+        # fi
+    # done
+    if state is _State.ESCAPE:
+        warnings.add(Warnings.UNCLOSED_ESCAPE)
+    if inside_optional != 0:
+        warnings.add(Warnings.UNCLOSED_OPTIONAL)
+    if state is _State.REPLACEMENT:
+        root_scope.append(_Replacement(pattern[start:]))
+    else:
+        # don't care about unclosed elements :P
+        _append_literal(root_scope, pattern[start:])
+    return root_scope
+
+
+class Pattern(object):
+    '''Stores preparsed rename pattern for repeated use.
+
+       If using the same pattern repeatedly it is much more effective
+       to parse the pattern into Pattern object and use it instead of
+       parsing the textual pattern on each substitution. To use Pattern
+       object for substitution simply call it as it was function
+       providing dictionary as an argument (see __call__()).'''
+
+    def __init__(self, pattern):
+        # type: (str)
+        self._warnings = set()
+        self._pattern = _parse_pattern(pattern, self._warnings)
+
+    def __call__(self, replacement):
+        # type: (Mapping[str,str]) -> str
+        '''Execute path rendering/substitution based on replacement dictionary.'''
+        return u"".join(p.render(replacement) for p in self._pattern)
+
+    def _get_warnings(self):
+        # type: () -> str
+        '''Getter for warnings property.'''
+        return self._warnings
+
+    warnings = property(_get_warnings, doc="Access warnings raised during pattern parsing")
+
+
+def render(pattern, replacement):
+    # type: (str, Mapping[str,str]) -> (str, AbstractSet[Warnings])
+    '''Render path name based on replacement pattern and dictionary.'''
+    p = Pattern(pattern)
+    return p(replacement), p.warnings
+
+if __name__ == "__main__":
+    # primitive test ;)
+    p = Pattern(u"[$Disc.]$Track - $Artist - $Title[ '['$Year']'")
+    d = {'$Disc': '', '$Track': '05', '$Artist': u'Grzegżółka', '$Title': u'Błona kapłona', '$Year': '2019'}
+    print(p(d).encode('utf8'), p.warnings)

headphones/request.py

@@ -13,16 +13,20 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import logger
-
 from xml.dom import minidom
-from bs4 import BeautifulSoup
+import collections
 
+import sys
+from bs4 import BeautifulSoup
 import requests
+from headphones import logger
 import feedparser
 import headphones
 import headphones.lock
-import collections
+
+
+# Disable SSL certificate warnings. We have our own handling
+requests.packages.urllib3.disable_warnings()
 
 # Dictionary with last request times, for rate limiting.
 last_requests = collections.defaultdict(int)
@@ -51,6 +55,14 @@ def request_response(url, method="get", auto_raise=True,
     # pose a security issue!
     kwargs["verify"] = bool(headphones.CONFIG.VERIFY_SSL_CERT)
 
+    #This fix is put in place for systems with broken SSL (like QNAP)
+    if not headphones.CONFIG.VERIFY_SSL_CERT and sys.version_info >= (2, 7, 9):
+        try:
+            import ssl
+            ssl._create_default_https_context = ssl._create_unverified_context
+        except:
+            pass
+
     # Map method to the request.XXX method. This is a simple hack, but it
     # allows requests to apply more magic per method. See lib/requests/api.py.
     request_method = getattr(requests, method.lower())
@@ -95,7 +107,8 @@ def request_response(url, method="get", auto_raise=True,
             "host is up and running.")
     except requests.Timeout:
         logger.error(
-            "Request timed out. The remote host did not respond in a timely manner.")
+            "Request timed out. The remote host did not respond in a timely "
+            "manner.")
     except requests.HTTPError as e:
         if e.response is not None:
             if e.response.status_code >= 500:
@@ -206,7 +219,8 @@ def server_message(response):
     message = None
 
     # First attempt is to 'read' the response as HTML
-    if response.headers.get("content-type") and "text/html" in response.headers.get("content-type"):
+    if response.headers.get("content-type") and \
+                    "text/html" in response.headers.get("content-type"):
         try:
             soup = BeautifulSoup(response.content, "html5lib")
         except Exception:

headphones/rutracker.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+
+import urllib
+import time
+from urlparse import urlparse
+
+import re
+import requests as requests
+from bs4 import BeautifulSoup
+import headphones
+from headphones import logger
+
+
+class Rutracker(object):
+    def __init__(self):
+        self.session = requests.session()
+        self.timeout = 60
+        self.loggedin = False
+        self.maxsize = 0
+        self.search_referer = 'http://rutracker.org/forum/tracker.php'
+
+    def logged_in(self):
+        return self.loggedin
+
+    def still_logged_in(self, html):
+        if not html or "action=\"http://login.rutracker.org/forum/login.php\">" in html:
+            return False
+        else:
+            return True
+
+    def login(self):
+        """
+        Logs in user
+        """
+
+        loginpage = 'http://login.rutracker.org/forum/login.php'
+        post_params = {
+            'login_username': headphones.CONFIG.RUTRACKER_USER,
+            'login_password': headphones.CONFIG.RUTRACKER_PASSWORD,
+            'login': b'\xc2\xf5\xee\xe4'  # '%C2%F5%EE%E4'
+        }
+
+        logger.info("Attempting to log in to rutracker...")
+
+        try:
+            r = self.session.post(loginpage, data=post_params, timeout=self.timeout)
+            # try again
+            if 'bb_data' not in r.cookies.keys():
+                time.sleep(10)
+                r = self.session.post(loginpage, data=post_params, timeout=self.timeout)
+            if r.status_code != 200:
+                logger.error("rutracker login returned status code %s" % r.status_code)
+                self.loggedin = False
+            else:
+                if 'bb_data' in r.cookies.keys():
+                    self.loggedin = True
+                    logger.info("Successfully logged in to rutracker")
+                else:
+                    logger.error(
+                        "Could not login to rutracker, credentials maybe incorrect, site is down or too many attempts. Try again later")
+                    self.loggedin = False
+            return self.loggedin
+        except Exception as e:
+            logger.error("Unknown error logging in to rutracker: %s" % e)
+            self.loggedin = False
+            return self.loggedin
+
+    def searchurl(self, artist, album, year, format):
+        """
+        Return the search url
+        """
+
+        # Build search url
+        searchterm = ''
+        if artist != 'Various Artists':
+            searchterm = artist
+            searchterm = searchterm + ' '
+        searchterm = searchterm + album
+        searchterm = searchterm + ' '
+        searchterm = searchterm + year
+
+        if format == 'lossless':
+            format = '+lossless'
+            self.maxsize = 10000000000
+        elif format == 'lossless+mp3':
+            format = '+lossless||mp3||aac'
+            self.maxsize = 10000000000
+        else:
+            format = '+mp3||aac'
+            self.maxsize = 300000000
+
+        # sort by size, descending.
+        sort = '&o=7&s=2'
+
+        searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort)
+
+        logger.info("Searching rutracker using term: %s", searchterm)
+
+        return searchurl
+
+    def search(self, searchurl):
+        """
+        Parse the search results and return valid torrent list
+        """
+
+        try:
+            headers = {'Referer': self.search_referer}
+            r = self.session.get(url=searchurl, headers=headers, timeout=self.timeout)
+
+            soup = BeautifulSoup(r.content, 'html5lib')
+
+            # Debug
+            # logger.debug (soup.prettify())
+
+            # Check if still logged in
+            if not self.still_logged_in(soup):
+                self.login()
+                r = self.session.get(url=searchurl, timeout=self.timeout)
+                soup = BeautifulSoup(r.content, 'html5lib')
+                if not self.still_logged_in(soup):
+                    logger.error("Error getting rutracker data")
+                    return None
+
+            # Process
+            rulist = []
+            i = soup.find('table', id='tor-tbl')
+            if not i:
+                logger.info("No valid results found from rutracker")
+                return None
+            minimumseeders = int(headphones.CONFIG.NUMBEROFSEEDERS) - 1
+
+            for item in zip(i.find_all(class_='hl-tags'), i.find_all(class_='dl-stub'),
+                            i.find_all(class_='seedmed')):
+                title = item[0].get_text()
+                url = item[1].get('href')
+                size_formatted = item[1].get_text()[:-2]
+                seeds = item[2].get_text()
+                size_parts = size_formatted.split()
+                size = float(size_parts[0])
+
+                if size_parts[1] == 'KB':
+                    size *= 1024
+                if size_parts[1] == 'MB':
+                    size *= 1024 ** 2
+                if size_parts[1] == 'GB':
+                    size *= 1024 ** 3
+                if size_parts[1] == 'TB':
+                    size *= 1024 ** 4
+
+                if size < self.maxsize and minimumseeders < int(seeds):
+                    logger.info('Found %s. Size: %s' % (title, size_formatted))
+                    # Torrent topic page
+                    torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])[
+                        't']
+                    topicurl = 'http://rutracker.org/forum/viewtopic.php?t=' + torrent_id
+                    rulist.append((title, size, topicurl, 'rutracker.org', 'torrent', True))
+                else:
+                    logger.info(
+                        "%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i)" % (
+                        title, size, int(seeds)))
+
+            if not rulist:
+                logger.info("No valid results found from rutracker")
+
+            return rulist
+
+        except Exception as e:
+            logger.error("An unknown error occurred in the rutracker parser: %s" % e)
+            return None
+
+    def get_torrent_data(self, url):
+        """
+        return the .torrent data
+        """
+
+        torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
+        downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id
+        cookie = {'bb_dl': torrent_id}
+        try:
+            headers = {'Referer': url}
+            r = self.session.post(url=downloadurl, cookies=cookie, headers=headers,
+                                  timeout=self.timeout)
+            return r.content
+        except Exception as e:
+            logger.error('Error getting torrent: %s', e)
+            return False
+
+    # TODO get this working in utorrent.py
+    def utorrent_add_file(self, data):
+
+        host = headphones.CONFIG.UTORRENT_HOST
+        if not host.startswith('http'):
+            host = 'http://' + host
+        if host.endswith('/'):
+            host = host[:-1]
+        if host.endswith('/gui'):
+            host = host[:-4]
+
+        base_url = host
+
+        url = base_url + '/gui/'
+        self.session.auth = (
+        headphones.CONFIG.UTORRENT_USERNAME, headphones.CONFIG.UTORRENT_PASSWORD)
+
+        try:
+            r = self.session.get(url + 'token.html')
+        except Exception as e:
+            logger.error('Error getting token: %s', e)
+            return
+
+        if r.status_code == 401:
+            logger.debug('Error reaching utorrent')
+            return
+
+        regex = re.search(r'.+>([^<]+)</div></html>', r.text)
+        if regex is None:
+            logger.debug('Error reading token')
+            return
+
+        self.session.params = {'token': regex.group(1)}
+        files = {'torrent_file': ("", data)}
+
+        try:
+            self.session.post(url, params={'action': 'add-file'}, files=files)
+        except Exception as e:
+            logger.exception('Error adding file to utorrent %s', e)

headphones/sab.py

@@ -13,26 +13,24 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-#####################################
-## Stolen from Sick-Beard's sab.py ##
-#####################################
+###################################
+# Stolen from Sick-Beard's sab.py #
+###################################
 
-import MultipartPostHandler
-import headphones
 import cookielib
-import httplib
 
+import headphones
 from headphones.common import USER_AGENT
 from headphones import logger, helpers, request
 
 
 def sab_api_call(request_type=None, params={}, **kwargs):
-
     if not headphones.CONFIG.SAB_HOST.startswith('http'):
         headphones.CONFIG.SAB_HOST = 'http://' + headphones.CONFIG.SAB_HOST
 
     if headphones.CONFIG.SAB_HOST.endswith('/'):
-        headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[0:len(headphones.CONFIG.SAB_HOST) - 1]
+        headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[
+                                     0:len(headphones.CONFIG.SAB_HOST) - 1]
 
     url = headphones.CONFIG.SAB_HOST + "/" + "api?"
 
@@ -42,11 +40,11 @@ def sab_api_call(request_type=None, params={}, **kwargs):
         params['ma_password'] = headphones.CONFIG.SAB_PASSWORD
     if headphones.CONFIG.SAB_APIKEY:
         params['apikey'] = headphones.CONFIG.SAB_APIKEY
-    
-    if request_type=='send_nzb' and headphones.CONFIG.SAB_CATEGORY:
+
+    if request_type == 'send_nzb' and headphones.CONFIG.SAB_CATEGORY:
         params['cat'] = headphones.CONFIG.SAB_CATEGORY
 
-    params['output']='json'
+    params['output'] = 'json'
 
     response = request.request_json(url, params=params, **kwargs)
 
@@ -57,8 +55,8 @@ def sab_api_call(request_type=None, params={}, **kwargs):
         logger.debug("Successfully connected to SABnzbd on url: %s" % headphones.CONFIG.SAB_HOST)
         return response
 
-def sendNZB(nzb):
 
+def sendNZB(nzb):
     params = {}
     # if it's a normal result we just pass SAB the URL
     if nzb.resultType == "nzb":
@@ -87,7 +85,8 @@ def sendNZB(nzb):
         response = sab_api_call('send_nzb', params=params)
     elif nzb.resultType == "nzbdata":
         cookies = cookielib.CookieJar()
-        response = sab_api_call('send_nzb', params=params, method="post", files=files, cookies=cookies, headers=headers)
+        response = sab_api_call('send_nzb', params=params, method="post", files=files,
+                                cookies=cookies, headers=headers)
 
     if not response:
         logger.info(u"No data returned from SABnzbd, NZB not sent")
@@ -102,15 +101,15 @@ def sendNZB(nzb):
 
 
 def checkConfig():
-
     params = {'mode': 'get_config',
-               'section': 'misc',
-               }
+              'section': 'misc',
+              }
 
     config_options = sab_api_call(params=params)
-    
+
     if not config_options:
-        logger.warn("Unable to read SABnzbd config file - cannot determine renaming options (might affect auto & forced post processing)")
+        logger.warn(
+            "Unable to read SABnzbd config file - cannot determine renaming options (might affect auto & forced post processing)")
         return (0, 0)
 
     replace_spaces = config_options['config']['misc']['replace_spaces']

headphones/searcher_rutracker.py

@@ -1,349 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-
-# Headphones rutracker.org search
-# Functions called from searcher.py
-
-from bencode import bencode as bencode, bdecode
-from urlparse import urlparse
-from bs4 import BeautifulSoup
-from tempfile import mkdtemp
-from hashlib import sha1
-
-import headphones
-import requests
-import cookielib
-import urllib2
-import urllib
-import re
-import os
-
-from headphones import db, logger
-
-
-class Rutracker():
-
-    logged_in = False
-
-    # Stores a number of login attempts to prevent recursion.
-    #login_counter = 0
-
-    def __init__(self):
-
-        self.cookiejar = cookielib.CookieJar()
-        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookiejar))
-        urllib2.install_opener(self.opener)
-
-    def login(self, login, password):
-        """Implements tracker login procedure."""
-
-        self.logged_in = False
-
-        if login is None or password is None:
-            return False
-
-        #self.login_counter += 1
-
-        # No recursion wanted.
-        #if self.login_counter > 1:
-        #    return False
-
-        params = urllib.urlencode({"login_username": login,
-                                   "login_password": password,
-                                   "login": "Вход"})
-
-        try:
-            self.opener.open("http://login.rutracker.org/forum/login.php", params)
-        except Exception:
-            pass
-
-        # Check if we're logged in
-        for cookie in self.cookiejar:
-            if cookie.name == 'bb_data':
-                self.logged_in = True
-
-        return self.logged_in
-
-    def searchurl(self, artist, album, year, format):
-        """
-        Return the search url
-        """
-
-        # Build search url
-        searchterm = ''
-        if artist != 'Various Artists':
-            searchterm = artist
-            searchterm = searchterm + ' '
-        searchterm = searchterm + album
-        searchterm = searchterm + ' '
-        searchterm = searchterm + year
-
-        providerurl = "http://rutracker.org/forum/tracker.php"
-
-        if format == 'lossless':
-            format = '+lossless'
-        elif format == 'lossless+mp3':
-            format = '+lossless||mp3||aac'
-        else:
-            format = '+mp3||aac'
-
-        # sort by size, descending.
-        sort = '&o=7&s=2'
-
-        searchurl = "%s?nm=%s%s%s" % (providerurl, urllib.quote(searchterm), format, sort)
-
-        return searchurl
-
-    def search(self, searchurl, maxsize, minseeders, albumid):
-        """
-        Parse the search results and return valid torrent list
-        """
-
-        titles = []
-        urls = []
-        seeders = []
-        sizes = []
-        torrentlist = []
-        rulist = []
-
-        try:
-
-            page = self.opener.open(searchurl, timeout=60)
-            soup = BeautifulSoup(page.read())
-
-            # Debug
-            #logger.debug (soup.prettify())
-
-            # Title
-            for link in soup.find_all('a', attrs={'class': 'med tLink hl-tags bold'}):
-                title = link.get_text()
-                titles.append(title)
-
-            # Download URL
-            for link in soup.find_all('a', attrs={'class': 'small tr-dl dl-stub'}):
-                url = link.get('href')
-                urls.append(url)
-
-            # Seeders
-            for link in soup.find_all('b', attrs={'class': 'seedmed'}):
-                seeder = link.get_text()
-                seeders.append(seeder)
-
-            # Size
-            for link in soup.find_all('td', attrs={'class': 'row4 small nowrap tor-size'}):
-                size = link.u.string
-                sizes.append(size)
-
-        except:
-            pass
-
-        # Combine lists
-        torrentlist = zip(titles, urls, seeders, sizes)
-
-        # return if nothing found
-        if not torrentlist:
-            return False
-
-        # don't bother checking track counts anymore, let searcher filter instead
-        # leave code in just in case
-        check_track_count = False
-
-        if check_track_count:
-
-            # get headphones track count for album, return if not found
-            myDB = db.DBConnection()
-            tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
-            hptrackcount = len(tracks)
-
-            if not hptrackcount:
-                logger.info('headphones track info not found, cannot compare to torrent')
-                return False
-
-            # Return all valid entries, ignored, required words now checked in searcher.py
-
-            #unwantedlist = ['promo', 'vinyl', '[lp]', 'songbook', 'tvrip', 'hdtv', 'dvd']
-
-            formatlist = ['ape', 'flac', 'ogg', 'm4a', 'aac', 'mp3', 'wav', 'aif']
-            deluxelist = ['deluxe', 'edition', 'japanese', 'exclusive']
-
-        for torrent in torrentlist:
-
-            returntitle = torrent[0].encode('utf-8')
-            url = torrent[1]
-            seeders = torrent[2]
-            size = torrent[3]
-
-            if int(size) <= maxsize and int(seeders) >= minseeders:
-
-                #Torrent topic page
-                torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
-                topicurl = 'http://rutracker.org/forum/viewtopic.php?t=' + torrent_id
-
-                # add to list
-                if not check_track_count:
-                    valid = True
-                else:
-
-                    # Check torrent info
-                    self.cookiejar.set_cookie(cookielib.Cookie(version=0, name='bb_dl', value=torrent_id, port=None, port_specified=False, domain='.rutracker.org', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False))
-
-                    # Debug
-                    #for cookie in self.cookiejar:
-                    #    logger.debug ('Cookie: %s' % cookie)
-
-                    try:
-                        page = self.opener.open(url)
-                        torrent = page.read()
-                        if torrent:
-                            decoded = bdecode(torrent)
-                            metainfo = decoded['info']
-                        page.close()
-                    except Exception as e:
-                        logger.error('Error getting torrent: %s' % e)
-                        return False
-
-                    # get torrent track count and check for cue
-                    trackcount = 0
-                    cuecount = 0
-
-                    if 'files' in metainfo: # multi
-                        for pathfile in metainfo['files']:
-                            path = pathfile['path']
-                            for file in path:
-                                if any(file.lower().endswith('.' + x.lower()) for x in formatlist):
-                                    trackcount += 1
-                                if '.cue' in file:
-                                    cuecount += 1
-
-                    title = returntitle.lower()
-                    logger.debug('torrent title: %s' % title)
-                    logger.debug('headphones trackcount: %s' % hptrackcount)
-                    logger.debug('rutracker trackcount: %s' % trackcount)
-
-                    # If torrent track count less than headphones track count, and there's a cue, then attempt to get track count from log(s)
-                    # This is for the case where we have a single .flac/.wav which can be split by cue
-                    # Not great, but shouldn't be doing this too often
-                    totallogcount = 0
-                    if trackcount < hptrackcount and cuecount > 0 and cuecount < hptrackcount:
-                        page = self.opener.open(topicurl, timeout=60)
-                        soup = BeautifulSoup(page.read())
-                        findtoc = soup.find_all(text='TOC of the extracted CD')
-                        if not findtoc:
-                            findtoc = soup.find_all(text='TOC извлечённого CD')
-                        for toc in findtoc:
-                            logcount = 0
-                            for toccontent in toc.find_all_next(text=True):
-                                cut_string = toccontent.split('|')
-                                new_string = cut_string[0].lstrip().rstrip()
-                                if new_string == '1' or new_string == '01':
-                                    logcount = 1
-                                elif logcount > 0:
-                                    if new_string.isdigit():
-                                        logcount += 1
-                                    else:
-                                        break
-                            totallogcount = totallogcount + logcount
-
-                    if totallogcount > 0:
-                        trackcount = totallogcount
-                        logger.debug('rutracker logtrackcount: %s' % totallogcount)
-
-                    # If torrent track count = hp track count then return torrent,
-                    # if greater, check for deluxe/special/foreign editions
-                    # if less, then allow if it's a single track with a cue
-                    valid = False
-
-                    if trackcount == hptrackcount:
-                        valid = True
-                    elif trackcount > hptrackcount:
-                        if any(deluxe in title for deluxe in deluxelist):
-                            valid = True
-
-                # Add to list
-                if valid:
-                    rulist.append((returntitle, size, topicurl))
-                else:
-                    if topicurl:
-                        logger.info(u'<a href="%s">Torrent</a> found with %s tracks but the selected headphones release has %s tracks, skipping for rutracker.org' % (topicurl, trackcount, hptrackcount))
-            else:
-                logger.info('%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i)' % (returntitle, int(size), int(seeders)))
-
-        return rulist
-
-    def get_torrent(self, url, savelocation=None):
-
-        torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
-        self.cookiejar.set_cookie(cookielib.Cookie(version=0, name='bb_dl', value=torrent_id, port=None, port_specified=False, domain='.rutracker.org', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False))
-        downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id
-        torrent_name = torrent_id + '.torrent'
-
-        try:
-            prev = os.umask(headphones.UMASK)
-            page = self.opener.open(downloadurl)
-            torrent = page.read()
-            decoded = bdecode(torrent)
-            metainfo = decoded['info']
-            tor_hash = sha1(bencode(metainfo)).hexdigest()
-            if savelocation:
-                download_path = os.path.join(savelocation, torrent_name)
-            else:
-                tempdir = mkdtemp(suffix='_rutracker_torrents')
-                download_path = os.path.join(tempdir, torrent_name)
-
-            with open(download_path, 'wb') as f:
-                f.write(torrent)
-            os.umask(prev)
-
-            # Add file to utorrent
-            if headphones.CONFIG.TORRENT_DOWNLOADER == 2:
-                self.utorrent_add_file(download_path)
-
-        except Exception as e:
-            logger.error('Error getting torrent: %s', e)
-            return False
-
-        return download_path, tor_hash
-
-    #TODO get this working in utorrent.py
-    def utorrent_add_file(self, filename):
-
-        host = headphones.CONFIG.UTORRENT_HOST
-        if not host.startswith('http'):
-            host = 'http://' + host
-        if host.endswith('/'):
-            host = host[:-1]
-        if host.endswith('/gui'):
-            host = host[:-4]
-
-        base_url = host
-        username = headphones.CONFIG.UTORRENT_USERNAME
-        password = headphones.CONFIG.UTORRENT_PASSWORD
-
-        session = requests.Session()
-        url = base_url + '/gui/'
-        session.auth = (username, password)
-
-        try:
-            r = session.get(url + 'token.html')
-        except Exception:
-            logger.exception('Error getting token')
-            return
-
-        if r.status_code == '401':
-            logger.debug('Error reaching utorrent')
-            return
-
-        regex = re.search(r'.+>([^<]+)</div></html>', r.text)
-        if regex is None:
-            logger.debug('Error reading token')
-            return
-
-        session.params = {'token': regex.group(1)}
-
-        with open(filename, 'rb') as f:
-            try:
-                session.post(url, params={'action': 'add-file'},
-                    files={'torrent_file': f})
-            except Exception:
-                logger.exception('Error adding file to utorrent')
-                return

headphones/softchroot.py

@@ -0,0 +1,71 @@
+import os
+from headphones.exceptions import SoftChrootError
+
+
+class SoftChroot(object):
+    """ SoftChroot provides SOFT chrooting for UI
+
+    IMPORTANT: call methods of this class just in modules, which generates data for client UI. Try to avoid unnecessary usage.
+    """
+
+    enabled = False
+    chroot = None
+
+    def __init__(self, path):
+        if not path:
+            #disabled
+            return
+
+        path = path.strip()
+        if not path:
+            return
+
+        if (not os.path.exists(path) or
+                not os.path.isdir(path)):
+            raise SoftChrootError('No such directory: %s' % path)
+
+        path = path.rstrip(os.path.sep) + os.path.sep
+
+        self.enabled = True
+        self.chroot = path
+
+    def isEnabled(self):
+        return self.enabled
+
+    def getRoot(self):
+        return self.chroot
+
+    def apply(self, path):
+        if not self.enabled:
+            return path
+
+        if not path:
+            return path
+
+        p = path.strip()
+        if not p:
+            return path
+
+        if path.startswith(self.chroot):
+            p = os.path.sep + path[len(self.chroot):]
+        else:
+            p = os.path.sep
+
+        return p
+
+    def revoke(self, path):
+        if not self.enabled:
+            return path
+
+        if not path:
+            return path
+
+        p = path.strip()
+        if not p:
+            return path
+
+        if os.path.sep == p[0]:
+            p = p[1:]
+
+        p = self.chroot + p
+        return p

headphones/softchroot_test.py

@@ -0,0 +1,122 @@
+import os
+import mock
+from headphones.unittestcompat import TestCase, TestArgs
+#from mock import MagicMock
+
+from headphones.softchroot import SoftChroot
+from headphones.exceptions import SoftChrootError
+
+
+class SoftChrootTest(TestCase):
+    def test_create(self):
+        """ create headphones.SoftChroot """
+
+        cf = SoftChroot('/tmp/')
+        self.assertIsInstance(cf, SoftChroot)
+        self.assertTrue(cf.isEnabled())
+        self.assertEqual(cf.getRoot(), '/tmp/')
+
+    @TestArgs(
+        (None),
+        (''),
+        ('      '),
+    )
+    def test_create_disabled(self, empty_path):
+        """ create DISABLED SoftChroot """
+
+        cf = SoftChroot(empty_path)
+        self.assertIsInstance(cf, SoftChroot)
+        self.assertFalse(cf.isEnabled())
+        self.assertIsNone(cf.getRoot())
+
+    def test_create_on_not_exists_dir(self):
+        """ create SoftChroot on non existent dir """
+
+        path = os.path.join('/tmp', 'notexist', 'asdf', '11', '12', 'np', 'itsssss')
+
+        cf = None
+        with self.assertRaises(SoftChrootError) as exc:
+            cf = SoftChroot(path)
+        self.assertIsNone(cf)
+
+        self.assertRegexpMatches(str(exc.exception), r'No such directory')
+        self.assertRegexpMatches(str(exc.exception), path)
+
+    @mock.patch('headphones.softchroot.os', wrap=os, name='OsMock')
+    def test_create_on_file(self, os_mock):
+        """ create SoftChroot on file, not a directory """
+
+        path = os.path.join('/tmp', 'notexist', 'asdf', '11', '12', 'np', 'itsssss')
+
+        os_mock.path.sep = os.path.sep
+        os_mock.path.isdir.side_effect = lambda x: x != path
+
+        cf = None
+        with self.assertRaises(SoftChrootError) as exc:
+            cf = SoftChroot(path)
+        self.assertIsNone(cf)
+
+        self.assertTrue(os_mock.path.isdir.called)
+
+        self.assertRegexpMatches(str(exc.exception), r'No such directory')
+        self.assertRegexpMatches(str(exc.exception), path)
+
+    @TestArgs(
+        (None, None),
+        ('', ''),
+        ('      ', '      '),
+        ('/tmp/', '/'),
+        ('/tmp/asdf', '/asdf'),
+    )
+    def test_apply(self, p, e):
+        """ apply SoftChroot """
+        sc = SoftChroot('/tmp/')
+        a = sc.apply(p)
+        self.assertEqual(a, e)
+
+    @TestArgs(
+        ('/'),
+        ('/nonch/path/asdf'),
+        ('tmp/asdf'),
+    )
+    def test_apply_out_of_root(self, p):
+        """ apply SoftChroot to paths outside of the chroot """
+        sc = SoftChroot('/tmp/')
+        a = sc.apply(p)
+        self.assertEqual(a, '/')
+
+    @TestArgs(
+        (None, None),
+        ('', ''),
+        ('      ', '      '),
+        ('/', '/tmp/'),
+        ('/asdf', '/tmp/asdf'),
+        ('/asdf/', '/tmp/asdf/'),
+        ('localdir/adf', '/tmp/localdir/adf'),
+        ('localdir/adf/', '/tmp/localdir/adf/'),
+    )
+    def test_revoke(self, p, e):
+        """ revoke SoftChroot """
+        sc = SoftChroot('/tmp/')
+        a = sc.revoke(p)
+        self.assertEqual(a, e)
+
+    @TestArgs(
+        (None),
+        (''),
+        ('     '),
+        ('/tmp'),
+        ('/tmp/'),
+        ('/tmp/asdf'),
+        ('/tmp/localdir/adf'),
+        ('localdir/adf'),
+        ('localdir/adf/'),
+    )
+    def test_actions_on_disabled(self, p):
+        """ disabled SoftChroot should not change args on apply and revoke """
+        sc = SoftChroot(None)
+        a = sc.apply(p)
+        self.assertEqual(a, p)
+
+        r = sc.revoke(p)
+        self.assertEqual(r, p)

headphones/torrentfinished.py

@@ -13,13 +13,14 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import db, utorrent, transmission, logger
-
 import threading
+
+from headphones import db, utorrent, transmission, logger
 import headphones
 
 postprocessor_lock = threading.Lock()
 
+
 def checkTorrentFinished():
     """
     Remove Torrent + data if Post Processed and finished Seeding
@@ -41,6 +42,7 @@ def checkTorrentFinished():
                 torrent_removed = utorrent.removeTorrent(hash, True)
 
             if torrent_removed:
-                myDB.action('DELETE from snatched WHERE status = "Seed_Processed" and AlbumID=?', [albumid])
+                myDB.action('DELETE from snatched WHERE status = "Seed_Processed" and AlbumID=?',
+                            [albumid])
 
     logger.info("Checking finished torrents completed")

headphones/transmission.py

@@ -13,27 +13,32 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import logger, request
-
 import time
 import json
 import base64
 import urlparse
+
+from headphones import logger, request
 import headphones
 
+
 # This is just a simple script to send torrents to transmission. The
 # intention is to turn this into a class where we can check the state
 # of the download, set the download dir, etc.
-# TODO: Store the session id so we don't need to make 2 calls
-#       Store torrent id so we can check up on it
+# TODO: Store torrent id so we can check up on it
+
+_session_id = None
 
 
-def addTorrent(link):
+def addTorrent(link, data=None):
     method = 'torrent-add'
 
-    if link.endswith('.torrent'):
-        with open(link, 'rb') as f:
-            metainfo = str(base64.b64encode(f.read()))
+    if link.endswith('.torrent') and not link.startswith('http') or data:
+        if data:
+            metainfo = str(base64.b64encode(data))
+        else:
+            with open(link, 'rb') as f:
+                metainfo = str(base64.b64encode(f.read()))
         arguments = {'metainfo': metainfo, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
     else:
         arguments = {'filename': link, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
@@ -93,7 +98,6 @@ def setSeedRatio(torrentid, ratio):
 
 
 def removeTorrent(torrentid, remove_data=False):
-
     method = 'torrent-get'
     arguments = {'ids': torrentid, 'fields': ['isFinished', 'name']}
 
@@ -115,7 +119,8 @@ def removeTorrent(torrentid, remove_data=False):
             response = torrentAction(method, arguments)
             return True
         else:
-            logger.info('%s has not finished seeding yet, torrent will not be removed, will try again on next run' % name)
+            logger.info(
+                '%s has not finished seeding yet, torrent will not be removed, will try again on next run' % name)
     except:
         return False
 
@@ -123,7 +128,7 @@ def removeTorrent(torrentid, remove_data=False):
 
 
 def torrentAction(method, arguments):
-
+    global _session_id
     host = headphones.CONFIG.TRANSMISSION_HOST
     username = headphones.CONFIG.TRANSMISSION_USERNAME
     password = headphones.CONFIG.TRANSMISSION_PASSWORD
@@ -145,43 +150,34 @@ def torrentAction(method, arguments):
         parts[2] += "/transmission/rpc"
 
     host = urlparse.urlunparse(parts)
-
-    # Retrieve session id
-    auth = (username, password) if username and password else None
-    response = request.request_response(host, auth=auth,
-        whitelist_status_code=[401, 409])
-
-    if response is None:
-        logger.error("Error gettings Transmission session ID")
-        return
-
-    # Parse response
-    if response.status_code == 401:
-        if auth:
-            logger.error("Username and/or password not accepted by " \
-                "Transmission")
-        else:
-            logger.error("Transmission authorization required")
-
-        return
-    elif response.status_code == 409:
-        session_id = response.headers['x-transmission-session-id']
-
-        if not session_id:
-            logger.error("Expected a Session ID from Transmission")
-            return
-
-    # Prepare next request
-    headers = {'x-transmission-session-id': session_id}
     data = {'method': method, 'arguments': arguments}
+    data_json = json.dumps(data)
+    auth = (username, password) if username and password else None
+    for retry in range(2):
+        if _session_id is not None:
+            headers = {'x-transmission-session-id': _session_id}
+            response = request.request_response(host, method="POST",
+                data=data_json, headers=headers, auth=auth,
+                whitelist_status_code=[200, 401, 409])
+        else:
+            response = request.request_response(host, auth=auth,
+                                            whitelist_status_code=[401, 409])
+        if response.status_code == 401:
+            if auth:
+                logger.error("Username and/or password not accepted by " \
+                            "Transmission")
+            else:
+                logger.error("Transmission authorization required")
+            return
+        elif response.status_code == 409:
+            _session_id = response.headers['x-transmission-session-id']
+            if _session_id is None:
+                logger.error("Expected a Session ID from Transmission, got None")
+                return
+            # retry request with new session id
+            logger.debug("Retrying Transmission request with new session id")
+            continue
 
-    response = request.request_json(host, method="POST", data=json.dumps(data),
-        headers=headers, auth=auth)
-
-    print response
-
-    if not response:
-        logger.error("Error sending torrent to Transmission")
-        return
-
-    return response
+        resp_json = response.json()
+        print resp_json
+        return resp_json

headphones/unittestcompat.py

@@ -0,0 +1,115 @@
+import sys
+if sys.version_info < (2, 7):
+    import unittest2 as unittest
+    from unittest2 import TestCase as TC
+else:
+    import unittest
+    from unittest import TestCase as TC
+
+skip = unittest.skip
+
+_dummy = False
+
+# less than 2.6 ...
+if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
+    _dummy = True
+
+
+def _d(f):
+    def decorate(self, *args, **kw):
+        if not _dummy:
+            return f(self, *args, **kw)
+        return self.assertTrue(True)
+    return decorate
+
+
+class TestCase(TC):
+    """
+    Wrapper for python 2.6 stubs
+    """
+
+    def assertIsInstance(self, obj, cls, msg=None):
+        if not _dummy:
+            return super(TestCase, self).assertIsInstance(obj, cls, msg)
+        tst = isinstance(obj, cls)
+        return self.assertTrue(tst, msg)
+
+    @_d
+    def assertNotIsInstance(self, *args, **kw):
+        return super(TestCase, self).assertNotIsInstance(*args, **kw)
+
+    @_d
+    def assertIn(self, *args, **kw):
+        return super(TestCase, self).assertIn(*args, **kw)
+
+    @_d
+    def assertRegexpMatches(self, *args, **kw):
+        return super(TestCase, self).assertRegexpMatches(*args, **kw)
+
+    # -----------------------------------------------------------
+    # NOT DUMMY ASSERTIONS
+    # -----------------------------------------------------------
+    def assertIsNone(self, val, msg=None):
+        if not _dummy:
+            return super(TestCase, self).assertIsNone(val, msg)
+        tst = val is None
+        return super(TestCase, self).assertTrue(tst, msg)
+
+    def assertIsNotNone(self, val, msg=None):
+        if not _dummy:
+            return super(TestCase, self).assertIsNotNone(val, msg)
+        tst = val is not None
+        return super(TestCase, self).assertTrue(tst, msg)
+
+    def assertRaises(self, exc, msg=None):
+        if not _dummy:
+            return super(TestCase, self).assertRaises(exc, msg)
+        return TestCase._TestCaseRaiseStub(self, exc, msg=msg)
+
+    def assertRaisesRegexp(self, exc, regex, msg=None):
+        if not _dummy:
+            return super(TestCase, self).assertRaises(exc, msg)
+        return TestCase._TestCaseRaiseStub(self, exc, regex=regex, msg=msg)
+
+    class _TestCaseRaiseStub:
+        """ Internal stuff for stubbing `assertRaises*` """
+
+        def __init__(self, test_case, exc, regex=None, msg=None):
+            self.exc = exc
+            self.test_case = test_case
+            self.regex = regex
+            self.msg = msg
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, tp, value, traceback):
+            tst = tp is self.exc
+            self.test_case.assertTrue(tst, msg=self.msg)
+            self.exception = value
+
+            # TODO: implement self.regex checking
+
+            # True indicates, that exception is handled
+            return True
+
+
+def TestArgs(*parameters):
+    def tuplify(x):
+        if not isinstance(x, tuple):
+            return (x,)
+        return x
+
+    def decorator(method, parameters=parameters):
+        for parameter in (tuplify(x) for x in parameters):
+
+            def method_for_parameter(self, method=method, parameter=parameter):
+                method(self, *parameter)
+            args_for_parameter = ",".join(repr(v) for v in parameter)
+            name_for_parameter = method.__name__ + "(" + args_for_parameter + ")"
+            frame = sys._getframe(1)    # pylint: disable-msg=W0212
+            frame.f_locals[name_for_parameter] = method_for_parameter
+            frame.f_locals[name_for_parameter].__doc__ = method.__doc__ + '(' + args_for_parameter + ')'
+            method_for_parameter.__name__ = name_for_parameter + '(' + args_for_parameter + ')'
+        return None
+    return decorator

headphones/updater.py

@@ -17,10 +17,10 @@ from headphones import logger, db, importer
 
 
 def dbUpdate(forcefull=False):
-
     myDB = db.DBConnection()
 
-    active_artists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
+    active_artists = myDB.select(
+        'SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
     logger.info('Starting update for %i active artists', len(active_artists))
 
     for artist in active_artists:

headphones/utorrent.py

@@ -14,26 +14,24 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import urllib
+import json
+import time
+from collections import namedtuple
 import urllib2
 import urlparse
 import cookielib
-import json
+
 import re
 import os
-import time
-
 import headphones
-
 from headphones import logger
-from collections import namedtuple
 
 
 class utorrentclient(object):
-
     TOKEN_REGEX = "<div id='token' style='display:none;'>([^<>]+)</div>"
     UTSetting = namedtuple("UTSetting", ["name", "int", "str", "access"])
 
-    def __init__(self, base_url=None, username=None, password=None,):
+    def __init__(self, base_url=None, username=None, password=None, ):
 
         host = headphones.CONFIG.UTORRENT_HOST
         if not host.startswith('http'):
@@ -50,7 +48,7 @@ class utorrentclient(object):
         self.password = headphones.CONFIG.UTORRENT_PASSWORD
         self.opener = self._make_opener('uTorrent', self.base_url, self.username, self.password)
         self.token = self._get_token()
-        #TODO refresh token, when necessary
+        # TODO refresh token, when necessary
 
     def _make_opener(self, realm, base_url, username, password):
         """uTorrent API need HTTP Basic Auth and cookie support for token verify."""
@@ -83,7 +81,7 @@ class utorrentclient(object):
         return self._action(params)
 
     def add_url(self, url):
-        #can receive magnet or normal .torrent link
+        # can receive magnet or normal .torrent link
         params = [('action', 'add-url'), ('s', url)]
         return self._action(params)
 
@@ -181,13 +179,15 @@ def removeTorrent(hash, remove_data=False):
     status, torrentList = uTorrentClient.list()
     torrents = torrentList['torrents']
     for torrent in torrents:
-        if torrent[0].lower() == hash:
+        if torrent[0].upper() == hash.upper():
             if torrent[21] == 'Finished':
                 logger.info('%s has finished seeding, removing torrent and data' % torrent[2])
                 uTorrentClient.remove(hash, remove_data)
                 return True
             else:
-                logger.info('%s has not finished seeding yet, torrent will not be removed, will try again on next run' % torrent[2])
+                logger.info(
+                    '%s has not finished seeding yet, torrent will not be removed, will try again on next run' %
+                    torrent[2])
                 return False
     return False
 
@@ -203,7 +203,6 @@ def setSeedRatio(hash, ratio):
 
 
 def dirTorrent(hash, cacheid=None, return_name=None):
-
     uTorrentClient = utorrentclient()
 
     if not cacheid:
@@ -220,7 +219,7 @@ def dirTorrent(hash, cacheid=None, return_name=None):
     cacheid = torrentList['torrentc']
 
     for torrent in torrents:
-        if torrent[0].upper() == hash:
+        if torrent[0].upper() == hash.upper():
             if not return_name:
                 return torrent[26], cacheid
             else:
@@ -229,18 +228,21 @@ def dirTorrent(hash, cacheid=None, return_name=None):
     return None, None
 
 
-def addTorrent(link, hash):
+def addTorrent(link):
     uTorrentClient = utorrentclient()
+    uTorrentClient.add_url(link)
+
+
+def getFolder(hash):
 
     # Get Active Directory from settings
     active_dir, completed_dir = getSettingsDirectories()
 
     if not active_dir:
-        logger.error('Could not get "Put new downloads in:" directory from uTorrent settings, please ensure it is set')
+        logger.error(
+            'Could not get "Put new downloads in:" directory from uTorrent settings, please ensure it is set')
         return None
 
-    uTorrentClient.add_url(link)
-
     # Get Torrent Folder Name
     torrent_folder, cacheid = dirTorrent(hash)
 
@@ -254,10 +256,8 @@ def addTorrent(link, hash):
 
     if torrent_folder == active_dir or not torrent_folder:
         torrent_folder, cacheid = dirTorrent(hash, cacheid, return_name=True)
-        labelTorrent(hash)
         return torrent_folder
     else:
-        labelTorrent(hash)
         if headphones.SYS_PLATFORM != "win32":
             torrent_folder = torrent_folder.replace('\\', '/')
         return os.path.basename(os.path.normpath(torrent_folder))

headphones/versioncheck.py

@@ -13,18 +13,17 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
-import os
 import tarfile
 import platform
-import headphones
 import subprocess
 
+import re
+import os
+import headphones
 from headphones import logger, version, request
 
 
 def runGit(args):
-
     if headphones.CONFIG.GIT_PATH:
         git_locations = ['"' + headphones.CONFIG.GIT_PATH + '"']
     else:
@@ -40,7 +39,8 @@ def runGit(args):
 
         try:
             logger.debug('Trying to execute: "' + cmd + '" with shell in ' + headphones.PROG_DIR)
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=headphones.PROG_DIR)
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True,
+                                 cwd=headphones.PROG_DIR)
             output, err = p.communicate()
             output = output.strip()
 
@@ -62,7 +62,6 @@ def runGit(args):
 
 
 def getVersion():
-
     if version.HEADPHONES_VERSION.startswith('win32build'):
         headphones.INSTALL_TYPE = 'win'
 
@@ -92,7 +91,8 @@ def getVersion():
             branch_name = branch_name
 
             if not branch_name and headphones.CONFIG.GIT_BRANCH:
-                logger.error('Could not retrieve branch name from git. Falling back to %s' % headphones.CONFIG.GIT_BRANCH)
+                logger.error(
+                    'Could not retrieve branch name from git. Falling back to %s' % headphones.CONFIG.GIT_BRANCH)
                 branch_name = headphones.CONFIG.GIT_BRANCH
             if not branch_name:
                 logger.error('Could not retrieve branch name from git. Defaulting to master')
@@ -123,11 +123,13 @@ def checkGithub():
 
     # Get the latest version available from github
     logger.info('Retrieving latest version information from GitHub')
-    url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH)
+    url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (
+    headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH)
     version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict)
 
     if version is None:
-        logger.warn('Could not get the latest version from GitHub. Are you running a local development version?')
+        logger.warn(
+            'Could not get the latest version from GitHub. Are you running a local development version?')
         return headphones.CURRENT_VERSION
 
     headphones.LATEST_VERSION = version['sha']
@@ -135,7 +137,8 @@ def checkGithub():
 
     # See how many commits behind we are
     if not headphones.CURRENT_VERSION:
-        logger.info('You are running an unknown version of Headphones. Run the updater to identify your version')
+        logger.info(
+            'You are running an unknown version of Headphones. Run the updater to identify your version')
         return headphones.LATEST_VERSION
 
     if headphones.LATEST_VERSION == headphones.CURRENT_VERSION:
@@ -143,8 +146,10 @@ def checkGithub():
         return headphones.LATEST_VERSION
 
     logger.info('Comparing currently installed version with latest GitHub version')
-    url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.CONFIG.GIT_USER, headphones.LATEST_VERSION, headphones.CURRENT_VERSION)
-    commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict)
+    url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (
+    headphones.CONFIG.GIT_USER, headphones.LATEST_VERSION, headphones.CURRENT_VERSION)
+    commits = request.request_json(url, timeout=20, whitelist_status_code=404,
+                                   validator=lambda x: type(x) == dict)
 
     if commits is None:
         logger.warn('Could not get commits behind from GitHub.')
@@ -158,7 +163,8 @@ def checkGithub():
         headphones.COMMITS_BEHIND = 0
 
     if headphones.COMMITS_BEHIND > 0:
-        logger.info('New version is available. You are %s commits behind' % headphones.COMMITS_BEHIND)
+        logger.info(
+            'New version is available. You are %s commits behind' % headphones.COMMITS_BEHIND)
     elif headphones.COMMITS_BEHIND == 0:
         logger.info('Headphones is up to date')
 
@@ -185,7 +191,8 @@ def update():
                 logger.info('Output: ' + str(output))
 
     else:
-        tar_download_url = 'https://github.com/%s/headphones/tarball/%s' % (headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH)
+        tar_download_url = 'https://github.com/%s/headphones/tarball/%s' % (
+        headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH)
         update_dir = os.path.join(headphones.PROG_DIR, 'update')
         version_path = os.path.join(headphones.PROG_DIR, 'version.txt')
 
@@ -214,7 +221,8 @@ def update():
         os.remove(tar_download_path)
 
         # Find update dir name
-        update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))]
+        update_dir_contents = [x for x in os.listdir(update_dir) if
+                               os.path.isdir(os.path.join(update_dir, x))]
         if len(update_dir_contents) != 1:
             logger.error("Invalid update data, update failed: " + str(update_dir_contents))
             return

headphones/webstart.py

@@ -13,18 +13,17 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import sys
+
+import os
 import cherrypy
 import headphones
-
 from headphones import logger
 from headphones.webserve import WebInterface
 from headphones.helpers import create_https_certificates
 
 
 def initialize(options):
-
     # HTTPS stuff stolen from sickbeard
     enable_https = options['enable_https']
     https_cert = options['https_cert']
@@ -33,15 +32,16 @@ def initialize(options):
     if enable_https:
         # If either the HTTPS certificate or key do not exist, try to make
         # self-signed ones.
-        if not (https_cert and os.path.exists(https_cert)) or not (https_key and os.path.exists(https_key)):
+        if not (https_cert and os.path.exists(https_cert)) or not (
+            https_key and os.path.exists(https_key)):
             if not create_https_certificates(https_cert, https_key):
                 logger.warn("Unable to create certificate and key. Disabling " \
-                    "HTTPS")
+                            "HTTPS")
                 enable_https = False
 
         if not (os.path.exists(https_cert) and os.path.exists(https_key)):
             logger.warn("Disabled HTTPS because of missing certificate and " \
-                "key.")
+                        "key.")
             enable_https = False
 
     options_dict = {
@@ -63,7 +63,7 @@ def initialize(options):
         protocol = "http"
 
     logger.info("Starting Headphones web server on %s://%s:%d/", protocol,
-        options['http_host'], options['http_port'])
+                options['http_host'], options['http_port'])
     cherrypy.config.update(options_dict)
 
     conf = {
@@ -99,7 +99,8 @@ def initialize(options):
     }
 
     if options['http_password']:
-        logger.info("Web server authentication is enabled, username is '%s'", options['http_username'])
+        logger.info("Web server authentication is enabled, username is '%s'",
+                    options['http_username'])
 
         conf['/'].update({
             'tools.auth_basic.on': True,
@@ -118,7 +119,8 @@ def initialize(options):
         cherrypy.process.servers.check_port(str(options['http_host']), options['http_port'])
         cherrypy.server.start()
     except IOError:
-        sys.stderr.write('Failed to start on port: %i. Is something else running?\n' % (options['http_port']))
+        sys.stderr.write(
+            'Failed to start on port: %i. Is something else running?\n' % (options['http_port']))
         sys.exit(1)
 
     cherrypy.server.wait()

init-scripts/init-alt.freebsd

@@ -27,17 +27,17 @@ rcvar=${name}_enable
 
 load_rc_config ${name}
 
-: ${headphones_enable:="NO"}
-: ${headphones_user:="_sabnzbd"}
-: ${headphones_dir:="/usr/local/headphones"}
-: ${headphones_chdir:="${headphones_dir}"}
-: ${headphones_pid:="${headphones_dir}/headphones.pid"}
-: ${headphones_conf:="${headphones_dir}/config.ini"}
+: "${headphones_enable:="NO"}"
+: "${headphones_user:="_sabnzbd"}"
+: "${headphones_dir:="/usr/local/headphones"}"
+: "${headphones_chdir:="${headphones_dir}"}"
+: "${headphones_pid:="${headphones_dir}/headphones.pid"}"
+: "${headphones_conf:="${headphones_dir}/config.ini"}"
 
 WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown Headphones.
 if [ -e "${headphones_conf}" ]; then
-        HOST=`grep -A64 "\[General\]" "${headphones_conf}"|egrep "^http_host"|perl -wple 's/^http_host = (.*)$/$1/'`
-        PORT=`grep -A64 "\[General\]" "${headphones_conf}"|egrep "^http_port"|perl -wple 's/^http_port = (.*)$/$1/'`
+        HOST=$(grep -A64 "\[General\]" "${headphones_conf}"|egrep "^http_host"|perl -wple 's/^http_host = (.*)$/$1/')
+        PORT=$(grep -A64 "\[General\]" "${headphones_conf}"|egrep "^http_port"|perl -wple 's/^http_port = (.*)$/$1/')
 fi
 
 status_cmd="${name}_status"
@@ -53,15 +53,15 @@ if [ ! -x "${WGET}" ]; then
 fi
 
 # Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
+if [ "$(id -u)" != "0" ]; then
   echo "Oops, you should be root before running this!"
   exit 1
 fi
 
 verify_headphones_pid() {
     # Make sure the pid corresponds to the Headphones process.
-    pid=`cat ${headphones_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${headphones_dir}/Headphones.py"
+    pid=$(cat "${headphones_pid}" 2>/dev/null)
+    pgrep -F "${headphones_pid}" -q "python ${headphones_dir}/Headphones.py"
     return $?
 }
 
@@ -73,10 +73,10 @@ headphones_stop() {
     fi
     echo "Stopping $name"
     verify_headphones_pid
-    ${WGET} -O - -q --user=${SBUSR} --password=${SBPWD} "http://${HOST}:${PORT}/shutdown/" >/dev/null
+    ${WGET} -O - -q --user="${SBUSR}" --password="${SBPWD}" "http://${HOST}:${PORT}/shutdown/" >/dev/null
     
     if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
+      wait_for_pids "${pid}"
       echo "Stopped $name"
     fi
 }

init-scripts/init.freebsd

@@ -28,11 +28,11 @@ rcvar=${name}_enable
 
 load_rc_config ${name}
 
-: ${headphones_enable:="NO"}
-: ${headphones_user:="_sabnzbd"}
-: ${headphones_dir:="/usr/local/headphones"}
-: ${headphones_chdir:="${headphones_dir}"}
-: ${headphones_pid:="${headphones_dir}/headphones.pid"}
+: "${headphones_enable:="NO"}"
+: "${headphones_user:="_sabnzbd"}"
+: "${headphones_dir:="/usr/local/headphones"}"
+: "${headphones_chdir:="${headphones_dir}"}"
+: "${headphones_pid:="${headphones_dir}/headphones.pid"}"
 
 status_cmd="${name}_status"
 stop_cmd="${name}_stop"
@@ -41,15 +41,15 @@ command="/usr/sbin/daemon"
 command_args="-f -p ${headphones_pid} python ${headphones_dir}/Headphones.py ${headphones_flags} --quiet --nolaunch"
 
 # Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
+if [ "$(id -u)" != "0" ]; then
   echo "Oops, you should be root before running this!"
   exit 1
 fi
 
 verify_headphones_pid() {
     # Make sure the pid corresponds to the Headphones process.
-    pid=`cat ${headphones_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${headphones_dir}/Headphones.py"
+    pid=$(cat "${headphones_pid}" 2>/dev/null)
+    pgrep -F "${headphones_pid}" -q "python ${headphones_dir}/Headphones.py"
     return $?
 }
 
@@ -58,7 +58,7 @@ headphones_stop() {
     echo "Stopping $name"
     verify_headphones_pid
     if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
+      wait_for_pids "${pid}"
       echo "Stopped"
     fi
 }

init-scripts/init.ubuntu

@@ -32,7 +32,6 @@
 ## HP_PIDFILE=      #$PID_FILE, the location of headphones.pid, the default is /var/run/headphones/headphones.pid
 ## PYTHON_BIN=      #$DAEMON, the location of the python binary, the default is /usr/bin/python
 ## HP_OPTS=         #$EXTRA_DAEMON_OPTS, extra cli option for headphones, i.e. " --config=/home/headphones/config.ini"
-## SSD_OPTS=        #$EXTRA_SSD_OPTS, extra start-stop-daemon option like " --group=users"
 ## HP_PORT=         #$PORT_OPTS, hardcoded port for the webserver, overrides value in config.ini
 ##
 ## EXAMPLE if want to run as different user
@@ -101,9 +100,6 @@ load_settings() {
         # Extra daemon option like: HP_OPTS=" --config=/home/headphones/config.ini"
         EXTRA_DAEMON_OPTS=${HP_OPTS-}
 
-        # Extra start-stop-daemon option like START_OPTS=" --group=users"
-        EXTRA_SSD_OPTS=${SSD_OPTS-}
-
         # Hardcoded port to run on, overrides config.ini settings
         [ -n "$HP_PORT" ] && {
             PORT_OPTS=" --port=${HP_PORT} "
@@ -114,7 +110,7 @@ load_settings() {
         SETTINGS_LOADED=TRUE
     fi
 
-    [ -x $DAEMON ] || {
+    [ -x "$DAEMON" ] || {
         log_warning_msg "$DESC: Can't execute daemon, aborting. See $DAEMON";
         return 1;}
 
@@ -125,8 +121,8 @@ load_settings || exit 0
 
 is_running () {
     # returns 1 when running, else 0.
-    if [ -e $PID_FILE ]; then
-      PID=`cat $PID_FILE`
+    if [ -e "$PID_FILE" ]; then
+      PID=$(cat "$PID_FILE")
 
       RET=$?
       [ $RET -gt 1 ] && exit 1 || return $RET
@@ -136,28 +132,28 @@ is_running () {
 }
 
 handle_pid () {
-    PID_PATH=`dirname $PID_FILE`
-    [ -d $PID_PATH ] || mkdir -p $PID_PATH && chown -R $RUN_AS $PID_PATH > /dev/null || {
+    PID_PATH=$(dirname "$PID_FILE")
+    [ -d "$PID_PATH" ] || mkdir -p "$PID_PATH" && chown -R "$RUN_AS" "$PID_PATH" > /dev/null || {
         log_warning_msg "$DESC: Could not create $PID_FILE, See $SETTINGS, aborting.";
         return 1;}
 
-    if [ -e $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        if ! kill -0 $PID > /dev/null 2>&1; then
+    if [ -e "$PID_FILE" ]; then
+        PID=$(cat "$PID_FILE")
+        if ! kill -0 "$PID" > /dev/null 2>&1; then
             log_warning_msg "Removing stale $PID_FILE"
-            rm $PID_FILE
+            rm "$PID_FILE"
         fi
     fi
 }
 
 handle_datadir () {
-    [ -d $DATA_DIR ] || mkdir -p $DATA_DIR && chown -R $RUN_AS $DATA_DIR > /dev/null || {
+    [ -d "$DATA_DIR" ] || mkdir -p "$DATA_DIR" && chown -R "$RUN_AS" "$DATA_DIR" > /dev/null || {
         log_warning_msg "$DESC: Could not create $DATA_DIR, See $SETTINGS, aborting.";
         return 1;}
 }
 
 handle_updates () {
-    chown -R $RUN_AS $APP_PATH > /dev/null || {
+    chown -R "$RUN_AS" "$APP_PATH" > /dev/null || {
         log_warning_msg "$DESC: $APP_PATH not writable by $RUN_AS for web-updates";
         return 0; }
 }
@@ -178,7 +174,7 @@ start_headphones () {
 stop_headphones () {
     if is_running; then
         log_daemon_msg "Stopping $DESC"
-        start-stop-daemon -o --stop --pidfile $PID_FILE --retry 15
+        start-stop-daemon -o --stop --pidfile "$PID_FILE" --retry 15
         check_retval
     else
         log_success_msg "$DESC: not running"

lib/bs4/__init__.py

@@ -17,8 +17,8 @@ http://www.crummy.com/software/BeautifulSoup/bs4/doc/
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.3.2"
-__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
+__version__ = "4.4.0"
+__copyright__ = "Copyright (c) 2004-2015 Leonard Richardson"
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
@@ -45,7 +45,7 @@ from .element import (
 
 # The very first thing we do is give a useful error if someone is
 # running this code under Python 3 without converting it.
-syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'<>'You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
 
 class BeautifulSoup(Tag):
     """
@@ -77,8 +77,11 @@ class BeautifulSoup(Tag):
 
     ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nTo get rid of this warning, change this:\n\n BeautifulSoup([your markup])\n\nto this:\n\n BeautifulSoup([your markup], \"%(parser)s\")\n"
+
     def __init__(self, markup="", features=None, builder=None,
-                 parse_only=None, from_encoding=None, **kwargs):
+                 parse_only=None, from_encoding=None, exclude_encodings=None,
+                 **kwargs):
         """The Soup object is initialized as the 'root tag', and the
         provided markup (which can be a string or a file-like object)
         is fed into the underlying parser."""
@@ -114,9 +117,9 @@ class BeautifulSoup(Tag):
             del kwargs['isHTML']
             warnings.warn(
                 "BS4 does not respect the isHTML argument to the "
-                "BeautifulSoup constructor. You can pass in features='html' "
-                "or features='xml' to get a builder capable of handling "
-                "one or the other.")
+                "BeautifulSoup constructor. Suggest you use "
+                "features='lxml' for HTML and features='lxml-xml' for "
+                "XML.")
 
         def deprecated_argument(old_name, new_name):
             if old_name in kwargs:
@@ -140,6 +143,7 @@ class BeautifulSoup(Tag):
                 "__init__() got an unexpected keyword argument '%s'" % arg)
 
         if builder is None:
+            original_features = features
             if isinstance(features, basestring):
                 features = [features]
             if features is None or len(features) == 0:
@@ -151,6 +155,16 @@ class BeautifulSoup(Tag):
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
             builder = builder_class()
+            if not (original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES):
+                if builder.is_xml:
+                    markup_type = "XML"
+                else:
+                    markup_type = "HTML"
+                warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict(
+                    parser=builder.NAME,
+                    markup_type=markup_type))
+
         self.builder = builder
         self.is_xml = builder.is_xml
         self.builder.soup = self
@@ -178,6 +192,8 @@ class BeautifulSoup(Tag):
                 # system. Just let it go.
                 pass
             if is_file:
+                if isinstance(markup, unicode):
+                    markup = markup.encode("utf8")
                 warnings.warn(
                     '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
             if markup[:5] == "http:" or markup[:6] == "https:":
@@ -185,12 +201,15 @@ class BeautifulSoup(Tag):
                 # Python 3 otherwise.
                 if ((isinstance(markup, bytes) and not b' ' in markup)
                     or (isinstance(markup, unicode) and not u' ' in markup)):
+                    if isinstance(markup, unicode):
+                        markup = markup.encode("utf8")
                     warnings.warn(
                         '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
 
         for (self.markup, self.original_encoding, self.declared_html_encoding,
          self.contains_replacement_characters) in (
-            self.builder.prepare_markup(markup, from_encoding)):
+             self.builder.prepare_markup(
+                 markup, from_encoding, exclude_encodings=exclude_encodings)):
             self.reset()
             try:
                 self._feed()
@@ -203,6 +222,16 @@ class BeautifulSoup(Tag):
         self.markup = None
         self.builder.soup = None
 
+    def __copy__(self):
+        return type(self)(self.encode(), builder=self.builder)
+
+    def __getstate__(self):
+        # Frequently a tree builder can't be pickled.
+        d = dict(self.__dict__)
+        if 'builder' in d and not self.builder.picklable:
+            del d['builder']
+        return d
+
     def _feed(self):
         # Convert the document to Unicode.
         self.builder.reset()
@@ -229,9 +258,7 @@ class BeautifulSoup(Tag):
 
     def new_string(self, s, subclass=NavigableString):
         """Create a new NavigableString associated with this soup."""
-        navigable = subclass(s)
-        navigable.setup()
-        return navigable
+        return subclass(s)
 
     def insert_before(self, successor):
         raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
@@ -290,14 +317,49 @@ class BeautifulSoup(Tag):
     def object_was_parsed(self, o, parent=None, most_recent_element=None):
         """Add an object to the parse tree."""
         parent = parent or self.currentTag
-        most_recent_element = most_recent_element or self._most_recent_element
-        o.setup(parent, most_recent_element)
+        previous_element = most_recent_element or self._most_recent_element
+
+        next_element = previous_sibling = next_sibling = None
+        if isinstance(o, Tag):
+            next_element = o.next_element
+            next_sibling = o.next_sibling
+            previous_sibling = o.previous_sibling
+            if not previous_element:
+                previous_element = o.previous_element
+
+        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
 
-        if most_recent_element is not None:
-            most_recent_element.next_element = o
         self._most_recent_element = o
         parent.contents.append(o)
 
+        if parent.next_sibling:
+            # This node is being inserted into an element that has
+            # already been parsed. Deal with any dangling references.
+            index = parent.contents.index(o)
+            if index == 0:
+                previous_element = parent
+                previous_sibling = None
+            else:
+                previous_element = previous_sibling = parent.contents[index-1]
+            if index == len(parent.contents)-1:
+                next_element = parent.next_sibling
+                next_sibling = None
+            else:
+                next_element = next_sibling = parent.contents[index+1]
+
+            o.previous_element = previous_element
+            if previous_element:
+                previous_element.next_element = o
+            o.next_element = next_element
+            if next_element:
+                next_element.previous_element = o
+            o.next_sibling = next_sibling
+            if next_sibling:
+                next_sibling.previous_sibling = o
+            o.previous_sibling = previous_sibling
+            if previous_sibling:
+                previous_sibling.next_sibling = o
+
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
         instance of the given tag. If inclusivePop is false, pops the tag

lib/bs4/builder/__init__.py

@@ -80,9 +80,12 @@ builder_registry = TreeBuilderRegistry()
 class TreeBuilder(object):
     """Turn a document into a Beautiful Soup object tree."""
 
+    NAME = "[Unknown tree builder]"
+    ALTERNATE_NAMES = []
     features = []
 
     is_xml = False
+    picklable = False
     preserve_whitespace_tags = set()
     empty_element_tags = None # A tag will be considered an empty-element
                               # tag when and only when it has no contents.

lib/bs4/builder/_html5lib.py

@@ -2,6 +2,7 @@ __all__ = [
     'HTML5TreeBuilder',
     ]
 
+from pdb import set_trace
 import warnings
 from bs4.builder import (
     PERMISSIVE,
@@ -9,7 +10,10 @@ from bs4.builder import (
     HTML_5,
     HTMLTreeBuilder,
     )
-from bs4.element import NamespacedAttribute
+from bs4.element import (
+    NamespacedAttribute,
+    whitespace_re,
+)
 import html5lib
 from html5lib.constants import namespaces
 from bs4.element import (
@@ -22,11 +26,20 @@ from bs4.element import (
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
-    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+    NAME = "html5lib"
 
-    def prepare_markup(self, markup, user_specified_encoding):
+    features = [NAME, PERMISSIVE, HTML_5, HTML]
+
+    def prepare_markup(self, markup, user_specified_encoding,
+                       document_declared_encoding=None, exclude_encodings=None):
         # Store the user-specified encoding for use later on.
         self.user_specified_encoding = user_specified_encoding
+
+        # document_declared_encoding and exclude_encodings aren't used
+        # ATM because the html5lib TreeBuilder doesn't use
+        # UnicodeDammit.
+        if exclude_encodings:
+            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
         yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
@@ -101,7 +114,13 @@ class AttrList(object):
     def __iter__(self):
         return list(self.attrs.items()).__iter__()
     def __setitem__(self, name, value):
-        "set attr", name, value
+        # If this attribute is a multi-valued attribute for this element,
+        # turn its value into a list.
+        list_attr = HTML5TreeBuilder.cdata_list_attributes
+        if (name in list_attr['*']
+            or (self.element.name in list_attr
+                and name in list_attr[self.element.name])):
+            value = whitespace_re.split(value)
         self.element[name] = value
     def items(self):
         return list(self.attrs.items())
@@ -161,6 +180,12 @@ class Element(html5lib.treebuilders._base.Node):
             # immediately after the parent, if it has no children.)
             if self.element.contents:
                 most_recent_element = self.element._last_descendant(False)
+            elif self.element.next_element is not None:
+                # Something from further ahead in the parse tree is
+                # being inserted into this earlier element. This is
+                # very annoying because it means an expensive search
+                # for the last element in the tree.
+                most_recent_element = self.soup._last_descendant()
             else:
                 most_recent_element = self.element
 
@@ -172,6 +197,7 @@ class Element(html5lib.treebuilders._base.Node):
         return AttrList(self.element)
 
     def setAttributes(self, attributes):
+
         if attributes is not None and len(attributes) > 0:
 
             converted_attributes = []
@@ -218,6 +244,9 @@ class Element(html5lib.treebuilders._base.Node):
 
     def reparentChildren(self, new_parent):
         """Move all of this tag's children into another tag."""
+        # print "MOVE", self.element.contents
+        # print "FROM", self.element
+        # print "TO", new_parent.element
         element = self.element
         new_parent_element = new_parent.element
         # Determine what this tag's next_element will be once all the children
@@ -236,17 +265,28 @@ class Element(html5lib.treebuilders._base.Node):
             new_parents_last_descendant_next_element = new_parent_element.next_element
 
         to_append = element.contents
-        append_after = new_parent.element.contents
+        append_after = new_parent_element.contents
         if len(to_append) > 0:
             # Set the first child's previous_element and previous_sibling
             # to elements within the new parent
             first_child = to_append[0]
-            first_child.previous_element = new_parents_last_descendant
+            if new_parents_last_descendant:
+                first_child.previous_element = new_parents_last_descendant
+            else:
+                first_child.previous_element = new_parent_element
             first_child.previous_sibling = new_parents_last_child
+            if new_parents_last_descendant:
+                new_parents_last_descendant.next_element = first_child
+            else:
+                new_parent_element.next_element = first_child
+            if new_parents_last_child:
+                new_parents_last_child.next_sibling = first_child
 
             # Fix the last child's next_element and next_sibling
             last_child = to_append[-1]
             last_child.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element:
+                new_parents_last_descendant_next_element.previous_element = last_child
             last_child.next_sibling = None
 
         for child in to_append:
@@ -257,6 +297,10 @@ class Element(html5lib.treebuilders._base.Node):
         element.contents = []
         element.next_element = final_next_element
 
+        # print "DONE WITH MOVE"
+        # print "FROM", self.element
+        # print "TO", new_parent_element
+
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)
         node = Element(tag, self.soup, self.namespace)
@@ -268,7 +312,7 @@ class Element(html5lib.treebuilders._base.Node):
         return self.element.contents
 
     def getNameTuple(self):
-        if self.namespace is None:
+        if self.namespace == None:
             return namespaces["html"], self.name
         else:
             return self.namespace, self.name

lib/bs4/builder/_htmlparser.py

@@ -4,10 +4,16 @@ __all__ = [
     'HTMLParserTreeBuilder',
     ]
 
-from HTMLParser import (
-    HTMLParser,
-    HTMLParseError,
-    )
+from HTMLParser import HTMLParser
+
+try:
+    from HTMLParser import HTMLParseError
+except ImportError, e:
+    # HTMLParseError is removed in Python 3.5. Since it can never be
+    # thrown in 3.5, we can just define our own class as a placeholder.
+    class HTMLParseError(Exception):
+        pass
+
 import sys
 import warnings
 
@@ -19,10 +25,10 @@ import warnings
 # At the end of this file, we monkeypatch HTMLParser so that
 # strict=True works well on Python 3.2.2.
 major, minor, release = sys.version_info[:3]
-CONSTRUCTOR_TAKES_STRICT = (
-    major > 3
-    or (major == 3 and minor > 2)
-    or (major == 3 and minor == 2 and release >= 3))
+CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
+CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
+CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
+
 
 from bs4.element import (
     CData,
@@ -63,7 +69,8 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_charref(self, name):
         # XXX workaround for a bug in HTMLParser. Remove this once
-        # it's fixed.
+        # it's fixed in all supported versions.
+        # http://bugs.python.org/issue13633
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
         elif name.startswith('X'):
@@ -113,14 +120,6 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_pi(self, data):
         self.soup.endData()
-        if data.endswith("?") and data.lower().startswith("xml"):
-            # "An XHTML processing instruction using the trailing '?'
-            # will cause the '?' to be included in data." - HTMLParser
-            # docs.
-            #
-            # Strip the question mark so we don't end up with two
-            # question marks.
-            data = data[:-1]
         self.soup.handle_data(data)
         self.soup.endData(ProcessingInstruction)
 
@@ -128,15 +127,19 @@ class BeautifulSoupHTMLParser(HTMLParser):
 class HTMLParserTreeBuilder(HTMLTreeBuilder):
 
     is_xml = False
-    features = [HTML, STRICT, HTMLPARSER]
+    picklable = True
+    NAME = HTMLPARSER
+    features = [NAME, HTML, STRICT]
 
     def __init__(self, *args, **kwargs):
-        if CONSTRUCTOR_TAKES_STRICT:
+        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
             kwargs['strict'] = False
+        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
+            kwargs['convert_charrefs'] = False
         self.parser_args = (args, kwargs)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
+                       document_declared_encoding=None, exclude_encodings=None):
         """
         :return: A 4-tuple (markup, original encoding, encoding
         declared within markup, whether any characters had to be
@@ -147,7 +150,8 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
             return
 
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
+                               exclude_encodings=exclude_encodings)
         yield (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)

lib/bs4/builder/_lxml.py

@@ -7,7 +7,12 @@ from io import BytesIO
 from StringIO import StringIO
 import collections
 from lxml import etree
-from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.element import (
+    Comment,
+    Doctype,
+    NamespacedAttribute,
+    ProcessingInstruction,
+)
 from bs4.builder import (
     FAST,
     HTML,
@@ -25,8 +30,11 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     is_xml = True
 
+    NAME = "lxml-xml"
+    ALTERNATE_NAMES = ["xml"]
+
     # Well, it's permissive by XML parser standards.
-    features = [LXML, XML, FAST, PERMISSIVE]
+    features = [NAME, LXML, XML, FAST, PERMISSIVE]
 
     CHUNK_SIZE = 512
 
@@ -70,6 +78,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             return (None, tag)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
+                       exclude_encodings=None,
                        document_declared_encoding=None):
         """
         :yield: A series of 4-tuples.
@@ -95,7 +104,8 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # the document as each one in turn.
         is_html = not self.is_xml
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        detector = EncodingDetector(markup, try_encodings, is_html)
+        detector = EncodingDetector(
+            markup, try_encodings, is_html, exclude_encodings)
         for encoding in detector.encodings:
             yield (detector.markup, encoding, document_declared_encoding, False)
 
@@ -189,7 +199,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             self.nsmaps.pop()
 
     def pi(self, target, data):
-        pass
+        self.soup.endData()
+        self.soup.handle_data(target + ' ' + data)
+        self.soup.endData(ProcessingInstruction)
 
     def data(self, content):
         self.soup.handle_data(content)
@@ -212,7 +224,10 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
 class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
 
-    features = [LXML, HTML, FAST, PERMISSIVE]
+    NAME = LXML
+    ALTERNATE_NAMES = ["lxml-html"]
+
+    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
     is_xml = False
 
     def default_parser(self, encoding):

lib/bs4/dammit.py

@@ -3,10 +3,11 @@
 
 This library converts a bytestream to Unicode through any means
 necessary. It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It works best on XML and XML, but it does not rewrite the
+Feed Parser. It works best on XML and HTML, but it does not rewrite the
 XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
 
+from pdb import set_trace
 import codecs
 from htmlentitydefs import codepoint2name
 import re
@@ -212,8 +213,11 @@ class EncodingDetector:
 
     5. Windows-1252.
     """
-    def __init__(self, markup, override_encodings=None, is_html=False):
+    def __init__(self, markup, override_encodings=None, is_html=False,
+                 exclude_encodings=None):
         self.override_encodings = override_encodings or []
+        exclude_encodings = exclude_encodings or []
+        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
         self.chardet_encoding = None
         self.is_html = is_html
         self.declared_encoding = None
@@ -224,6 +228,8 @@ class EncodingDetector:
     def _usable(self, encoding, tried):
         if encoding is not None:
             encoding = encoding.lower()
+            if encoding in self.exclude_encodings:
+                return False
             if encoding not in tried:
                 tried.add(encoding)
                 return True
@@ -266,6 +272,9 @@ class EncodingDetector:
     def strip_byte_order_mark(cls, data):
         """If a byte-order mark is present, strip it and return the encoding it implies."""
         encoding = None
+        if isinstance(data, unicode):
+            # Unicode data cannot have a byte-order mark.
+            return data, encoding
         if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
                and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16be'
@@ -299,14 +308,14 @@ class EncodingDetector:
         else:
             xml_endpos = 1024
             html_endpos = max(2048, int(len(markup) * 0.05))
-
+            
         declared_encoding = None
         declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
         if not declared_encoding_match and is_html:
             declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
         if declared_encoding_match is not None:
             declared_encoding = declared_encoding_match.groups()[0].decode(
-                'ascii')
+                'ascii', 'replace')
         if declared_encoding:
             return declared_encoding.lower()
         return None
@@ -331,13 +340,14 @@ class UnicodeDammit:
         ]
 
     def __init__(self, markup, override_encodings=[],
-                 smart_quotes_to=None, is_html=False):
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
         self.is_html = is_html
 
-        self.detector = EncodingDetector(markup, override_encodings, is_html)
+        self.detector = EncodingDetector(
+            markup, override_encodings, is_html, exclude_encodings)
 
         # Short-circuit if the data is in Unicode to begin with.
         if isinstance(markup, unicode) or markup == '':

lib/bs4/diagnose.py

@@ -33,12 +33,21 @@ def diagnose(data):
 
     if 'lxml' in basic_parsers:
         basic_parsers.append(["lxml", "xml"])
-        from lxml import etree
-        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+        try:
+            from lxml import etree
+            print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+        except ImportError, e:
+            print (
+                "lxml is not installed or couldn't be imported.")
+
 
     if 'html5lib' in basic_parsers:
-        import html5lib
-        print "Found html5lib version %s" % html5lib.__version__
+        try:
+            import html5lib
+            print "Found html5lib version %s" % html5lib.__version__
+        except ImportError, e:
+            print (
+                "html5lib is not installed or couldn't be imported.")
 
     if hasattr(data, 'read'):
         data = data.read()
@@ -135,7 +144,7 @@ def rword(length=5):
 def rsentence(length=4):
     "Generate a random sentence-like string."
     return " ".join(rword(random.randint(4,9)) for i in range(length))
-
+        
 def rdoc(num_elements=1000):
     """Randomly generate an invalid HTML document."""
     tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
@@ -159,7 +168,7 @@ def benchmark_parsers(num_elements=100000):
     print "Comparative parser benchmark on Beautiful Soup %s" % __version__
     data = rdoc(num_elements)
     print "Generated a large invalid HTML document (%d bytes)." % len(data)
-
+    
     for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
         success = False
         try:

lib/bs4/element.py

@@ -1,3 +1,4 @@
+from pdb import set_trace
 import collections
 import re
 import sys
@@ -185,24 +186,40 @@ class PageElement(object):
             return self.HTML_FORMATTERS.get(
                 name, HTMLAwareEntitySubstitution.substitute_xml)
 
-    def setup(self, parent=None, previous_element=None):
+    def setup(self, parent=None, previous_element=None, next_element=None,
+              previous_sibling=None, next_sibling=None):
         """Sets up the initial relations between this element and
         other elements."""
         self.parent = parent
+
         self.previous_element = previous_element
         if previous_element is not None:
             self.previous_element.next_element = self
-        self.next_element = None
-        self.previous_sibling = None
-        self.next_sibling = None
-        if self.parent is not None and self.parent.contents:
-            self.previous_sibling = self.parent.contents[-1]
+
+        self.next_element = next_element
+        if self.next_element:
+            self.next_element.previous_element = self
+
+        self.next_sibling = next_sibling
+        if self.next_sibling:
+            self.next_sibling.previous_sibling = self
+
+        if (not previous_sibling
+            and self.parent is not None and self.parent.contents):
+            previous_sibling = self.parent.contents[-1]
+
+        self.previous_sibling = previous_sibling
+        if previous_sibling:
             self.previous_sibling.next_sibling = self
 
     nextSibling = _alias("next_sibling")  # BS3
     previousSibling = _alias("previous_sibling")  # BS3
 
     def replace_with(self, replace_with):
+        if not self.parent:
+            raise ValueError(
+                "Cannot replace one element with another when the"
+                "element to be replaced is not part of a tree.")
         if replace_with is self:
             return
         if replace_with is self.parent:
@@ -216,6 +233,10 @@ class PageElement(object):
 
     def unwrap(self):
         my_parent = self.parent
+        if not self.parent:
+            raise ValueError(
+                "Cannot replace an element with its contents when that"
+                "element is not part of a tree.")
         my_index = self.parent.index(self)
         self.extract()
         for child in reversed(self.contents[:]):
@@ -240,17 +261,20 @@ class PageElement(object):
         last_child = self._last_descendant()
         next_element = last_child.next_element
 
-        if self.previous_element is not None:
+        if (self.previous_element is not None and
+            self.previous_element != next_element):
             self.previous_element.next_element = next_element
-        if next_element is not None:
+        if next_element is not None and next_element != self.previous_element:
             next_element.previous_element = self.previous_element
         self.previous_element = None
         last_child.next_element = None
 
         self.parent = None
-        if self.previous_sibling is not None:
+        if (self.previous_sibling is not None
+            and self.previous_sibling != self.next_sibling):
             self.previous_sibling.next_sibling = self.next_sibling
-        if self.next_sibling is not None:
+        if (self.next_sibling is not None
+            and self.next_sibling != self.previous_sibling):
             self.next_sibling.previous_sibling = self.previous_sibling
         self.previous_sibling = self.next_sibling = None
         return self
@@ -478,6 +502,10 @@ class PageElement(object):
     def _find_all(self, name, attrs, text, limit, generator, **kwargs):
         "Iterates over a generator looking for things that match."
 
+        if text is None and 'string' in kwargs:
+            text = kwargs['string']
+            del kwargs['string']
+
         if isinstance(name, SoupStrainer):
             strainer = name
         else:
@@ -548,17 +576,17 @@ class PageElement(object):
 
     # Methods for supporting CSS selectors.
 
-    tag_name_re = re.compile('^[a-z0-9]+$')
+    tag_name_re = re.compile('^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$')
 
-    # /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
-    #   \---/  \---/\-------------/    \-------/
-    #     |      |         |               |
-    #     |      |         |           The value
-    #     |      |    ~,|,^,$,* or =
-    #     |   Attribute
+    # /^([a-zA-Z0-9][-.a-zA-Z0-9:_]*)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
+    #   \---------------------------/  \---/\-------------/    \-------/
+    #     |                              |         |               |
+    #     |                              |         |           The value
+    #     |                              |    ~,|,^,$,* or =
+    #     |                           Attribute
     #    Tag
     attribselect_re = re.compile(
-        r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' +
+        r'^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)' +
         r'=?"?(?P<value>[^\]"]*)"?\]$'
         )
 
@@ -654,11 +682,17 @@ class NavigableString(unicode, PageElement):
         how to handle non-ASCII characters.
         """
         if isinstance(value, unicode):
-            return unicode.__new__(cls, value)
-        return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
+            u = unicode.__new__(cls, value)
+        else:
+            u = unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
+        u.setup()
+        return u
 
     def __copy__(self):
-        return self
+        """A copy of a NavigableString has the same contents and class
+        as the original, but it is not connected to the parse tree.
+        """
+        return type(self)(self)
 
     def __getnewargs__(self):
         return (unicode(self),)
@@ -707,7 +741,7 @@ class CData(PreformattedString):
 class ProcessingInstruction(PreformattedString):
 
     PREFIX = u'<?'
-    SUFFIX = u'?>'
+    SUFFIX = u'>'
 
 class Comment(PreformattedString):
 
@@ -759,9 +793,12 @@ class Tag(PageElement):
         self.prefix = prefix
         if attrs is None:
             attrs = {}
-        elif attrs and builder.cdata_list_attributes:
-            attrs = builder._replace_cdata_list_attribute_values(
-                self.name, attrs)
+        elif attrs:
+            if builder is not None and builder.cdata_list_attributes:
+                attrs = builder._replace_cdata_list_attribute_values(
+                    self.name, attrs)
+            else:
+                attrs = dict(attrs)
         else:
             attrs = dict(attrs)
         self.attrs = attrs
@@ -778,6 +815,18 @@ class Tag(PageElement):
 
     parserClass = _alias("parser_class")  # BS3
 
+    def __copy__(self):
+        """A copy of a Tag is a new Tag, unconnected to the parse tree.
+        Its contents are a copy of the old Tag's contents.
+        """
+        clone = type(self)(None, self.builder, self.name, self.namespace,
+                           self.nsprefix, self.attrs)
+        for attr in ('can_be_empty_element', 'hidden'):
+            setattr(clone, attr, getattr(self, attr))
+        for child in self.contents:
+            clone.append(child.__copy__())
+        return clone
+
     @property
     def is_empty_element(self):
         """Is this tag an empty-element tag? (aka a self-closing tag)
@@ -971,15 +1020,25 @@ class Tag(PageElement):
         as defined in __eq__."""
         return not self == other
 
-    def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
+    def __repr__(self, encoding="unicode-escape"):
         """Renders this tag as a string."""
-        return self.encode(encoding)
+        if PY3K:
+            # "The return value must be a string object", i.e. Unicode
+            return self.decode()
+        else:
+            # "The return value must be a string object", i.e. a bytestring.
+            # By convention, the return value of __repr__ should also be
+            # an ASCII string.
+            return self.encode(encoding)
 
     def __unicode__(self):
         return self.decode()
 
     def __str__(self):
-        return self.encode()
+        if PY3K:
+            return self.decode()
+        else:
+            return self.encode()
 
     if PY3K:
         __str__ = __repr__ = __unicode__
@@ -1103,12 +1162,18 @@ class Tag(PageElement):
                        formatter="minimal"):
         """Renders the contents of this tag as a Unicode string.
 
+        :param indent_level: Each line of the rendering will be
+           indented this many spaces.
+
         :param eventual_encoding: The tag is destined to be
            encoded into this encoding. This method is _not_
            responsible for performing that encoding. This information
            is passed in so that it can be substituted in if the
            document contains a <META> tag that mentions the document's
            encoding.
+
+        :param formatter: The output formatter responsible for converting
+           entities to Unicode characters.
         """
         # First off, turn a string formatter into a function. This
         # will stop the lookup from happening over and over again.
@@ -1137,7 +1202,17 @@ class Tag(PageElement):
     def encode_contents(
         self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING,
         formatter="minimal"):
-        """Renders the contents of this tag as a bytestring."""
+        """Renders the contents of this tag as a bytestring.
+
+        :param indent_level: Each line of the rendering will be
+           indented this many spaces.
+
+        :param eventual_encoding: The bytestring will be in this encoding.
+
+        :param formatter: The output formatter responsible for converting
+           entities to Unicode characters.
+        """
+
         contents = self.decode_contents(indent_level, encoding, formatter)
         return contents.encode(encoding)
 
@@ -1201,63 +1276,89 @@ class Tag(PageElement):
 
     _selector_combinators = ['>', '+', '~']
     _select_debug = False
-    def select(self, selector, _candidate_generator=None):
+    def select_one(self, selector):
         """Perform a CSS selection operation on the current element."""
-        tokens = selector.split()
+        value = self.select(selector, limit=1)
+        if value:
+            return value[0]
+        return None
+
+    def select(self, selector, _candidate_generator=None, limit=None):
+        """Perform a CSS selection operation on the current element."""
+
+        # Remove whitespace directly after the grouping operator ','
+        # then split into tokens.
+        tokens = re.sub(',[\s]*',',', selector).split()
         current_context = [self]
 
         if tokens[-1] in self._selector_combinators:
             raise ValueError(
                 'Final combinator "%s" is missing an argument.' % tokens[-1])
+
         if self._select_debug:
             print 'Running CSS selector "%s"' % selector
-        for index, token in enumerate(tokens):
-            if self._select_debug:
-                print ' Considering token "%s"' % token
-            recursive_candidate_generator = None
-            tag_name = None
+
+        for index, token_group in enumerate(tokens):
+            new_context = []
+            new_context_ids = set([])
+
+            # Grouping selectors, ie: p,a
+            grouped_tokens = token_group.split(',')
+            if '' in grouped_tokens:
+                raise ValueError('Invalid group selection syntax: %s' % token_group)
+
             if tokens[index-1] in self._selector_combinators:
                 # This token was consumed by the previous combinator. Skip it.
                 if self._select_debug:
                     print '  Token was consumed by the previous combinator.'
                 continue
-            # Each operation corresponds to a checker function, a rule
-            # for determining whether a candidate matches the
-            # selector. Candidates are generated by the active
-            # iterator.
-            checker = None
 
-            m = self.attribselect_re.match(token)
-            if m is not None:
-                # Attribute selector
-                tag_name, attribute, operator, value = m.groups()
-                checker = self._attribute_checker(operator, attribute, value)
+            for token in grouped_tokens:
+                if self._select_debug:
+                    print ' Considering token "%s"' % token
+                recursive_candidate_generator = None
+                tag_name = None
 
-            elif '#' in token:
-                # ID selector
-                tag_name, tag_id = token.split('#', 1)
-                def id_matches(tag):
-                    return tag.get('id', None) == tag_id
-                checker = id_matches
+                # Each operation corresponds to a checker function, a rule
+                # for determining whether a candidate matches the
+                # selector. Candidates are generated by the active
+                # iterator.
+                checker = None
 
-            elif '.' in token:
-                # Class selector
-                tag_name, klass = token.split('.', 1)
-                classes = set(klass.split('.'))
-                def classes_match(candidate):
-                    return classes.issubset(candidate.get('class', []))
-                checker = classes_match
+                m = self.attribselect_re.match(token)
+                if m is not None:
+                    # Attribute selector
+                    tag_name, attribute, operator, value = m.groups()
+                    checker = self._attribute_checker(operator, attribute, value)
 
-            elif ':' in token:
-                # Pseudo-class
-                tag_name, pseudo = token.split(':', 1)
-                if tag_name == '':
-                    raise ValueError(
-                        "A pseudo-class must be prefixed with a tag name.")
-                pseudo_attributes = re.match('([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo)
-                found = []
-                if pseudo_attributes is not None:
-                    pseudo_type, pseudo_value = pseudo_attributes.groups()
+                elif '#' in token:
+                    # ID selector
+                    tag_name, tag_id = token.split('#', 1)
+                    def id_matches(tag):
+                        return tag.get('id', None) == tag_id
+                    checker = id_matches
+
+                elif '.' in token:
+                    # Class selector
+                    tag_name, klass = token.split('.', 1)
+                    classes = set(klass.split('.'))
+                    def classes_match(candidate):
+                        return classes.issubset(candidate.get('class', []))
+                    checker = classes_match
+
+                elif ':' in token:
+                    # Pseudo-class
+                    tag_name, pseudo = token.split(':', 1)
+                    if tag_name == '':
+                        raise ValueError(
+                            "A pseudo-class must be prefixed with a tag name.")
+                    pseudo_attributes = re.match('([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo)
+                    found = []
+                    if pseudo_attributes is None:
+                        pseudo_type = pseudo
+                        pseudo_value = None
+                    else:
+                        pseudo_type, pseudo_value = pseudo_attributes.groups()
                     if pseudo_type == 'nth-of-type':
                         try:
                             pseudo_value = int(pseudo_value)
@@ -1286,109 +1387,110 @@ class Tag(PageElement):
                         raise NotImplementedError(
                             'Only the following pseudo-classes are implemented: nth-of-type.')
 
-            elif token == '*':
-                # Star selector -- matches everything
-                pass
-            elif token == '>':
-                # Run the next token as a CSS selector against the
-                # direct children of each tag in the current context.
-                recursive_candidate_generator = lambda tag: tag.children
-            elif token == '~':
-                # Run the next token as a CSS selector against the
-                # siblings of each tag in the current context.
-                recursive_candidate_generator = lambda tag: tag.next_siblings
-            elif token == '+':
-                # For each tag in the current context, run the next
-                # token as a CSS selector against the tag's next
-                # sibling that's a tag.
-                def next_tag_sibling(tag):
-                    yield tag.find_next_sibling(True)
-                recursive_candidate_generator = next_tag_sibling
+                elif token == '*':
+                    # Star selector -- matches everything
+                    pass
+                elif token == '>':
+                    # Run the next token as a CSS selector against the
+                    # direct children of each tag in the current context.
+                    recursive_candidate_generator = lambda tag: tag.children
+                elif token == '~':
+                    # Run the next token as a CSS selector against the
+                    # siblings of each tag in the current context.
+                    recursive_candidate_generator = lambda tag: tag.next_siblings
+                elif token == '+':
+                    # For each tag in the current context, run the next
+                    # token as a CSS selector against the tag's next
+                    # sibling that's a tag.
+                    def next_tag_sibling(tag):
+                        yield tag.find_next_sibling(True)
+                    recursive_candidate_generator = next_tag_sibling
 
-            elif self.tag_name_re.match(token):
-                # Just a tag name.
-                tag_name = token
-            else:
-                raise ValueError(
-                    'Unsupported or invalid CSS selector: "%s"' % token)
-
-            if recursive_candidate_generator:
-                # This happens when the selector looks like  "> foo".
-                #
-                # The generator calls select() recursively on every
-                # member of the current context, passing in a different
-                # candidate generator and a different selector.
-                #
-                # In the case of "> foo", the candidate generator is
-                # one that yields a tag's direct children (">"), and
-                # the selector is "foo".
-                next_token = tokens[index+1]
-                def recursive_select(tag):
-                    if self._select_debug:
-                        print '    Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs)
-                        print '-' * 40
-                    for i in tag.select(next_token, recursive_candidate_generator):
-                        if self._select_debug:
-                            print '(Recursive select picked up candidate %s %s)' % (i.name, i.attrs)
-                        yield i
-                    if self._select_debug:
-                        print '-' * 40
-                _use_candidate_generator = recursive_select
-            elif _candidate_generator is None:
-                # By default, a tag's candidates are all of its
-                # children. If tag_name is defined, only yield tags
-                # with that name.
-                if self._select_debug:
-                    if tag_name:
-                        check = "[any]"
-                    else:
-                        check = tag_name
-                    print '   Default candidate generator, tag name="%s"' % check
-                if self._select_debug:
-                    # This is redundant with later code, but it stops
-                    # a bunch of bogus tags from cluttering up the
-                    # debug log.
-                    def default_candidate_generator(tag):
-                        for child in tag.descendants:
-                            if not isinstance(child, Tag):
-                                continue
-                            if tag_name and not child.name == tag_name:
-                                continue
-                            yield child
-                    _use_candidate_generator = default_candidate_generator
+                elif self.tag_name_re.match(token):
+                    # Just a tag name.
+                    tag_name = token
                 else:
-                    _use_candidate_generator = lambda tag: tag.descendants
-            else:
-                _use_candidate_generator = _candidate_generator
-
-            new_context = []
-            new_context_ids = set([])
-            for tag in current_context:
-                if self._select_debug:
-                    print "    Running candidate generator on %s %s" % (
-                        tag.name, repr(tag.attrs))
-                for candidate in _use_candidate_generator(tag):
-                    if not isinstance(candidate, Tag):
-                        continue
-                    if tag_name and candidate.name != tag_name:
-                        continue
-                    if checker is not None:
-                        try:
-                            result = checker(candidate)
-                        except StopIteration:
-                            # The checker has decided we should no longer
-                            # run the generator.
-                            break
-                    if checker is None or result:
+                    raise ValueError(
+                        'Unsupported or invalid CSS selector: "%s"' % token)
+                if recursive_candidate_generator:
+                    # This happens when the selector looks like  "> foo".
+                    #
+                    # The generator calls select() recursively on every
+                    # member of the current context, passing in a different
+                    # candidate generator and a different selector.
+                    #
+                    # In the case of "> foo", the candidate generator is
+                    # one that yields a tag's direct children (">"), and
+                    # the selector is "foo".
+                    next_token = tokens[index+1]
+                    def recursive_select(tag):
                         if self._select_debug:
-                            print "     SUCCESS %s %s" % (candidate.name, repr(candidate.attrs))
-                        if id(candidate) not in new_context_ids:
-                            # If a tag matches a selector more than once,
-                            # don't include it in the context more than once.
-                            new_context.append(candidate)
-                            new_context_ids.add(id(candidate))
-                    elif self._select_debug:
-                        print "     FAILURE %s %s" % (candidate.name, repr(candidate.attrs))
+                            print '    Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs)
+                            print '-' * 40
+                        for i in tag.select(next_token, recursive_candidate_generator):
+                            if self._select_debug:
+                                print '(Recursive select picked up candidate %s %s)' % (i.name, i.attrs)
+                            yield i
+                        if self._select_debug:
+                            print '-' * 40
+                    _use_candidate_generator = recursive_select
+                elif _candidate_generator is None:
+                    # By default, a tag's candidates are all of its
+                    # children. If tag_name is defined, only yield tags
+                    # with that name.
+                    if self._select_debug:
+                        if tag_name:
+                            check = "[any]"
+                        else:
+                            check = tag_name
+                        print '   Default candidate generator, tag name="%s"' % check
+                    if self._select_debug:
+                        # This is redundant with later code, but it stops
+                        # a bunch of bogus tags from cluttering up the
+                        # debug log.
+                        def default_candidate_generator(tag):
+                            for child in tag.descendants:
+                                if not isinstance(child, Tag):
+                                    continue
+                                if tag_name and not child.name == tag_name:
+                                    continue
+                                yield child
+                        _use_candidate_generator = default_candidate_generator
+                    else:
+                        _use_candidate_generator = lambda tag: tag.descendants
+                else:
+                    _use_candidate_generator = _candidate_generator
+
+                count = 0
+                for tag in current_context:
+                    if self._select_debug:
+                        print "    Running candidate generator on %s %s" % (
+                            tag.name, repr(tag.attrs))
+                    for candidate in _use_candidate_generator(tag):
+                        if not isinstance(candidate, Tag):
+                            continue
+                        if tag_name and candidate.name != tag_name:
+                            continue
+                        if checker is not None:
+                            try:
+                                result = checker(candidate)
+                            except StopIteration:
+                                # The checker has decided we should no longer
+                                # run the generator.
+                                break
+                        if checker is None or result:
+                            if self._select_debug:
+                                print "     SUCCESS %s %s" % (candidate.name, repr(candidate.attrs))
+                            if id(candidate) not in new_context_ids:
+                                # If a tag matches a selector more than once,
+                                # don't include it in the context more than once.
+                                new_context.append(candidate)
+                                new_context_ids.add(id(candidate))
+                                if limit and len(new_context) >= limit:
+                                    break
+                        elif self._select_debug:
+                            print "     FAILURE %s %s" % (candidate.name, repr(candidate.attrs))
+
 
             current_context = new_context
 

lib/bs4/testing.py

@@ -1,5 +1,6 @@
 """Helper classes for tests."""
 
+import pickle
 import copy
 import functools
 import unittest
@@ -43,6 +44,16 @@ class SoupTest(unittest.TestCase):
 
         self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
 
+    def assertConnectedness(self, element):
+        """Ensure that next_element and previous_element are properly
+        set for all descendants of the given element.
+        """
+        earlier = None
+        for e in element.descendants:
+            if earlier:
+                self.assertEqual(e, earlier.next_element)
+                self.assertEqual(earlier, e.previous_element)
+            earlier = e
 
 class HTMLTreeBuilderSmokeTest(object):
 
@@ -54,6 +65,15 @@ class HTMLTreeBuilderSmokeTest(object):
     markup in these tests, there's not much room for interpretation.
     """
 
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def assertDoctypeHandled(self, doctype_fragment):
         """Assert that a given doctype string is handled correctly."""
         doctype_str, soup = self._document_with_doctype(doctype_fragment)
@@ -114,6 +134,11 @@ class HTMLTreeBuilderSmokeTest(object):
             soup.encode("utf-8").replace(b"\n", b""),
             markup.replace(b"\n", b""))
 
+    def test_processing_instruction(self):
+        markup = b"""<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
     def test_deepcopy(self):
         """Make sure you can copy the tree builder.
 
@@ -155,6 +180,23 @@ class HTMLTreeBuilderSmokeTest(object):
     def test_nested_formatting_elements(self):
         self.assertSoupEquals("<em><em></em></em>")
 
+    def test_double_head(self):
+        html = '''<!DOCTYPE html>
+<html>
+<head>
+<title>Ordinary HEAD element test</title>
+</head>
+<script type="text/javascript">
+alert("Help!");
+</script>
+<body>
+Hello, world!
+</body>
+</html>
+'''
+        soup = self.soup(html)
+        self.assertEqual("text/javascript", soup.find('script')['type'])
+
     def test_comment(self):
         # Comments are represented as Comment objects.
         markup = "<p>foo<!--foobar-->baz</p>"
@@ -221,6 +263,14 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup(markup)
         self.assertEqual(["css"], soup.div.div['class'])
 
+    def test_multivalued_attribute_on_html(self):
+        # html5lib uses a different API to set the attributes ot the
+        # <html> tag. This has caused problems with multivalued
+        # attributes.
+        markup = '<html class="a b"></html>'
+        soup = self.soup(markup)
+        self.assertEqual(["a", "b"], soup.html['class'])
+
     def test_angle_brackets_in_attribute_values_are_escaped(self):
         self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
 
@@ -253,6 +303,35 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
         self.assertEqual("p", soup.h2.string.next_element.name)
         self.assertEqual("p", soup.p.name)
+        self.assertConnectedness(soup)
+
+    def test_head_tag_between_head_and_body(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<html><head></head>
+  <link></link>
+  <body>foo</body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertNotEqual(None, soup.html.body)
+        self.assertConnectedness(soup)
+
+    def test_multiple_copies_of_a_tag(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<!DOCTYPE html>
+<html>
+ <body>
+   <article id="a" >
+   <div><a href="1"></div>
+   <footer>
+     <a href="2"></a>
+   </footer>
+  </article>
+  </body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertConnectedness(soup.article)
 
     def test_basic_namespaces(self):
         """Parsers don't need to *understand* namespaces, but at the
@@ -463,6 +542,15 @@ class HTMLTreeBuilderSmokeTest(object):
 
 class XMLTreeBuilderSmokeTest(object):
 
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def test_docstring_generated(self):
         soup = self.soup("<root/>")
         self.assertEqual(
@@ -485,7 +573,7 @@ class XMLTreeBuilderSmokeTest(object):
   <script type="text/javascript">
   </script>
 """
-        soup = BeautifulSoup(doc, "xml")
+        soup = BeautifulSoup(doc, "lxml-xml")
         # lxml would have stripped this while parsing, but we can add
         # it later.
         soup.script.string = 'console.log("< < hey > > ");'

lib/html5lib/__init__.py

@@ -20,4 +20,6 @@ from .serializer import serialize
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "0.999"
+
+# this has to be at the top level, see how setup.py parses this
+__version__ = "0.999999"

lib/html5lib/constants.py

@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import string
-import gettext
-_ = gettext.gettext
 
 EOF = None
 
 E = {
     "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
     "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
     "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
     "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
     "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
     "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
     "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
     "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
     "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
     "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
     "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
     "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
     "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
     'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
     "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
     "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
     "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
     "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
     "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
     "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
     "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
     "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
     "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
     "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
     "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
     "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
     "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
     "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
     'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
     'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
     "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
     "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
     "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
     "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
     "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
     "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
     "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
     "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
     "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
     "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
     "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
     "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
     "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
     "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
     "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
     "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
     "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
     "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
     "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
     "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
     "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
     "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
     "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
     "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
     "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
     "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
     "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
     "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
     "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
     "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
     "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
     "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
     "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
     "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
     "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
     "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
     "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
     "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
     "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
     "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
     "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
     "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
     "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
     "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
     "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
     "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
     "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
     "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
     "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
     "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
     "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
     "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
     "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
     "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
     "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
     "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
     "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
     "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
     "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
     "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
     "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
     "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
     "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
     "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
     "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
     "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
     "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
     "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
     "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
     "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
     "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
     "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
     "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
     "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
     "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
     "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
     "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
     "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
     "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
     "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
     "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
     "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
     "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
     "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
     "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
     "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }
 
 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
     "xmlns": "http://www.w3.org/2000/xmlns/"
 }
 
-scopingElements = frozenset((
+scopingElements = frozenset([
     (namespaces["html"], "applet"),
     (namespaces["html"], "caption"),
     (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title"),
-))
+])
 
-formattingElements = frozenset((
+formattingElements = frozenset([
     (namespaces["html"], "a"),
     (namespaces["html"], "b"),
     (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
     (namespaces["html"], "strong"),
     (namespaces["html"], "tt"),
     (namespaces["html"], "u")
-))
+])
 
-specialElements = frozenset((
+specialElements = frozenset([
     (namespaces["html"], "address"),
     (namespaces["html"], "applet"),
     (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
     (namespaces["html"], "wbr"),
     (namespaces["html"], "xmp"),
     (namespaces["svg"], "foreignObject")
-))
+])
 
-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
     (namespaces["mathml"], "annotaion-xml"),
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title")
-))
+])
 
-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
     (namespaces["mathml"], "mi"),
     (namespaces["mathml"], "mo"),
     (namespaces["mathml"], "mn"),
     (namespaces["mathml"], "ms"),
     (namespaces["mathml"], "mtext")
-))
+])
 
 adjustForeignAttributes = {
     "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                   adjustForeignAttributes.items()])
 
-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
     "\t",
     "\n",
     "\u000C",
     " ",
     "\r"
-))
+])
 
-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
     "table",
     "tbody",
     "tfoot",
     "thead",
     "tr"
-))
+])
 
 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
     "h6"
 )
 
-voidElements = frozenset((
+voidElements = frozenset([
     "base",
     "command",
     "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
     "input",
     "source",
     "track"
-))
+])
 
-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])
 
-rcdataElements = frozenset((
+rcdataElements = frozenset([
     'style',
     'script',
     'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
     'noembed',
     'noframes',
     'noscript'
-))
+])
 
 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }
 
 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
     376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])
 
 entities = {
     "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
     "ParseError": 7
 }
 
-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])
 
 
 prefixes = dict([(v, k) for k, v in namespaces.items()])

lib/html5lib/filters/lint.py

@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements
 
@@ -23,24 +20,24 @@ class Filter(_base.Filter):
             if type in ("StartTag", "EmptyTag"):
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                 elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                 if type == "StartTag":
                     open_elements.append(name)
                 for name, value in token["data"]:
                     if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                     if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                     if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                 if name in cdataElements:
                     contentModelFlag = "CDATA"
                 elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
             elif type == "EndTag":
                 name = token["name"]
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                 start_name = open_elements.pop()
                 if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                 contentModelFlag = "PCDATA"
 
             elif type == "Comment":
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")
 
             elif type in ("Characters", "SpaceCharacters"):
                 data = token["data"]
                 if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                 if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                 if type == "SpaceCharacters":
                     data = data.strip(spaceCharacters)
                     if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})
 
             elif type == "Doctype":
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 # XXX: what to do with token["data"] ?
 
             elif type in ("ParseError", "SerializeError"):
                 pass
 
             else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})
 
             yield token

lib/html5lib/html5parser.py

@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E
 
 
 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):
 
         self.framesetOK = True
 
+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
     def isHTMLIntegrationPoint(self, element):
         if (element.name == "annotation-xml" and
                 element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
         # XXX The idea is to make errorcode mandatory.
         self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
         if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)
 
     def normalizeToken(self, token):
         """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
             self.startTagHandler = utils.MethodDispatcher([
                 ("html", self.startTagHtml),
                 (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                  self.startTagProcessInHead),
                 ("body", self.startTagBody),
                 ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
             attributes["name"] = "isindex"
             self.processStartTag(impliedTagToken("input", "StartTag",
                                                  attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
             self.processEndTag(impliedTagToken("label"))
             self.processStartTag(impliedTagToken("hr", "StartTag"))
             self.processEndTag(impliedTagToken("form"))

lib/html5lib/inputstream.py

@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
 
-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
 
 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                   0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):
 
         """
 
-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                 self._bufferedCharacter = data[-1]
                 data = data[:-1]
 
-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)
 
-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")

lib/html5lib/sanitizer.py

@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals
 
 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse
 
 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes
 
 
+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
 
@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                       'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                       'y1', 'y2', 'zoomAndPan']
 
-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']
 
     svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                                'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
     acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                             'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                             'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']
 
     # subclasses may define their own versions of these constants
     allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
     allowed_css_keywords = acceptable_css_keywords
     allowed_svg_properties = acceptable_svg_properties
     allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types
 
     # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
     # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                        unescape(attrs[attr])).lower()
                 # remove replacement characters from unescaped characters
                 val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri and uri.scheme:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
             for attr in self.svg_attr_val_allows_ref:
                 if attr in attrs:
                     attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
             elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                 'padding']:
                 for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                             not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                         break
                 else:

lib/html5lib/serializer/htmlserializer.py

@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type
 
-import gettext
-_ = gettext.gettext
-
 try:
     from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                 v = utils.surrogatePairToCodepoint(v)
             else:
                 v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                 # prefer < over < and similarly for &, >, etc.
                 encode_entity_map[v] = k
 
@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                 if token["systemId"]:
                     if token["systemId"].find('"') >= 0:
                         if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                         quote_char = "'"
                     else:
                         quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                     yield self.encode(token["data"])
                 else:
                     yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 for (attr_namespace, attr_name), attr_value in token["data"].items():
                     # TODO: Add namespace support here
                     k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 yield self.encodeStrict("</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                 yield self.encodeStrict("<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
                 key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                 if self.resolve_entities and key not in xmlEntities:
                     data = entities[key]
                 else:

lib/html5lib/treebuilders/dom.py

@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                         self.dom._child_node_types = list(self.dom._child_node_types)
                         self.dom._child_node_types.append(Node.TEXT_NODE)
                 self.dom.appendChild(self.dom.createTextNode(data))

lib/html5lib/treewalkers/__init__.py

@@ -10,8 +10,12 @@ returning an iterator generating tokens.
 
 from __future__ import absolute_import, division, unicode_literals
 
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys
 
+from .. import constants
 from ..utils import default_etree
 
 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             # XXX: NEVER cache here, caching is done in the etree submodule
             return etree.getETreeModule(implementation, **kwargs).TreeWalker
     return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)

lib/html5lib/treewalkers/_base.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types
 
-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]
 
 from xml.dom import Node
 
@@ -58,7 +58,7 @@ class TreeWalker(object):
                "namespace": to_text(namespace),
                "data": attrs}
         if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")
 
     def startTag(self, namespace, name, attrs):
         assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
         return {"type": "Entity", "name": text_type(name)}
 
     def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)
 
 
 class NonRecursiveTreeWalker(TreeWalker):

lib/html5lib/treewalkers/dom.py

@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals
 
 from xml.dom import Node
 
-import gettext
-_ = gettext.gettext
-
 from . import _base
 
 

lib/html5lib/treewalkers/etree.py

@@ -7,12 +7,10 @@ except ImportError:
         from ordereddict import OrderedDict
     except ImportError:
         OrderedDict = dict
-import gettext
-_ = gettext.gettext
 
 import re
 
-from six import text_type
+from six import string_types
 
 from . import _base
 from ..utils import moduleFactoryFactory
@@ -60,7 +58,7 @@ def getETreeBuilder(ElementTreeImplementation):
                 return _base.COMMENT, node.text
 
             else:
-                assert type(node.tag) == text_type, type(node.tag)
+                assert isinstance(node.tag, string_types), type(node.tag)
                 # This is assumed to be an ordinary element
                 match = tag_regexp.match(node.tag)
                 if match:

lib/html5lib/treewalkers/lxmletree.py

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 
 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             return _base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"
 
         assert len(node) or node.text, "Node has no children"
         if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNextSibling(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                 # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getParentNode(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 return node
             # else: fallback to "normal" processing

lib/html5lib/utils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals
 
 from types import ModuleType
 
+from six import text_type
+
 try:
     import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:
 
 
 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True
 
 
 class MethodDispatcher(dict):

lib/requests/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2014 Kenneth Reitz
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

lib/requests/NOTICE

@@ -1,54 +0,0 @@
-Requests includes some vendorized python libraries to ease installation.
-
-Urllib3 License
-===============
-
-This is the MIT license: http://www.opensource.org/licenses/mit-license.php
-
-Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt),
-Modifications copyright 2012 Kenneth Reitz.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-Chardet License
-===============
-
-This library is free software; you can redistribute it and/or
-modify it under the terms of the GNU Lesser General Public
-License as published by the Free Software Foundation; either
-version 2.1 of the License, or (at your option) any later version.
-
-This library is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-02110-1301  USA
-
-
-CA Bundle License
-=================
-
-This Source Code Form is subject to the terms of the Mozilla Public
-License, v. 2.0. If a copy of the MPL was not distributed with this
-file, You can obtain one at http://mozilla.org/MPL/2.0/.

lib/requests/README.rst

@@ -1,85 +0,0 @@
-Requests: HTTP for Humans
-=========================
-
-.. image:: https://badge.fury.io/py/requests.png
-    :target: http://badge.fury.io/py/requests
-
-.. image:: https://pypip.in/d/requests/badge.png
-        :target: https://crate.io/packages/requests/
-
-
-Requests is an Apache2 Licensed HTTP library, written in Python, for human
-beings.
-
-Most existing Python modules for sending HTTP requests are extremely
-verbose and cumbersome. Python's builtin urllib2 module provides most of
-the HTTP capabilities you should need, but the api is thoroughly broken.
-It requires an enormous amount of work (even method overrides) to
-perform the simplest of tasks.
-
-Things shouldn't be this way. Not in Python.
-
-.. code-block:: python
-
-    >>> r = requests.get('https://api.github.com', auth=('user', 'pass'))
-    >>> r.status_code
-    204
-    >>> r.headers['content-type']
-    'application/json'
-    >>> r.text
-    ...
-
-See `the same code, without Requests <https://gist.github.com/973705>`_.
-
-Requests allow you to send HTTP/1.1 requests. You can add headers, form data,
-multipart files, and parameters with simple Python dictionaries, and access the
-response data in the same way. It's powered by httplib and `urllib3
-<https://github.com/shazow/urllib3>`_, but it does all the hard work and crazy
-hacks for you.
-
-
-Features
---------
-
-- International Domains and URLs
-- Keep-Alive & Connection Pooling
-- Sessions with Cookie Persistence
-- Browser-style SSL Verification
-- Basic/Digest Authentication
-- Elegant Key/Value Cookies
-- Automatic Decompression
-- Unicode Response Bodies
-- Multipart File Uploads
-- Connection Timeouts
-- Thread-safety
-- HTTP(S) proxy support
-
-
-Installation
-------------
-
-To install Requests, simply:
-
-.. code-block:: bash
-
-    $ pip install requests
-
-
-Documentation
--------------
-
-Documentation is available at http://docs.python-requests.org/.
-
-
-Contribute
-----------
-
-#. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet.
-#. If you feel uncomfortable or uncertain about an issue or your changes, feel free to email @sigmavirus24 and he will happily help you via email, Skype, remote pairing or whatever you are comfortable with.
-#. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it).
-#. Write a test which shows that the bug was fixed or that the feature works as expected.
-#. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_.
-
-.. _`the repository`: http://github.com/kennethreitz/requests
-.. _AUTHORS: https://github.com/kennethreitz/requests/blob/master/AUTHORS.rst
-.. _Contributor Friendly: https://github.com/kennethreitz/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open

lib/requests/__init__.py

@@ -6,7 +6,7 @@
 #          /
 
 """
-requests HTTP library
+Requests HTTP library
 ~~~~~~~~~~~~~~~~~~~~~
 
 Requests is an HTTP library, written in Python, for human beings. Basic GET
@@ -36,17 +36,17 @@ usage:
 The other HTTP methods are supported - see `requests.api`. Full documentation
 is at <http://python-requests.org>.
 
-:copyright: (c) 2014 by Kenneth Reitz.
+:copyright: (c) 2015 by Kenneth Reitz.
 :license: Apache 2.0, see LICENSE for more details.
 
 """
 
 __title__ = 'requests'
-__version__ = '2.5.1'
-__build__ = 0x020501
+__version__ = '2.7.0'
+__build__ = 0x020700
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
-__copyright__ = 'Copyright 2014 Kenneth Reitz'
+__copyright__ = 'Copyright 2015 Kenneth Reitz'
 
 # Attempt to enable urllib3's SNI support, if possible
 try:

lib/requests/adapters.py

@@ -11,13 +11,14 @@ and maintain connections.
 import socket
 
 from .models import Response
-from .packages.urllib3 import Retry
 from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
 from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.util import Timeout as TimeoutSauce
+from .packages.urllib3.util.retry import Retry
 from .compat import urlparse, basestring
 from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
-                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth)
+                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth,
+                    select_proxy)
 from .structures import CaseInsensitiveDict
 from .packages.urllib3.exceptions import ConnectTimeoutError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
@@ -35,6 +36,7 @@ from .auth import _basic_auth_str
 DEFAULT_POOLBLOCK = False
 DEFAULT_POOLSIZE = 10
 DEFAULT_RETRIES = 0
+DEFAULT_POOL_TIMEOUT = None
 
 
 class BaseAdapter(object):
@@ -237,8 +239,7 @@ class HTTPAdapter(BaseAdapter):
         :param url: The URL to connect to.
         :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
         """
-        proxies = proxies or {}
-        proxy = proxies.get(urlparse(url.lower()).scheme)
+        proxy = select_proxy(url, proxies)
 
         if proxy:
             proxy = prepend_scheme_if_needed(proxy, 'http')
@@ -271,12 +272,10 @@ class HTTPAdapter(BaseAdapter):
         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 
         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
-        :param proxies: A dictionary of schemes to proxy URLs.
+        :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
         """
-        proxies = proxies or {}
+        proxy = select_proxy(request.url, proxies)
         scheme = urlparse(request.url).scheme
-        proxy = proxies.get(scheme)
-
         if proxy and scheme != 'https':
             url = urldefragauth(request.url)
         else:
@@ -309,7 +308,6 @@ class HTTPAdapter(BaseAdapter):
         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 
         :param proxies: The url of the proxy being used for this request.
-        :param kwargs: Optional additional keyword arguments.
         """
         headers = {}
         username, password = get_auth_from_url(proxy)
@@ -326,8 +324,8 @@ class HTTPAdapter(BaseAdapter):
         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
         :param stream: (optional) Whether to stream the request content.
         :param timeout: (optional) How long to wait for the server to send
-            data before giving up, as a float, or a (`connect timeout, read
-            timeout <user/advanced.html#timeouts>`_) tuple.
+            data before giving up, as a float, or a :ref:`(connect timeout,
+            read timeout) <timeouts>` tuple.
         :type timeout: float or tuple
         :param verify: (optional) Whether to verify SSL certificates.
         :param cert: (optional) Any user-provided SSL certificate to be trusted.
@@ -375,7 +373,7 @@ class HTTPAdapter(BaseAdapter):
                 if hasattr(conn, 'proxy_pool'):
                     conn = conn.proxy_pool
 
-                low_conn = conn._get_conn(timeout=timeout)
+                low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
 
                 try:
                     low_conn.putrequest(request.method,
@@ -407,9 +405,6 @@ class HTTPAdapter(BaseAdapter):
                     # Then, reraise so that we can handle the actual exception.
                     low_conn.close()
                     raise
-                else:
-                    # All is well, return the connection to the pool.
-                    conn._put_conn(low_conn)
 
         except (ProtocolError, socket.error) as err:
             raise ConnectionError(err, request=request)

lib/requests/api.py

@@ -16,7 +16,6 @@ from . import sessions
 
 def request(method, url, **kwargs):
     """Constructs and sends a :class:`Request <Request>`.
-    Returns :class:`Response <Response>` object.
 
     :param method: method for the new :class:`Request` object.
     :param url: URL for the new :class:`Request` object.
@@ -28,8 +27,8 @@ def request(method, url, **kwargs):
     :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload.
     :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
     :param timeout: (optional) How long to wait for the server to send data
-        before giving up, as a float, or a (`connect timeout, read timeout
-        <user/advanced.html#timeouts>`_) tuple.
+        before giving up, as a float, or a :ref:`(connect timeout, read
+        timeout) <timeouts>` tuple.
     :type timeout: float or tuple
     :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
     :type allow_redirects: bool
@@ -37,6 +36,8 @@ def request(method, url, **kwargs):
     :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
     :param stream: (optional) if ``False``, the response content will be immediately downloaded.
     :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
 
     Usage::
 
@@ -54,22 +55,27 @@ def request(method, url, **kwargs):
     return response
 
 
-def get(url, **kwargs):
-    """Sends a GET request. Returns :class:`Response` object.
+def get(url, params=None, **kwargs):
+    """Sends a GET request.
 
     :param url: URL for the new :class:`Request` object.
+    :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', True)
-    return request('get', url, **kwargs)
+    return request('get', url, params=params, **kwargs)
 
 
 def options(url, **kwargs):
-    """Sends a OPTIONS request. Returns :class:`Response` object.
+    """Sends a OPTIONS request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', True)
@@ -77,10 +83,12 @@ def options(url, **kwargs):
 
 
 def head(url, **kwargs):
-    """Sends a HEAD request. Returns :class:`Response` object.
+    """Sends a HEAD request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', False)
@@ -88,44 +96,52 @@ def head(url, **kwargs):
 
 
 def post(url, data=None, json=None, **kwargs):
-    """Sends a POST request. Returns :class:`Response` object.
+    """Sends a POST request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param json: (optional) json data to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('post', url, data=data, json=json, **kwargs)
 
 
 def put(url, data=None, **kwargs):
-    """Sends a PUT request. Returns :class:`Response` object.
+    """Sends a PUT request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('put', url, data=data, **kwargs)
 
 
 def patch(url, data=None, **kwargs):
-    """Sends a PATCH request. Returns :class:`Response` object.
+    """Sends a PATCH request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('patch', url,  data=data, **kwargs)
 
 
 def delete(url, **kwargs):
-    """Sends a DELETE request. Returns :class:`Response` object.
+    """Sends a DELETE request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('delete', url, **kwargs)

lib/requests/auth.py

@@ -103,7 +103,8 @@ class HTTPDigestAuth(AuthBase):
         # XXX not implemented yet
         entdig = None
         p_parsed = urlparse(url)
-        path = p_parsed.path
+        #: path is request-uri defined in RFC 2616 which should not be empty
+        path = p_parsed.path or "/"
         if p_parsed.query:
             path += '?' + p_parsed.query
 
@@ -124,13 +125,15 @@ class HTTPDigestAuth(AuthBase):
         s += os.urandom(8)
 
         cnonce = (hashlib.sha1(s).hexdigest()[:16])
-        noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, HA2)
         if _algorithm == 'MD5-SESS':
             HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce))
 
         if qop is None:
             respdig = KD(HA1, "%s:%s" % (nonce, HA2))
         elif qop == 'auth' or 'auth' in qop.split(','):
+            noncebit = "%s:%s:%s:%s:%s" % (
+                nonce, ncvalue, cnonce, 'auth', HA2
+                )
             respdig = KD(HA1, noncebit)
         else:
             # XXX handle auth-int.
@@ -176,7 +179,7 @@ class HTTPDigestAuth(AuthBase):
             # Consume content and release the original connection
             # to allow our new request to reuse the same one.
             r.content
-            r.raw.release_conn()
+            r.close()
             prep = r.request.copy()
             extract_cookies_to_jar(prep._cookies, r.request, r.raw)
             prep.prepare_cookies(prep._cookies)

lib/requests/compat.py

@@ -21,58 +21,6 @@ is_py2 = (_ver[0] == 2)
 #: Python 3.x?
 is_py3 = (_ver[0] == 3)
 
-#: Python 3.0.x
-is_py30 = (is_py3 and _ver[1] == 0)
-
-#: Python 3.1.x
-is_py31 = (is_py3 and _ver[1] == 1)
-
-#: Python 3.2.x
-is_py32 = (is_py3 and _ver[1] == 2)
-
-#: Python 3.3.x
-is_py33 = (is_py3 and _ver[1] == 3)
-
-#: Python 3.4.x
-is_py34 = (is_py3 and _ver[1] == 4)
-
-#: Python 2.7.x
-is_py27 = (is_py2 and _ver[1] == 7)
-
-#: Python 2.6.x
-is_py26 = (is_py2 and _ver[1] == 6)
-
-#: Python 2.5.x
-is_py25 = (is_py2 and _ver[1] == 5)
-
-#: Python 2.4.x
-is_py24 = (is_py2 and _ver[1] == 4)   # I'm assuming this is not by choice.
-
-
-# ---------
-# Platforms
-# ---------
-
-
-# Syntax sugar.
-_ver = sys.version.lower()
-
-is_pypy = ('pypy' in _ver)
-is_jython = ('jython' in _ver)
-is_ironpython = ('iron' in _ver)
-
-# Assume CPython, if nothing else.
-is_cpython = not any((is_pypy, is_jython, is_ironpython))
-
-# Windows-based system.
-is_windows = 'win32' in str(sys.platform).lower()
-
-# Standard Linux 2+ system.
-is_linux = ('linux' in str(sys.platform).lower())
-is_osx = ('darwin' in str(sys.platform).lower())
-is_hpux = ('hpux' in str(sys.platform).lower())   # Complete guess.
-is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
-
 try:
     import simplejson as json
 except (ImportError, SyntaxError):
@@ -99,7 +47,6 @@ if is_py2:
     basestring = basestring
     numeric_types = (int, long, float)
 
-
 elif is_py3:
     from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
     from urllib.request import parse_http_list, getproxies, proxy_bypass

lib/requests/cookies.py

@@ -6,6 +6,7 @@ Compatibility code to be able to use `cookielib.CookieJar` with requests.
 requests.utils imports from here, so be careful with imports.
 """
 
+import copy
 import time
 import collections
 from .compat import cookielib, urlparse, urlunparse, Morsel
@@ -157,26 +158,28 @@ class CookieConflictError(RuntimeError):
 
 
 class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
-    """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface.
+    """Compatibility class; is a cookielib.CookieJar, but exposes a dict
+    interface.
 
     This is the CookieJar we create by default for requests and sessions that
     don't specify one, since some clients may expect response.cookies and
     session.cookies to support dict operations.
 
-    Don't use the dict interface internally; it's just for compatibility with
-    with external client code. All `requests` code should work out of the box
-    with externally provided instances of CookieJar, e.g., LWPCookieJar and
-    FileCookieJar.
-
-    Caution: dictionary operations that are normally O(1) may be O(n).
+    Requests does not use the dict interface internally; it's just for
+    compatibility with external client code. All requests code should work
+    out of the box with externally provided instances of ``CookieJar``, e.g.
+    ``LWPCookieJar`` and ``FileCookieJar``.
 
     Unlike a regular CookieJar, this class is pickleable.
-    """
 
+    .. warning:: dictionary operations that are normally O(1) may be O(n).
+    """
     def get(self, name, default=None, domain=None, path=None):
         """Dict-like get() that also supports optional domain and path args in
         order to resolve naming collisions from using one cookie jar over
-        multiple domains. Caution: operation is O(n), not O(1)."""
+        multiple domains.
+
+        .. warning:: operation is O(n), not O(1)."""
         try:
             return self._find_no_duplicates(name, domain, path)
         except KeyError:
@@ -199,37 +202,38 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         return c
 
     def iterkeys(self):
-        """Dict-like iterkeys() that returns an iterator of names of cookies from the jar.
-        See itervalues() and iteritems()."""
+        """Dict-like iterkeys() that returns an iterator of names of cookies
+        from the jar. See itervalues() and iteritems()."""
         for cookie in iter(self):
             yield cookie.name
 
     def keys(self):
-        """Dict-like keys() that returns a list of names of cookies from the jar.
-        See values() and items()."""
+        """Dict-like keys() that returns a list of names of cookies from the
+        jar. See values() and items()."""
         return list(self.iterkeys())
 
     def itervalues(self):
-        """Dict-like itervalues() that returns an iterator of values of cookies from the jar.
-        See iterkeys() and iteritems()."""
+        """Dict-like itervalues() that returns an iterator of values of cookies
+        from the jar. See iterkeys() and iteritems()."""
         for cookie in iter(self):
             yield cookie.value
 
     def values(self):
-        """Dict-like values() that returns a list of values of cookies from the jar.
-        See keys() and items()."""
+        """Dict-like values() that returns a list of values of cookies from the
+        jar. See keys() and items()."""
         return list(self.itervalues())
 
     def iteritems(self):
-        """Dict-like iteritems() that returns an iterator of name-value tuples from the jar.
-        See iterkeys() and itervalues()."""
+        """Dict-like iteritems() that returns an iterator of name-value tuples
+        from the jar. See iterkeys() and itervalues()."""
         for cookie in iter(self):
             yield cookie.name, cookie.value
 
     def items(self):
-        """Dict-like items() that returns a list of name-value tuples from the jar.
-        See keys() and values(). Allows client-code to call "dict(RequestsCookieJar)
-        and get a vanilla python dict of key value pairs."""
+        """Dict-like items() that returns a list of name-value tuples from the
+        jar. See keys() and values(). Allows client-code to call
+        ``dict(RequestsCookieJar)`` and get a vanilla python dict of key value
+        pairs."""
         return list(self.iteritems())
 
     def list_domains(self):
@@ -259,8 +263,9 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         return False  # there is only one domain in jar
 
     def get_dict(self, domain=None, path=None):
-        """Takes as an argument an optional domain and path and returns a plain old
-        Python dict of name-value pairs of cookies that meet the requirements."""
+        """Takes as an argument an optional domain and path and returns a plain
+        old Python dict of name-value pairs of cookies that meet the
+        requirements."""
         dictionary = {}
         for cookie in iter(self):
             if (domain is None or cookie.domain == domain) and (path is None
@@ -269,21 +274,24 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         return dictionary
 
     def __getitem__(self, name):
-        """Dict-like __getitem__() for compatibility with client code. Throws exception
-        if there are more than one cookie with name. In that case, use the more
-        explicit get() method instead. Caution: operation is O(n), not O(1)."""
+        """Dict-like __getitem__() for compatibility with client code. Throws
+        exception if there are more than one cookie with name. In that case,
+        use the more explicit get() method instead.
+
+        .. warning:: operation is O(n), not O(1)."""
 
         return self._find_no_duplicates(name)
 
     def __setitem__(self, name, value):
-        """Dict-like __setitem__ for compatibility with client code. Throws exception
-        if there is already a cookie of that name in the jar. In that case, use the more
-        explicit set() method instead."""
+        """Dict-like __setitem__ for compatibility with client code. Throws
+        exception if there is already a cookie of that name in the jar. In that
+        case, use the more explicit set() method instead."""
 
         self.set(name, value)
 
     def __delitem__(self, name):
-        """Deletes a cookie given a name. Wraps cookielib.CookieJar's remove_cookie_by_name()."""
+        """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s
+        ``remove_cookie_by_name()``."""
         remove_cookie_by_name(self, name)
 
     def set_cookie(self, cookie, *args, **kwargs):
@@ -295,15 +303,16 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         """Updates this jar with cookies from another CookieJar or dict-like"""
         if isinstance(other, cookielib.CookieJar):
             for cookie in other:
-                self.set_cookie(cookie)
+                self.set_cookie(copy.copy(cookie))
         else:
             super(RequestsCookieJar, self).update(other)
 
     def _find(self, name, domain=None, path=None):
-        """Requests uses this method internally to get cookie values. Takes as args name
-        and optional domain and path. Returns a cookie.value. If there are conflicting cookies,
-        _find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown
-        if there are conflicting cookies."""
+        """Requests uses this method internally to get cookie values. Takes as
+        args name and optional domain and path. Returns a cookie.value. If
+        there are conflicting cookies, _find arbitrarily chooses one. See
+        _find_no_duplicates if you want an exception thrown if there are
+        conflicting cookies."""
         for cookie in iter(self):
             if cookie.name == name:
                 if domain is None or cookie.domain == domain:
@@ -313,10 +322,11 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
 
     def _find_no_duplicates(self, name, domain=None, path=None):
-        """__get_item__ and get call _find_no_duplicates -- never used in Requests internally.
-        Takes as args name and optional domain and path. Returns a cookie.value.
-        Throws KeyError if cookie is not found and CookieConflictError if there are
-        multiple cookies that match name and optionally domain and path."""
+        """Both ``__get_item__`` and ``get`` call this function: it's never
+        used elsewhere in Requests. Takes as args name and optional domain and
+        path. Returns a cookie.value. Throws KeyError if cookie is not found
+        and CookieConflictError if there are multiple cookies that match name
+        and optionally domain and path."""
         toReturn = None
         for cookie in iter(self):
             if cookie.name == name:
@@ -350,6 +360,21 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         return new_cj
 
 
+def _copy_cookie_jar(jar):
+    if jar is None:
+        return None
+
+    if hasattr(jar, 'copy'):
+        # We're dealing with an instane of RequestsCookieJar
+        return jar.copy()
+    # We're dealing with a generic CookieJar instance
+    new_jar = copy.copy(jar)
+    new_jar.clear()
+    for cookie in jar:
+        new_jar.set_cookie(copy.copy(cookie))
+    return new_jar
+
+
 def create_cookie(name, value, **kwargs):
     """Make a cookie from underspecified parameters.
 
@@ -390,11 +415,14 @@ def morsel_to_cookie(morsel):
 
     expires = None
     if morsel['max-age']:
-        expires = time.time() + morsel['max-age']
+        try:
+            expires = int(time.time() + int(morsel['max-age']))
+        except ValueError:
+            raise TypeError('max-age: %s must be integer' % morsel['max-age'])
     elif morsel['expires']:
         time_template = '%a, %d-%b-%Y %H:%M:%S GMT'
-        expires = time.mktime(
-            time.strptime(morsel['expires'], time_template)) - time.timezone
+        expires = int(time.mktime(
+            time.strptime(morsel['expires'], time_template)) - time.timezone)
     return create_cookie(
         comment=morsel['comment'],
         comment_url=bool(morsel['comment']),
@@ -440,7 +468,7 @@ def merge_cookies(cookiejar, cookies):
     """
     if not isinstance(cookiejar, cookielib.CookieJar):
         raise ValueError('You can only merge into CookieJar')
-    
+
     if isinstance(cookies, dict):
         cookiejar = cookiejar_from_dict(
             cookies, cookiejar=cookiejar, overwrite=False)

lib/requests/models.py

@@ -15,7 +15,7 @@ from .hooks import default_hooks
 from .structures import CaseInsensitiveDict
 
 from .auth import HTTPBasicAuth
-from .cookies import cookiejar_from_dict, get_cookie_header
+from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
 from .packages.urllib3.fields import RequestField
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .packages.urllib3.util import parse_url
@@ -30,7 +30,8 @@ from .utils import (
     iter_slices, guess_json_utf, super_len, to_native_string)
 from .compat import (
     cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
-    is_py2, chardet, json, builtin_str, basestring)
+    is_py2, chardet, builtin_str, basestring)
+from .compat import json as complexjson
 from .status_codes import codes
 
 #: The set of HTTP status codes that indicate an automatically
@@ -42,12 +43,11 @@ REDIRECT_STATI = (
     codes.temporary_redirect, # 307
     codes.permanent_redirect, # 308
 )
+
 DEFAULT_REDIRECT_LIMIT = 30
 CONTENT_CHUNK_SIZE = 10 * 1024
 ITER_CHUNK_SIZE = 512
 
-json_dumps = json.dumps
-
 
 class RequestEncodingMixin(object):
     @property
@@ -143,13 +143,13 @@ class RequestEncodingMixin(object):
             else:
                 fn = guess_filename(v) or k
                 fp = v
-            if isinstance(fp, str):
-                fp = StringIO(fp)
-            if isinstance(fp, bytes):
-                fp = BytesIO(fp)
 
-            rf = RequestField(name=k, data=fp.read(),
-                              filename=fn, headers=fh)
+            if isinstance(fp, (str, bytes, bytearray)):
+                fdata = fp
+            else:
+                fdata = fp.read()
+
+            rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
             rf.make_multipart(content_type=ft)
             new_fields.append(rf)
 
@@ -206,17 +206,8 @@ class Request(RequestHooksMixin):
       <PreparedRequest [GET]>
 
     """
-    def __init__(self,
-        method=None,
-        url=None,
-        headers=None,
-        files=None,
-        data=None,
-        params=None,
-        auth=None,
-        cookies=None,
-        hooks=None,
-        json=None):
+    def __init__(self, method=None, url=None, headers=None, files=None,
+        data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
 
         # Default empty dicts for dict params.
         data = [] if data is None else data
@@ -295,8 +286,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.hooks = default_hooks()
 
     def prepare(self, method=None, url=None, headers=None, files=None,
-                data=None, params=None, auth=None, cookies=None, hooks=None,
-                json=None):
+        data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
         """Prepares the entire request with the given parameters."""
 
         self.prepare_method(method)
@@ -305,6 +295,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.prepare_cookies(cookies)
         self.prepare_body(data, files, json)
         self.prepare_auth(auth, url)
+
         # Note that prepare_auth must be last to enable authentication schemes
         # such as OAuth to work on a fully prepared request.
 
@@ -319,7 +310,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         p.method = self.method
         p.url = self.url
         p.headers = self.headers.copy() if self.headers is not None else None
-        p._cookies = self._cookies.copy() if self._cookies is not None else None
+        p._cookies = _copy_cookie_jar(self._cookies)
         p.body = self.body
         p.hooks = self.hooks
         return p
@@ -356,8 +347,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             raise InvalidURL(*e.args)
 
         if not scheme:
-            raise MissingSchema("Invalid URL {0!r}: No schema supplied. "
-                                "Perhaps you meant http://{0}?".format(url))
+            error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
+            error = error.format(to_native_string(url, 'utf8'))
+
+            raise MissingSchema(error)
 
         if not host:
             raise InvalidURL("Invalid URL %r: No host supplied" % url)
@@ -423,7 +416,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
 
         if json is not None:
             content_type = 'application/json'
-            body = json_dumps(json)
+            body = complexjson.dumps(json)
 
         is_stream = all([
             hasattr(data, '__iter__'),
@@ -500,7 +493,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             self.prepare_content_length(self.body)
 
     def prepare_cookies(self, cookies):
-        """Prepares the given HTTP cookie data."""
+        """Prepares the given HTTP cookie data.
+
+        This function eventually generates a ``Cookie`` header from the
+        given cookies using cookielib. Due to cookielib's design, the header
+        will not be regenerated if it already exists, meaning this function
+        can only be called once for the life of the
+        :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
+        to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
+        header is removed beforehand."""
 
         if isinstance(cookies, cookielib.CookieJar):
             self._cookies = cookies
@@ -513,6 +514,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
 
     def prepare_hooks(self, hooks):
         """Prepares the given hooks."""
+        # hooks can be passed as None to the prepare method and to this
+        # method. To prevent iterating over None, simply use an empty list
+        # if hooks is False-y
+        hooks = hooks or []
         for event in hooks:
             self.register_hook(event, hooks[event])
 
@@ -523,16 +528,8 @@ class Response(object):
     """
 
     __attrs__ = [
-        '_content',
-        'status_code',
-        'headers',
-        'url',
-        'history',
-        'encoding',
-        'reason',
-        'cookies',
-        'elapsed',
-        'request',
+        '_content', 'status_code', 'headers', 'url', 'history',
+        'encoding', 'reason', 'cookies', 'elapsed', 'request'
     ]
 
     def __init__(self):
@@ -572,7 +569,11 @@ class Response(object):
         self.cookies = cookiejar_from_dict({})
 
         #: The amount of time elapsed between sending the request
-        #: and the arrival of the response (as a timedelta)
+        #: and the arrival of the response (as a timedelta).
+        #: This property specifically measures the time taken between sending
+        #: the first byte of the request and finishing parsing the headers. It
+        #: is therefore unaffected by consuming the response content or the
+        #: value of the ``stream`` keyword argument.
         self.elapsed = datetime.timedelta(0)
 
         #: The :class:`PreparedRequest <PreparedRequest>` object to which this
@@ -648,9 +649,10 @@ class Response(object):
         If decode_unicode is True, content will be decoded using the best
         available encoding based on the response.
         """
+
         def generate():
-            try:
-                # Special case for urllib3.
+            # Special case for urllib3.
+            if hasattr(self.raw, 'stream'):
                 try:
                     for chunk in self.raw.stream(chunk_size, decode_content=True):
                         yield chunk
@@ -660,7 +662,7 @@ class Response(object):
                     raise ContentDecodingError(e)
                 except ReadTimeoutError as e:
                     raise ConnectionError(e)
-            except AttributeError:
+            else:
                 # Standard file-like object.
                 while True:
                     chunk = self.raw.read(chunk_size)
@@ -688,6 +690,8 @@ class Response(object):
         """Iterates over the response data, one line at a time.  When
         stream=True is set on the request, this avoids reading the
         content at once into memory for large responses.
+
+        .. note:: This method is not reentrant safe.
         """
 
         pending = None
@@ -789,14 +793,16 @@ class Response(object):
             encoding = guess_json_utf(self.content)
             if encoding is not None:
                 try:
-                    return json.loads(self.content.decode(encoding), **kwargs)
+                    return complexjson.loads(
+                        self.content.decode(encoding), **kwargs
+                    )
                 except UnicodeDecodeError:
                     # Wrong UTF codec detected; usually because it's not UTF-8
                     # but some other 8-bit codec.  This is an RFC violation,
                     # and the server didn't bother to tell us what codec *was*
                     # used.
                     pass
-        return json.loads(self.text, **kwargs)
+        return complexjson.loads(self.text, **kwargs)
 
     @property
     def links(self):
@@ -822,10 +828,10 @@ class Response(object):
         http_error_msg = ''
 
         if 400 <= self.status_code < 500:
-            http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
+            http_error_msg = '%s Client Error: %s for url: %s' % (self.status_code, self.reason, self.url)
 
         elif 500 <= self.status_code < 600:
-            http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
+            http_error_msg = '%s Server Error: %s for url: %s' % (self.status_code, self.reason, self.url)
 
         if http_error_msg:
             raise HTTPError(http_error_msg, response=self)
@@ -836,4 +842,7 @@ class Response(object):
 
         *Note: Should not normally need to be called explicitly.*
         """
+        if not self._content_consumed:
+            return self.raw.close()
+
         return self.raw.release_conn()

lib/requests/packages/README.rst

@@ -0,0 +1,8 @@
+If you are planning to submit a pull request to requests with any changes in 
+this library do not go any further. These are independent libraries which we 
+vendor into requests. Any changes necessary to these libraries must be made in 
+them and submitted as separate pull requests to those libraries.
+
+urllib3 pull requests go here: https://github.com/shazow/urllib3
+
+chardet pull requests go here: https://github.com/chardet/chardet

lib/requests/packages/urllib3/__init__.py

@@ -55,9 +55,14 @@ def add_stderr_logger(level=logging.DEBUG):
 del NullHandler
 
 
-# Set security warning to only go off once by default.
 import warnings
-warnings.simplefilter('always', exceptions.SecurityWarning)
+# SecurityWarning's always go off by default.
+warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
+# SubjectAltNameWarning's should go off once per host
+warnings.simplefilter('default', exceptions.SubjectAltNameWarning)
+# InsecurePlatformWarning's don't vary between requests, so we keep it default.
+warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
+                      append=True)
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """