Fix for transmission bug introduced in last update

- is executed only if the url is not already ending with rpc
  - detects ports up to 6 digits long (changed from 4)

.gitignore

@@ -15,6 +15,11 @@
 logs/*
 cache/*
 
+# HTTPS Cert/Key #
+##################
+*.crt
+*.key
+
 # OS generated files #
 ######################
 .DS_Store?

Headphones.py

@@ -142,7 +142,7 @@ def main():
     # Force the http port if neccessary
     if args.port:
         http_port = args.port
-        logger.info('Starting Headphones on forced port: %i' % http_port)
+        logger.info('Using forced port: %i' % http_port)
     else:
         http_port = int(headphones.HTTP_PORT)
 
@@ -152,12 +152,13 @@ def main():
                     'http_host':        headphones.HTTP_HOST,
                     'http_root':        headphones.HTTP_ROOT,
                     'http_proxy':       headphones.HTTP_PROXY,
+                    'enable_https':     headphones.ENABLE_HTTPS,
+                    'https_cert':       headphones.HTTPS_CERT,
+                    'https_key':        headphones.HTTPS_KEY,
                     'http_username':    headphones.HTTP_USERNAME,
                     'http_password':    headphones.HTTP_PASSWORD,
             })
 
-    logger.info('Starting Headphones on port: %i' % http_port)
-
     if headphones.LAUNCH_BROWSER and not args.nolaunch:
         headphones.launch_browser(headphones.HTTP_HOST, http_port, headphones.HTTP_ROOT)
 

data/interfaces/default/album.html

@@ -21,7 +21,7 @@
 				<div id="dialog" title="Choose an Alternate Release" style="display:none" class="configtable">
 					<div class="links">
 						<%
-							alternate_albums = myDB.select("SELECT * from allalbums WHERE AlbumID=?", [album['AlbumID']])
+							alternate_albums = myDB.select("SELECT * from allalbums WHERE AlbumID=? ORDER BY ReleaseDate ASC", [album['AlbumID']])
 						%>
 						%if not alternate_albums:
 							<p>No alternate releases found. Try refreshing the artist (if the artist is being refreshed, please wait until it's finished)</p>
@@ -32,9 +32,9 @@
 									track_count = len(myDB.select("SELECT * from alltracks WHERE ReleaseID=?", [alternate_album['ReleaseID']]))
 									have_track_count = len(myDB.select("SELECT * from alltracks WHERE ReleaseID=? AND Location IS NOT NULL", [alternate_album['ReleaseID']]))
 									if alternate_album['AlbumID'] == alternate_album['ReleaseID']:
-										alternate_album_name = "Headphones Default Release [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
+										alternate_album_name = "Headphones Default Release (" + str(alternate_album['ReleaseDate']) + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
 									else:
-										alternate_album_name = alternate_album['AlbumTitle'] + " (" + alternate_album['ReleaseCountry'] + ", " + alternate_album['ReleaseFormat'] + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
+										alternate_album_name = alternate_album['AlbumTitle'] + " (" + alternate_album['ReleaseCountry'] + ", " + str(alternate_album['ReleaseDate']) + ", " + alternate_album['ReleaseFormat'] + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
 									
 								%>
 								<a href="#" onclick="doAjaxCall('switchAlbum?AlbumID=${album['AlbumID']}&ReleaseID=${alternate_album['ReleaseID']}', $(this), 'table');" data-success="Switched release to: ${alternate_album_name}">${alternate_album_name}</a><br>
@@ -138,7 +138,7 @@
 					</tr>
 				%endfor
 				<%
-				unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])			
+				unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched = "Failed" ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']])
 				%>
 				%if unmatched:
 					 %for track in unmatched:

data/interfaces/default/artist.html

@@ -93,7 +93,7 @@
 
 				myDB = db.DBConnection()					
 				totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']]))
-				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]))
+				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']]))
 				
 				try:
 					percent = (havetracks*100.0)/totaltracks

data/interfaces/default/config.html

@@ -54,6 +54,19 @@
                                 <div class="row checkbox">
                                     <input type="checkbox" name="launch_browser" value="1" ${config['launch_browser']} /> <label>Launch Browser on Startup</label>
                                 </div>
+                                <div class="row checkbox">
+                                    <input type="checkbox" name="enable_https" id="enable_https" value="1" ${config['enable_https']} /> <label>Enable HTTPS</label>
+                                </div>
+                                <div id="https_options">
+                                    <div class="row">
+                                        <label>HTTPS Cert</label>
+                                        <input type="text" name="https_cert" value="${config['https_cert']}" size="30">
+                                    </div>
+                                    <div class="row">
+                                        <label>HTTPS Key</label>
+                                        <input type="text" name="https_key" value="${config['https_key']}" size="30">
+                                    </div>
+                                </div>
                             </fieldset>
                         </td>
                         <td>
@@ -82,7 +95,15 @@
                                 </div>
                                 <div class="row">
                                     <label>Library Scan Interval</label>
-                                    <input type="text" name="libraryscan_interval" value="${config['libraryscan_interval']}" size="4">mins
+                                    <input type="text" name="libraryscan_interval" value="${config['libraryscan_interval']}" size="4">hours
+                                </div>
+                                <div class="row">
+                                    <label>MusicBrainz Update Interval</label>
+                                    <input type="text" name="update_db_interval" value="${config['update_db_interval']}" size="4">hours
+                                </div>
+                                <div class="row">
+                                    <label>Ignore Album Updates</label>
+                                    <input type="text" name="mb_ignore_age" value="${config['mb_ignore_age']}" size="4">days
                                 </div>
                             </fieldset>
                         </td>
@@ -341,12 +362,34 @@
                                     </div>
                                 </div>
                             </fieldset>
+                            <fieldset>
+                                <legend>omgwtfnzbs</legend>
+                                <div class="row checkbox">
+                                    <input id="useomgwtfnzbs" type="checkbox" name="omgwtfnzbs" onclick="initConfigCheckbox($(this));" value="1" ${config['use_omgwtfnzbs']} /><label>Use omgwtfnzbs</label>
+                                </div>
+                                <div class="config">
+                                    <div class="row">
+                                        <label>omgwtfnzbs UID</label>
+                                        <input type="text" name="omgwtfnzbs_uid" value="${config['omgwtfnzbs_uid']}" size="10">
+                                    </div>
+                                    <div class="row">
+                                        <label>omgwtfnzbs API Key</label>
+                                        <input type="text" name="omgwtfnzbs_apikey" value="${config['omgwtfnzbs_apikey']}" size="10">
+                                    </div>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
                                 <legend>Torrents</legend>
                                 <div class="row checkbox">
-                                    <input type="checkbox" name="use_piratebay" value="1" ${config['use_piratebay']} /><label>The Pirate Bay</label>
+                                    <input id="usepiratebay" type="checkbox" name="use_piratebay" value="1" ${config['use_piratebay']} /><label>The Pirate Bay</label>
+                                </div>
+                                <div class="config">
+                                    <div class="row">
+                                        <label>Proxy URL (Optional): </label>
+                                        <input type="text" name="piratebay_proxy_url" value="${config['piratebay_proxy_url']}" size="36">
+                                    </div>
                                 </div>
                                 <div class="row checkbox">
                                     <input type="checkbox" name="use_isohunt" value="1" ${config['use_isohunt']} /><label>Isohunt</label>
@@ -367,7 +410,7 @@
                                     </div>
                                     <div class="row">
                                         <label>Waffles Passkey: </label>
-                                        <input type="text" name="waffles_passkey" value="${config['waffles_passkey']}" size="36">
+                                        <input type="password" name="waffles_passkey" value="${config['waffles_passkey']}" size="36">
                                     </div>
                                 </div>
                                 <div class="row checkbox">
@@ -498,6 +541,9 @@
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
                                 <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year</small>
                             </div>
+                            <div class="checkbox row clearfix">
+                                <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
+                            </div>
                         </fieldset>
 
                         <fieldset>
@@ -669,6 +715,14 @@
                                 <div class="row left checkbox">
                                     <input type="checkbox" name="autowant_all" value="1" ${config['autowant_all']} /><label>Automatically Mark All Albums as Wanted</label>
                                 </div>
+                                <div class="row">
+                                    <label>Folder Permissions:</label>
+                                    <input type="text" name="folder_permissions" value="${config['folder_permissions']}" size="7">
+                                </div>
+                                <div class="row">
+                                    <label>File Permissions:</label>
+                                    <input type="text" name="file_permissions" value="${config['file_permissions']}" size="7">
+                                </div>
                                 <div class="row">
                                     <label>Cache Size (in MB):</label>
                                     <input type="text" name="cache_sizemb" value="${config['cache_sizemb']}" size="7">
@@ -944,6 +998,25 @@
                     }
                     $('#api_key').val(data);
             });
+        });
+                if ($("#enable_https").is(":checked"))
+                        {
+                                $("#https_options").show();
+                        }
+                else
+                        {
+                                $("#https_options").hide();
+                        }
+
+                $("#enable_https").click(function(){
+                        if ($("#enable_https").is(":checked"))
+                        {
+                                $("#https_options").slideDown();
+                        }
+                        else
+                        {
+                                $("#https_options").slideUp();
+                        }
                 });
                 if ($("#music_encoder").is(":checked"))
                         {
@@ -1210,10 +1283,13 @@
                 initConfigCheckbox("#usenewznab");
                 initConfigCheckbox("#usenzbsrus");
                 initConfigCheckbox("#usenzbsorg");
+                initConfigCheckbox("#useomgwtfnzbs");
+                initConfigCheckbox("#usepiratebay");
                 initConfigCheckbox("#usewaffles");
                 initConfigCheckbox("#userutracker");
                 initConfigCheckbox("#usewhatcd");
                 initConfigCheckbox("#useapi");
+                initConfigCheckbox("#enable_https");
         }
         $(document).ready(function() {
             initThisPage();

data/interfaces/default/js/script.js

@@ -315,6 +315,7 @@ function doAjaxCall(url,elem,reload,form) {
 	  				console.log('refresh'); refreshTable();
 	  			}
 	  			if ( reload == "tabs") 	refreshTab();
+	  			if ( reload == "page") 	location.reload();
 	  			if ( form ) {
 	  				// Change the option to 'choose...'
 	  				$(formID + " select").children('option[disabled=disabled]').attr('selected','selected');

data/interfaces/default/manage.html

@@ -22,6 +22,7 @@
 			%if not headphones.ADD_ARTISTS:
 			<a class="menu_link_edit" href="manageNew">Manage New Artists</a>
 			%endif
+			<a class="menu_link_edit" href="manageUnmatched">Manage Unmatched</a>
 		</div>
 	</div>	
 </%def>
@@ -35,6 +36,7 @@
 			<li><a href="#tabs-1">Scan Music Library</a></li>
 			<li><a href="#tabs-2">Imports</a></li>
 			<li><a href="#tabs-3">Force Actions</a></li>
+			<li><a href="#tabs-4">Force Legacy</a></li>
 		</ul>
 		<div id="tabs-1" class="configtable">
 			<fieldset>
@@ -119,7 +121,7 @@
 			<legend>Force Search</legend>
 			<div class="links">
 				<a href="#" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><span class="ui-icon ui-icon-search"></span>Force Check for Wanted Albums</a>
-				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists</a>
+				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists [Fast]</a>
 				<a href="#" onclick="doAjaxCall('forcePostProcess',$(this))" data-success="Post-Processor is being loaded" data-error="Error during Post-Processing"><span class="ui-icon ui-icon-wrench"></span>Force Post-Process Albums in Download Folder</a>
 				<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><span class="ui-icon ui-icon-refresh"></span>Check for Headphones Updates</a>
 				<a href="#" id="delete_empty_artists"><span class="ui-icon ui-icon-trash"></span>Delete empty Artists</a>
@@ -139,6 +141,26 @@
 		</fieldset>
 		
 	</div>
+
+
+	<div id="tabs-4" class="configtable">
+		<fieldset>
+			<legend>Force Legacy</legend>
+				<p>Please note that these functions will take a significant amount of time to complete.</p>
+			<div class="links">
+				<a href="#" onclick="doAjaxCall('forceFullUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists [Comprehensive]</a> 
+				<BR>
+				<a href="#" onclick="doAjaxCall('forceScan',$(this))" data-success="Library scan successful" data-error="Error forcing library scan"><span class="ui-icon ui-icon-refresh"></span>Force Re-scan Library [Comprehensive]</a> 
+				<BR>
+				<small>*Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched".</small>
+
+		</div>
+		</fieldset>
+		
+	</div>
+
+
+
 </div>
 </%def>
 <%def name="javascriptIncludes()">

data/interfaces/default/managealbums.html

@@ -57,7 +57,7 @@
 
 				myDB = db.DBConnection()					
 				totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']]))
-				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]))
+				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']]))
 				
 				try:
 					percent = (havetracks*100.0)/totaltracks

data/interfaces/default/managemanual.html

@@ -0,0 +1,121 @@
+<%inherit file="base.html" />
+<%!
+	import headphones
+	from headphones import db, helpers
+	myDB = db.DBConnection()
+%>
+
+<%def name="headerIncludes()">
+	<div id="subhead_container">
+		<div id="subhead_menu">
+		</div>		
+	</div>	
+	<a href="manageUnmatched" class="back">&laquo; Back to Unmatched Albums</a>
+</%def>
+
+
+<%def name="body()">
+	<div class="table_wrapper">
+	<div id="manageheader" class="title">
+		<h1 class="clearfix"><img src="interfaces/default/images/icon_manage.png" alt="manage"/>Manage Manually Changed Albums</h1>
+	</div>
+
+	<table class="display" id="artist_table">
+		<thead>
+			<tr>
+				<th id="artist">Local Artist</th>
+				<th id="album">Local Album</th>
+				<th id="status">Previous Action</th>
+			</tr>
+		</thead>
+		<tbody>
+		<% count_albums=0 %>
+		%for album in manualalbums:
+			<tr class="gradeZ">
+			<%
+				old_artist_clean = album['ArtistName'].replace('&','%26').replace('+', '%2B').replace("'","%27")
+				old_album_clean = album['AlbumTitle'].replace('&','%26').replace('+', '%2B').replace("'","%27")
+			%>
+			<td id="artist">${album['ArtistName']}<BR>
+				<button id="restore_artist${count_albums}" onClick="restore_Artist(this.id)">(<-) Restore Artist</button>
+				<div id="restore_artist_dialog${count_albums}" title="Restore Artist" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to restore Local Artist: ${album['ArtistName']} to unmatched?</td></tr>
+					<tr><td align="right"><BR>
+					%if album['AlbumStatus'] == "Ignored":
+						<button href="#" onclick="doAjaxCall('markManual?action=unignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Restore Artist</button>
+					%elif album['AlbumStatus'] == "Matched":
+						<button href="#" onclick="doAjaxCall('markManual?action=unmatchArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Restore Artist</button>
+					%endif
+					</td></tr>
+					</table>
+				</div>
+			</td>
+			<td id="album">${album['AlbumTitle']}<BR>
+				<button id="restore_album${count_albums}" onClick="restore_Album(this.id)">(<-) Restore Album</button>
+				<div id="restore_album_dialog${count_albums}" title="Restore Album" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to restore Local Album: ${album['AlbumTitle']} to unmatched?</td></tr>
+					<tr><td align="right"><BR>
+					%if album['AlbumStatus'] == "Ignored":
+						<button href="#" onclick="doAjaxCall('markManual?action=unignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully restored ${album['AlbumTitle']} to unmatched">Restore Album</button>
+					%elif album['AlbumStatus'] == "Matched":
+						<button href="#" onclick="doAjaxCall('markManual?action=unmatchAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully restored ${album['AlbumTitle']} to unmatched">Restore Album</button>
+					%endif
+					</td></tr>
+					</table>
+				</div>
+
+			</td>
+			<td id="status">${album['AlbumStatus']}
+
+			</td>
+			</tr>
+		<% count_albums+=1 %>
+		%endfor
+		</tbody>
+	</table>
+	</div>
+</%def>
+
+<%def name="headIncludes()">
+	<link rel="stylesheet" href="interfaces/default/css/data_table.css">
+</%def>
+
+<%def name="javascriptIncludes()">
+	<script src="js/libs/jquery.dataTables.min.js"></script>
+	<script>
+	$(document).ready(function()
+	{
+		$('#artist_table').dataTable(
+			{
+				"bStateSave": true,
+				"bPaginate": true,
+				"oLanguage": {
+					"sSearch": "",
+	                "sLengthMenu":"Show _MENU_ albums per page",
+						"sInfo":"Showing _START_ to _END_ of _TOTAL_ albums",
+						"sInfoEmpty":"Showing 0 to 0 of 0 albums",
+						"sInfoFiltered":"(filtered from _MAX_ total albums)",
+						"sEmptyTable": " ",
+	                    },
+	            "sPaginationType": "full_numbers",
+	         });
+
+initActions();
+	});
+
+	function restore_Artist(clicked_id) {
+		n=clicked_id.replace("restore_artist","");
+		$("#restore_artist_dialog"+n).dialog();
+		return false;
+	}
+
+	function restore_Album(clicked_id) {
+		n=clicked_id.replace("restore_album","");
+		$("#restore_album_dialog"+n).dialog();
+		return false;
+	}
+
+	</script>
+</%def>

data/interfaces/default/managenew.html

@@ -20,7 +20,10 @@
 	</div>
 	<form action="addArtists" method="get">
 	<div id="markalbum">
-		Add selected artists
+	    <select name="action">
+        	<option value="add">(+) ADD Selected Artists</option>
+        	<option value="ignore">(-) IGNORE Selected Artists</option>
+       	</select>
 		<input type="submit" value="Go">
 	</div>
 	<table class="display" id="artist_table">

data/interfaces/default/manageunmatched.html

@@ -0,0 +1,259 @@
+<%inherit file="base.html" />
+<%!
+	import headphones
+	import json
+	from headphones import db, helpers
+	myDB = db.DBConnection()
+	artist_json = {}
+	counter = 0
+	artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE")
+	for artist in artist_list:
+		artist_json[counter] = artist['ArtistName'] 
+		counter+=1
+	json_artists = json.dumps(artist_json)
+%>
+
+<%def name="headerIncludes()">
+	<div id="subhead_container">
+		<div id="subhead_menu">
+		<a class="menu_link_edit" href="manageManual">Manage Manually Matched & Ignored</a>
+		</div>		
+	</div>	
+	<a href="manage" class="back">&laquo; Back to manage overview</a>
+</%def>
+
+
+<%def name="body()">
+	<div class="table_wrapper">
+	<div id="manageheader" class="title">
+		<h1 class="clearfix"><img src="interfaces/default/images/icon_manage.png" alt="manage"/>Manage Unmatched Albums</h1>
+	</div>
+
+	<table class="display" id="artist_table">
+		<thead>
+			<tr>
+				<th id="artist">Local Artist</th>
+				<th id="album">Local Album</th>
+			</tr>
+		</thead>
+		<tbody>
+		<% count_albums=0 %>
+		%for album in unmatchedalbums:
+			<tr class="gradeZ">
+				<% 
+					old_artist_clean = album['ArtistName'].replace('&','%26').replace("'","%27")
+					old_album_clean = album['AlbumTitle'].replace('&','%26').replace("'","%27")
+					old_artist_js = album['ArtistName'].replace("'","\\'").replace('"','\\"')
+					old_album_js = album['AlbumTitle'].replace("'","\\'").replace('"','\\"')
+				%>
+				<td id="artist">${album['ArtistName']}<BR>
+				<button id="ignore_artists${count_albums}" onClick="ignore_Artist(this.id)">(-) Ignore Artist</button>
+				<div id="ignore_artist_dialog${count_albums}" title="Ignore Artist" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to ignore Local Artist: ${album['ArtistName']} from future matching?</td></tr>
+					<tr><td align="right"><BR>
+					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully ignored ${album['ArtistName']} from future matching">Ignore Artist</button>
+					</td></tr>
+					</table>
+				</div>
+
+				<button id="match_artists${count_albums}" onClick="load_Artist(this.id)">(->) Match Artist</button>
+				<div id="artist_dialog${count_albums}" title="Match Artist" style="display:none">
+					<table width=400>
+					<tr><td>Local Artist:</td><td>${album['ArtistName']}</td></tr>
+						<tr><td>Match Artist</td><td>
+						<select id="artist_options${count_albums}" name="new_artist">
+						</select>
+						</td></tr>
+					<tr><td></td><td align="right"><BR>
+					<button href="#" onclick="artist_matcher(${count_albums}, '${old_artist_js}')">Match Artist</button>
+					</td></tr>
+					</table>
+				</div>
+
+				</td>
+				<td id="album">${album['AlbumTitle']}<BR>
+				<button id="ignore_albums${count_albums}" onClick="ignore_Album(this.id)">(-) Ignore Album</button>
+				<div id="ignore_album_dialog${count_albums}" title="Ignore Album" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to ignore Local Album: ${album['AlbumTitle']} from future matching?</td></tr>
+					<tr><td align="right"><BR>
+					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully ignored ${album['AlbumTitle']} from future matching">Ignore Album</button>
+					</td></tr>
+					</table>
+				</div>
+
+				<button id="match_albums${count_albums}" onClick="load_AlbumArtist(this.id)">(->) Match Album</button>
+				<div id="album_dialog${count_albums}" title="Match Album" style="display:none">
+					<table width=650>
+					<tr><td>Local Artist:</td><td>${album['ArtistName']}</td></tr>
+					<tr><td>Local Album:</td><td>${album['AlbumTitle']}</td></tr>
+					<tr><td>Match Artist</td><td>
+					<select id="album_artist_options${count_albums}" name="new_artist">
+					</select>
+					</td></tr>
+					<tr><td>Match Album</td><td>
+					<select id="album_options${count_albums}" name="new_album">
+					</select>
+					</td></tr>
+					<tr><td></td><td align="right"><BR>
+					<button href="#" onclick="album_matcher(${count_albums}, '${old_artist_js}', '${old_album_js}')">Match Album</button>
+					</td></tr>
+					</table>
+				</div>
+
+				</td>
+			</tr>
+		<% count_albums+=1 %>
+		%endfor
+		</tbody>
+	</table>
+	</div>
+</%def>
+
+<%def name="headIncludes()">
+	<link rel="stylesheet" href="interfaces/default/css/data_table.css">
+</%def>
+
+<%def name="javascriptIncludes()">
+	<script src="js/libs/jquery.dataTables.min.js"></script>
+	<script>
+	$(document).ready(function()
+	{
+		$('#artist_table').dataTable(
+			{
+				"bStateSave": true,
+				"bPaginate": true,
+				"oLanguage": {
+					"sSearch": "",
+	                "sLengthMenu":"Show _MENU_ albums per page",
+						"sInfo":"Showing _START_ to _END_ of _TOTAL_ albums",
+						"sInfoEmpty":"Showing 0 to 0 of 0 albums",
+						"sInfoFiltered":"(filtered from _MAX_ total albums)",
+						"sEmptyTable": " ",
+	                    },
+	            "sPaginationType": "full_numbers",
+	         });
+
+initActions();
+	});
+
+	function ignore_Artist(clicked_id) {
+		n=clicked_id.replace("ignore_artists","");
+		$("#ignore_artist_dialog"+n).dialog();
+		return false;
+	}
+
+	function ignore_Album(clicked_id) {
+		n=clicked_id.replace("ignore_albums","");
+		$("#ignore_album_dialog"+n).dialog();
+		return false;
+	}
+
+
+	function load_Artist(clicked_id) {
+		n=clicked_id.replace("match_artists","");
+		var d = $("#artist_dialog"+n).dialog();
+    	d.dialog("option", "width", 450);
+    	d.dialog("option", "position", "center");
+		$('#artist_options'+n).html('');
+		$.each(${json_artists}, function(key, value) {
+    		$('#artist_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    		}));
+		});
+		change_artist(n)
+		return false;
+	}
+
+	function change_artist(n) {
+		selected_artist = $("#artist_options"+n).find("option:selected").text();
+		selected_artist_clean = selected_artist.replace('&', '%26').replace('+', '%2B');
+		$("#artist_options"+n).change(function() {
+			selected_artist = $("#artist_options"+n).find("option:selected").text();
+			selected_artist_clean = selected_artist.replace('&', '%26').replace('+', '%2B');
+			});
+	}
+
+	function artist_matcher(n, existing_artist) {
+		var existing_artist = existing_artist.toString();
+		var existing_artist_clean = existing_artist.replace('&', '%26').replace('+', '%2B');
+		$('#match_artists'+n).attr('data-success', 'Successfully matched '+existing_artist+' with '+selected_artist);
+		doAjaxCall('markUnmatched?action=matchArtist&existing_artist='+existing_artist_clean+'&new_artist='+selected_artist_clean, $('#match_artists'+n), 'page'); 
+		}
+
+
+	function load_AlbumArtist(clicked_id) {
+		n=clicked_id.replace("match_albums","");
+		var d = $("#album_dialog"+n).dialog();
+    	d.dialog("option", "width", 700);
+    	d.dialog("option", "position", "center");
+    	$('#album_artist_options'+n).html('');
+		$.each(${json_artists}, function(key, value) {
+    		$('#album_artist_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    		}));
+		});
+		load_json_albums(n)
+		return false;
+	}
+
+
+	function load_json_albums(n) {
+		selected_album_artist = $("#album_artist_options"+n).find("option:selected").text();
+		selected_album_artist_clean = selected_album_artist.replace('&', '%26').replace('+', '%2B');
+		$('#album_options'+n).html('')
+		$.getJSON("getAlbumsByArtist_json?artist="+selected_album_artist_clean, function( data ) {
+			$.each( data, function( key, value ) {
+			    $('#album_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    			}));
+			});
+		selected_album= $("#album_options"+n).find("option:selected").text();
+		selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+		});
+		change_json_albums(n)
+		change_album(n)
+	}
+
+
+	function change_json_albums(n) {
+		$("#album_artist_options"+n).change(function(){
+			selected_album_artist = $("#album_artist_options"+n).find("option:selected").text();
+			selected_album_artist_clean = selected_album_artist.replace('&', '%26').replace('+', '%2B');
+			$('#album_options'+n).html('')
+			$.getJSON("getAlbumsByArtist_json?artist="+selected_album_artist_clean, function( data ) {
+				$.each( data, function( key, value ) {
+				    $('#album_options'+n).append($("<option/>", {
+	        		value: value,
+	        		text: value
+	    			}));
+				});
+				selected_album= $("#album_options"+n).find("option:selected").text();
+				selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+			});
+		change_album(n)
+	    });
+	}
+
+	function change_album(n) {
+		$("#album_options"+n).change(function() {
+			selected_album= $("#album_options"+n).find("option:selected").text();
+			selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+			});
+	}
+
+	function album_matcher(n, existing_artist, existing_album) {
+		var existing_artist = existing_artist.toString();
+		var existing_artist_clean = existing_artist.replace('&', '%26').replace('+', '%2B');
+		var existing_album = existing_album.toString();
+		var existing_album_clean = existing_album.replace('&', '%26').replace('+', '%2B');
+		$('#match_albums'+n).attr('data-success', 'Successfully matched '+existing_album+' with '+selected_album);
+		doAjaxCall('markUnmatched?action=matchAlbum&existing_artist='+existing_artist_clean+'&new_artist='+selected_album_artist_clean+'&existing_album='+existing_album_clean+'&new_album='+selected_album_clean, $('#match_albums'+n), 'page'); 
+	}
+
+	</script>
+</%def>

headphones/__init__.py

@@ -73,6 +73,10 @@ HTTP_ROOT = None
 HTTP_PROXY = False
 LAUNCH_BROWSER = False
 
+ENABLE_HTTPS = False
+HTTPS_CERT = None
+HTTPS_KEY = None
+
 API_ENABLED = False
 API_KEY = None
 
@@ -93,6 +97,7 @@ DESTINATION_DIR = None
 LOSSLESS_DESTINATION_DIR = None
 FOLDER_FORMAT = None
 FILE_FORMAT = None
+FILE_UNDERSCORES = False
 PATH_TO_XML = None
 PREFERRED_QUALITY = None
 PREFERRED_BITRATE = None
@@ -125,6 +130,8 @@ SEARCH_INTERVAL = 360
 LIBRARYSCAN = False
 LIBRARYSCAN_INTERVAL = 300
 DOWNLOAD_SCAN_INTERVAL = 5
+UPDATE_DB_INTERVAL = 24
+MB_IGNORE_AGE = 365
 
 SAB_HOST = None
 SAB_USERNAME = None
@@ -161,6 +168,10 @@ NZBSRUS = False
 NZBSRUS_UID = None
 NZBSRUS_APIKEY = None
 
+OMGWTFNZBS = False
+OMGWTFNZBS_UID = None
+OMGWTFNZBS_APIKEY = None
+
 PREFERRED_WORDS = None
 IGNORED_WORDS = None
 REQUIRED_WORDS = None
@@ -179,6 +190,7 @@ ISOHUNT = None
 KAT = None
 MININOVA = None
 PIRATEBAY = None
+PIRATEBAY_PROXY_URL = None
 WAFFLES = None
 WAFFLES_UID = None
 WAFFLES_PASSKEY = None
@@ -192,6 +204,7 @@ DOWNLOAD_TORRENT_DIR = None
 
 INTERFACE = None
 FOLDER_PERMISSIONS = None
+FILE_PERMISSIONS = None
 
 MUSIC_ENCODER = False
 ENCODERFOLDER = None
@@ -291,20 +304,21 @@ def initialize():
                 HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, HTTP_PROXY, LAUNCH_BROWSER, API_ENABLED, API_KEY, GIT_PATH, GIT_USER, GIT_BRANCH, \
                 CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \
                 LOSSLESS_DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, ADD_ARTISTS, CORRECT_METADATA, MOVE_FILES, \
-                RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, KEEP_TORRENT_FILES, \
+                RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, FILE_UNDERSCORES, CLEANUP_FILES, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, KEEP_TORRENT_FILES, \
                 ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \
-                TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \
+                TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, PIRATEBAY_PROXY_URL, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \
                 RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \
-                LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
+                LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
                 NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \
                 UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \
-                NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \
+                NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \
-                LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \
+		NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \
+                INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \
                 MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, \
                 PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \
                 MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \
                 XBMC_NOTIFY, NMA_ENABLED, NMA_APIKEY, NMA_PRIORITY, NMA_ONSNATCH, SYNOINDEX_ENABLED, ALBUM_COMPLETION_PCT, PREFERRED_BITRATE_HIGH_BUFFER, \
-                PREFERRED_BITRATE_LOW_BUFFER, PREFERRED_BITRATE_ALLOW_LOSSLESS, CACHE_SIZEMB, JOURNAL_MODE, UMASK
+                PREFERRED_BITRATE_LOW_BUFFER, PREFERRED_BITRATE_ALLOW_LOSSLESS, CACHE_SIZEMB, JOURNAL_MODE, UMASK, ENABLE_HTTPS, HTTPS_CERT, HTTPS_KEY
 
         if __INITIALIZED__:
             return False
@@ -319,6 +333,7 @@ def initialize():
         CheckSection('Newznab')
         CheckSection('NZBsorg')
         CheckSection('NZBsRus')
+        CheckSection('omgwtfnzbs')
         CheckSection('Waffles')
         CheckSection('Rutracker')
         CheckSection('What.cd')
@@ -345,6 +360,9 @@ def initialize():
         HTTP_PASSWORD = check_setting_str(CFG, 'General', 'http_password', '')
         HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/')
         HTTP_PROXY = bool(check_setting_int(CFG, 'General', 'http_proxy', 0))
+        ENABLE_HTTPS = bool(check_setting_int(CFG, 'General', 'enable_https', 0))
+        HTTPS_CERT = check_setting_str(CFG, 'General', 'https_cert', os.path.join(DATA_DIR, 'server.crt'))
+        HTTPS_KEY = check_setting_str(CFG, 'General', 'https_key', os.path.join(DATA_DIR, 'server.key'))
         LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1))
         API_ENABLED = bool(check_setting_int(CFG, 'General', 'api_enabled', 0))
         API_KEY = check_setting_str(CFG, 'General', 'api_key', '')
@@ -373,6 +391,7 @@ def initialize():
         RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0))
         FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', 'Artist/Album [Year]')
         FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', 'Track Artist - Album [Year] - Title')
+        FILE_UNDERSCORES = bool(check_setting_int(CFG, 'General', 'file_underscores', 0))
         CLEANUP_FILES = bool(check_setting_int(CFG, 'General', 'cleanup_files', 0))
         ADD_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'add_album_art', 0))
         ALBUM_ART_FORMAT = check_setting_str(CFG, 'General', 'album_art_format', 'folder')
@@ -394,12 +413,15 @@ def initialize():
         LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1))
         LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300)
         DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5)
+        UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24)
+        MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365)
 
         TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '')
         NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10')
         ISOHUNT = bool(check_setting_int(CFG, 'General', 'isohunt', 0))
         KAT = bool(check_setting_int(CFG, 'General', 'kat', 0))
         PIRATEBAY = bool(check_setting_int(CFG, 'General', 'piratebay', 0))
+        PIRATEBAY_PROXY_URL = check_setting_str(CFG, 'General', 'piratebay_proxy_url', '')
         MININOVA = bool(check_setting_int(CFG, 'General', 'mininova', 0))
         DOWNLOAD_TORRENT_DIR = check_setting_str(CFG, 'General', 'download_torrent_dir', '')
 
@@ -453,6 +475,10 @@ def initialize():
         NZBSRUS_UID = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_uid', '')
         NZBSRUS_APIKEY = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_apikey', '')
 
+        OMGWTFNZBS = bool(check_setting_int(CFG, 'omgwtfnzbs', 'omgwtfnzbs', 0))
+        OMGWTFNZBS_UID = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_uid', '')
+        OMGWTFNZBS_APIKEY = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_apikey', '')
+
         PREFERRED_WORDS = check_setting_str(CFG, 'General', 'preferred_words', '')
         IGNORED_WORDS = check_setting_str(CFG, 'General', 'ignored_words', '')
         REQUIRED_WORDS = check_setting_str(CFG, 'General', 'required_words', '')
@@ -461,6 +487,7 @@ def initialize():
 
         INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default')
         FOLDER_PERMISSIONS = check_setting_str(CFG, 'General', 'folder_permissions', '0755')
+        FILE_PERMISSIONS = check_setting_str(CFG, 'General', 'file_permissions', '0644')
 
         ENCODERFOLDER = check_setting_str(CFG, 'General', 'encoderfolder', '')
         ENCODER_PATH = check_setting_str(CFG, 'General', 'encoder_path', '')
@@ -680,8 +707,13 @@ def launch_browser(host, port, root):
     if host == '0.0.0.0':
         host = 'localhost'
         
+    if ENABLE_HTTPS:
+        protocol = 'https'
+    else:
+        protocol = 'http'
+
     try:
-        webbrowser.open('http://%s:%i%s' % (host, port, root))
+        webbrowser.open('%s://%s:%i%s' % (protocol, host, port, root))
     except Exception, e:
         logger.error('Could not launch browser: %s' % e)
 
@@ -698,6 +730,9 @@ def config_write():
     new_config['General']['http_password'] = HTTP_PASSWORD
     new_config['General']['http_root'] = HTTP_ROOT
     new_config['General']['http_proxy'] = int(HTTP_PROXY)
+    new_config['General']['enable_https'] = int(ENABLE_HTTPS)
+    new_config['General']['https_cert'] = HTTPS_CERT
+    new_config['General']['https_key'] = HTTPS_KEY
     new_config['General']['launch_browser'] = int(LAUNCH_BROWSER)
     new_config['General']['api_enabled'] = int(API_ENABLED)
     new_config['General']['api_key'] = API_KEY
@@ -726,6 +761,7 @@ def config_write():
     new_config['General']['rename_files'] = int(RENAME_FILES)
     new_config['General']['folder_format'] = FOLDER_FORMAT
     new_config['General']['file_format'] = FILE_FORMAT
+    new_config['General']['file_underscores'] = int(FILE_UNDERSCORES)
     new_config['General']['cleanup_files'] = int(CLEANUP_FILES)
     new_config['General']['add_album_art'] = int(ADD_ALBUM_ART)
     new_config['General']['album_art_format'] = ALBUM_ART_FORMAT
@@ -748,6 +784,7 @@ def config_write():
     new_config['General']['kat'] = int(KAT)
     new_config['General']['mininova'] = int(MININOVA)
     new_config['General']['piratebay'] = int(PIRATEBAY)
+    new_config['General']['piratebay_proxy_url'] = PIRATEBAY_PROXY_URL
     new_config['General']['download_torrent_dir'] = DOWNLOAD_TORRENT_DIR
 
     new_config['Waffles'] = {}
@@ -769,6 +806,8 @@ def config_write():
     new_config['General']['libraryscan'] = int(LIBRARYSCAN)
     new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL
     new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL
+    new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL
+    new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE
 
     new_config['SABnzbd'] = {}
     new_config['SABnzbd']['sab_host'] = SAB_HOST
@@ -819,6 +858,11 @@ def config_write():
     new_config['NZBsRus']['nzbsrus_uid'] = NZBSRUS_UID
     new_config['NZBsRus']['nzbsrus_apikey'] = NZBSRUS_APIKEY
 
+    new_config['omgwtfnzbs'] = {}
+    new_config['omgwtfnzbs']['omgwtfnzbs'] = int(OMGWTFNZBS)
+    new_config['omgwtfnzbs']['omgwtfnzbs_uid'] = OMGWTFNZBS_UID
+    new_config['omgwtfnzbs']['omgwtfnzbs_apikey'] = OMGWTFNZBS_APIKEY
+
     new_config['General']['preferred_words'] = PREFERRED_WORDS
     new_config['General']['ignored_words'] = IGNORED_WORDS
     new_config['General']['required_words'] = REQUIRED_WORDS
@@ -855,6 +899,7 @@ def config_write():
     new_config['General']['lastfm_username'] = LASTFM_USERNAME
     new_config['General']['interface'] = INTERFACE
     new_config['General']['folder_permissions'] = FOLDER_PERMISSIONS
+    new_config['General']['file_permissions'] = FILE_PERMISSIONS
 
     new_config['General']['music_encoder'] = int(MUSIC_ENCODER)
     new_config['General']['encoder'] = ENCODER
@@ -893,9 +938,9 @@ def start():
         # Start our scheduled background tasks
         from headphones import updater, searcher, librarysync, postprocessor
 
-        SCHED.add_interval_job(updater.dbUpdate, hours=24)
+        SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL)
         SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL)
-        SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True})
+        SCHED.add_interval_job(librarysync.libraryScan, hours=LIBRARYSCAN_INTERVAL, kwargs={'cron':True})
 
         if CHECK_GITHUB:
             SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL)
@@ -929,6 +974,19 @@ def dbcheck():
     c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))')
     c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)')
     c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)')
+    #Below creates indices to speed up Active Artist updating
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)')
+    #Below creates indices to speed up library scanning & matching
+    c.execute('CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)')
 
     try:
         c.execute('SELECT IncludeExtras from artists')
@@ -1094,6 +1152,7 @@ def dbcheck():
     except sqlite3.OperationalError:
         c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL')
 
+
     conn.commit()
     c.close()
 

headphones/api.py

@@ -123,8 +123,9 @@ class Api(object):
     
         artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"')
         albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC')
+        description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"')
         
-        self.data = { 'artist': artist, 'albums': albums }
+        self.data = { 'artist': artist, 'albums': albums, 'description' : description }
         return
     
     def _getAlbum(self, **kwargs):

headphones/db.py

@@ -28,8 +28,6 @@ import headphones
 
 from headphones import logger
 
-db_lock = threading.Lock()
-
 def dbFilename(filename="headphones.db"):
 
     return os.path.join(headphones.DATA_DIR, filename)
@@ -57,8 +55,6 @@ class DBConnection:
         
     def action(self, query, args=None):
 
-        with db_lock:
-
         if query == None:
             return
             

headphones/helpers.py

@@ -345,3 +345,37 @@ def split_string(mystring):
     for each_word in mystring.split(','):
         mylist.append(each_word.strip())
     return mylist
+
+def create_https_certificates(ssl_cert, ssl_key):
+    """
+    Stolen from SickBeard (http://github.com/midgetspy/Sick-Beard):
+    Create self-signed HTTPS certificares and store in paths 'ssl_cert' and 'ssl_key'
+    """
+    from headphones import logger
+    
+    try:
+        from OpenSSL import crypto #@UnresolvedImport
+        from lib.certgen import createKeyPair, createCertRequest, createCertificate, TYPE_RSA, serial #@UnresolvedImport
+    except:
+        logger.warn(u"pyopenssl module missing, please install for https access")
+        return False
+
+    # Create the CA Certificate
+    cakey = createKeyPair(TYPE_RSA, 1024)
+    careq = createCertRequest(cakey, CN='Certificate Authority')
+    cacert = createCertificate(careq, (careq, cakey), serial, (0, 60*60*24*365*10)) # ten years
+
+    cname = 'Headphones'
+    pkey = createKeyPair(TYPE_RSA, 1024)
+    req = createCertRequest(pkey, CN=cname)
+    cert = createCertificate(req, (cacert, cakey), serial, (0, 60*60*24*365*10)) # ten years
+
+    # Save the key and certificate to disk
+    try:
+        open(ssl_key, 'w').write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
+        open(ssl_cert, 'w').write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
+    except:
+        logger.error(u"Error creating SSL key and certificate")
+        return False
+
+    return True

headphones/importer.py

@@ -15,6 +15,7 @@
 
 from lib.pyItunes import *
 import time
+import threading
 import os
 from lib.beets.mediafile import MediaFile
 
@@ -108,7 +109,7 @@ def addArtistIDListToDB(artistidlist):
     for artistid in artistidlist:
         addArtisttoDB(artistid)
 
-def addArtisttoDB(artistid, extrasonly=False):
+def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
     
     # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
     from headphones import cache
@@ -191,90 +192,128 @@ def addArtisttoDB(artistid, extrasonly=False):
     except IndexError:
         includeExtras = False
 
+    #Clean all references to release group in dB that are no longer referenced in musicbrainz
+    group_list = []
+    force_repackage = 0
+    #Don't nuke the database if there's a MusicBrainz error
+    if len(artist['releasegroups']) != 0 and not extrasonly:
+        for groups in artist['releasegroups']:
+            group_list.append(groups['id'])
+        remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
+        for items in remove_missing_groups_from_albums:
+            if items['AlbumID'] not in group_list:
+                # Remove all from albums/tracks that aren't in release groups
+                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
+                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID']))
+                force_repackage = 1
+    else:
+        logger.info("[%s] Error pulling data from MusicBrainz:  Maintaining dB" % artist['artist_name'])
+    
+    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
+
+
     for rg in artist['releasegroups']:
         
-        logger.info("Now adding/updating: " + rg['title'])
+        al_title = rg['title']
-        
+        today = helpers.today()
         rgid = rg['id']
+        skip_log = 0
+        #Make a user configurable variable to skip update of albums with release dates older than this date (in days)
+        pause_delta = headphones.MB_IGNORE_AGE
 
-        # check if the album already exists
+        if not forcefull:
-        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
+            check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone()
+            if check_release_date:
+                if check_release_date[0] is None:
+                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                    new_releases = mb.get_new_releases(rgid,includeExtras,True)          
+                else:
+                    if len(check_release_date[0]) == 10:
+                        release_date = check_release_date[0]
+                    elif len(check_release_date[0]) == 7:
+                        release_date = check_release_date[0]+"-31"
+                    elif len(check_release_date[0]) == 4:
+                        release_date = check_release_date[0]+"-12-31"
+                    else:
+                        release_date = today
+                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                        logger.info("[%s] Now updating: %s (Release Date <%s Days) " % (artist['artist_name'], rg['title'], pause_delta))
+                        new_releases = mb.get_new_releases(rgid,includeExtras,True)
+                    else:
+                        logger.info("[%s] Skipping: %s (Release Date >%s Days)" % (artist['artist_name'], rg['title'], pause_delta))
+                        skip_log = 1
+                        new_releases = 0
+            else:
+                logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
+                new_releases = mb.get_new_releases(rgid,includeExtras)
 
-        releases = mb.get_all_releases(rgid,includeExtras)
+            if force_repackage == 1:
-        if releases == []:
+                new_releases = -1
-            logger.info('No official releases in release group %s' % rg['title'])
+                logger.info('[%s] Forcing repackage of %s (Release Group Removed)' % (artist['artist_name'], al_title))
-            continue
+            else:
-        if not releases:
+                new_releases = new_releases
-            errors = True
+        else:
-            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
+            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist['artist_name'], rg['title']))
-            continue
+            new_releases = mb.get_new_releases(rgid,includeExtras,forcefull)
         
+        #What this does is adds new releases per artist to the allalbums + alltracks databases
+        #new_releases = mb.get_new_releases(rgid,includeExtras)
+        #print al_title
+        #print new_releases
+
+        if new_releases != 0:
+            #Dump existing hybrid release since we're repackaging/replacing it
+            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
             # This will be used later to build a hybrid release     
             fullreleaselist = []
-
+            #Search for releases within a release group
-        for release in releases:
+            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
-        # What we're doing here now is first updating the allalbums & alltracks table to the most
-        # current info, then moving the appropriate release into the album table and its associated
-        # tracks into the tracks table
-            controlValueDict = {"ReleaseID" : release['ReleaseID']}
-
-            newValueDict = {"ArtistID":         release['ArtistID'],
-                            "ArtistName":       release['ArtistName'],
-                            "AlbumTitle":       release['AlbumTitle'],
-                            "AlbumID":          release['AlbumID'],
-                            "AlbumASIN":        release['AlbumASIN'],
-                            "ReleaseDate":      release['ReleaseDate'],
-                            "Type":             release['Type'],
-                            "ReleaseCountry":   release['ReleaseCountry'],
-                            "ReleaseFormat":    release['ReleaseFormat']
-                        }
-
-            myDB.upsert("allalbums", newValueDict, controlValueDict)
-            
             # Build the dictionary for the fullreleaselist
-            newValueDict['ReleaseID'] = release['ReleaseID']
+            for items in find_hybrid_releases:
-            newValueDict['Tracks'] = release['Tracks']
+                if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
+                    hybrid_release_id = items['ReleaseID']
+                    newValueDict = {"ArtistID":         items['ArtistID'],
+                        "ArtistName":       items['ArtistName'],
+                        "AlbumTitle":       items['AlbumTitle'],
+                        "AlbumID":          items['AlbumID'],
+                        "AlbumASIN":        items['AlbumASIN'],
+                        "ReleaseDate":      items['ReleaseDate'],
+                        "Type":             items['Type'],
+                        "ReleaseCountry":   items['ReleaseCountry'],
+                        "ReleaseFormat":    items['ReleaseFormat']
+                    }
+                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
+                    totalTracks = 1
+                    hybrid_track_array = []
+                    for hybrid_tracks in find_hybrid_tracks:
+                        hybrid_track_array.append({
+                            'number':        hybrid_tracks['TrackNumber'],
+                            'title':         hybrid_tracks['TrackTitle'],
+                            'id':            hybrid_tracks['TrackID'],
+                            #'url':           hybrid_tracks['TrackURL'],
+                            'duration':      hybrid_tracks['TrackDuration']
+                            })
+                        totalTracks += 1  
+                    newValueDict['ReleaseID'] = hybrid_release_id
+                    newValueDict['Tracks'] = hybrid_track_array
                     fullreleaselist.append(newValueDict)
 
-            for track in release['Tracks']:
+                #print fullreleaselist
-
-                cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
-        
-                controlValueDict = {"TrackID":      track['id'],
-                                    "ReleaseID":    release['ReleaseID']}
-
-                newValueDict = {"ArtistID":         release['ArtistID'],
-                                "ArtistName":       release['ArtistName'],
-                                "AlbumTitle":       release['AlbumTitle'],
-                                "AlbumID":          release['AlbumID'],
-                                "AlbumASIN":        release['AlbumASIN'],
-                                "TrackTitle":       track['title'],
-                                "TrackDuration":    track['duration'],
-                                "TrackNumber":      track['number'],
-                                "CleanName":        cleanname
-                            }
-                            
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
-            
-                if not match:
-                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
-                if not match:
-                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
-                if match:
-                    newValueDict['Location'] = match['Location']
-                    newValueDict['BitRate'] = match['BitRate']
-                    newValueDict['Format'] = match['Format']
-                    myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
-                                
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
 
             # Basically just do the same thing again for the hybrid release
             # This may end up being called with an empty fullreleaselist
             try:
                 hybridrelease = getHybridRelease(fullreleaselist)
+                logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
             except Exception, e:
                 errors = True
-            logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
+                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e))
                 continue
             
             # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
@@ -315,39 +354,26 @@ def addArtisttoDB(artistid, extrasonly=False):
             
                 if not match:
                     match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
-            if not match:
+                #if not match:
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
+                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
                 if match:
                     newValueDict['Location'] = match['Location']
                     newValueDict['BitRate'] = match['BitRate']
                     newValueDict['Format'] = match['Format']
-                myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))
                                 
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
             
             # Delete matched tracks from the have table
-        myDB.action('DELETE from have WHERE Matched="True"')
+            #myDB.action('DELETE from have WHERE Matched="True"')
             
             # If there's no release in the main albums tables, add the default (hybrid)
             # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
+            # check if the album already exists
+            rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
             if not rg_exists:
                 releaseid = rg['id']
-        elif rg_exists and not rg_exists['ReleaseID']:
-            # Need to do some importing here - to transition the old format of using the release group
-            # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID
-            # so we'll need to move over the locations, bitrates & formats from the tracks table to the new
-            # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups
-            logger.info("Copying current track information to alternate releases")
-            tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall()
-            for track in tracks:
-                if track['Location']:
-                    controlValueDict = {"TrackID":  track['TrackID']}
-                    newValueDict = {"Location":     track['Location'],
-                                    "BitRate":      track['BitRate'],
-                                    "Format":       track['Format'],
-                                    }
-                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-            releaseid = rg['id']
             else:
                 releaseid = rg_exists['ReleaseID']
             
@@ -385,7 +411,6 @@ def addArtisttoDB(artistid, extrasonly=False):
             
             myDB.upsert("albums", newValueDict, controlValueDict) 
 
-        myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']])
             tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
 
             # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
@@ -425,17 +450,21 @@ def addArtisttoDB(artistid, extrasonly=False):
                     myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
                     marked_as_downloaded = True
 
-        logger.info(u"Seeing if we need album art for " + rg['title'])
+            logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
             cache.getThumb(AlbumID=rg['id'])
             
             #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
             if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:    
                 from headphones import searcher
                 searcher.searchforalbum(albumid=rg['id'])
+        else:
+            if skip_log == 0:
+                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
 
     latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone()
     totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid]))
-    havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
+    #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
+    havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist['artist_name']]))
 
     controlValueDict = {"ArtistID":     artistid}
     
@@ -456,13 +485,15 @@ def addArtisttoDB(artistid, extrasonly=False):
     
     myDB.upsert("artists", newValueDict, controlValueDict)
     
-    logger.info(u"Seeing if we need album art for: " + artist['artist_name'])
+    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
     cache.getThumb(ArtistID=artistid)
     
     if errors:
-        logger.info("Finished updating artist: " + artist['artist_name'] + " but with errors, so not marking it as updated in the database")
+        logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name']))
     else:
-        logger.info(u"Updating complete for: " + artist['artist_name'])
+        myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
+        logger.info(u"Updating complete for: %s" % artist['artist_name'])
+
     
 def addReleaseById(rid):
     
@@ -562,14 +593,14 @@ def addReleaseById(rid):
             if not match:
                 match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone()
             
-            if not match:
+            #if not match:
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
+                #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                     
             if match:
                 newValueDict['Location'] = match['Location']
                 newValueDict['BitRate'] = match['BitRate']
                 newValueDict['Format'] = match['Format']
-                myDB.action('DELETE from have WHERE Location=?', [match['Location']])
+                #myDB.action('DELETE from have WHERE Location=?', [match['Location']])
         
             myDB.upsert("tracks", newValueDict, controlValueDict)
                 

headphones/librarysync.py

@@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer
 # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName
 def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False):
 
+
     if cron and not headphones.LIBRARYSCAN:
         return
         
@@ -43,23 +44,39 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
         return
 
     myDB = db.DBConnection()
-    
+    new_artists = []
-    if not append:
-        # Clean up bad filepaths
-        tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
-    
-        for track in tracks:
-            if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
-                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])
-
-        myDB.action('DELETE from have')
 
     logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
     
-    new_artists = []
+    if not append:
+        # Clean up bad filepaths
+        tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL')
+
+        for track in tracks:
+            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
+            if not os.path.isfile(encoded_track_string):
+                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
+                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
+    
+        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
+
+        for track in del_have_tracks:
+            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
+            if not os.path.isfile(encoded_track_string):
+                if track['ArtistName']:
+                    #Make sure deleted files get accounted for when updating artist track counts
+                    new_artists.append(track['ArtistName'])
+                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
+                logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace'))
+           ###############myDB.action('DELETE from have')
+
     bitrates = []
     
     song_list = []
+    new_song_count = 0
+    file_count = 0
+
+    latest_subdirectory = []
     
     for r,d,f in os.walk(dir):
         #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one
@@ -68,9 +85,17 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
             if directory.startswith("."):
                 d.remove(directory)
         for files in f:
+
             # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
             if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
 
+                subdirectory = r.replace(dir,'')
+                latest_subdirectory.append(subdirectory)
+                if file_count == 0 and r.replace(dir,'') !='':
+                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+                elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0:
+                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+
                 song = os.path.join(r, files)
 
                 # We need the unicode path to use for logging, inserting into database
@@ -99,8 +124,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                 # Add the song to our song list - 
                 # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
 
-                song_dict = { 'TrackID' : f.mb_trackid,
+                if f_artist and f.album and f.title:
-                              'ReleaseID' : f.mb_albumid,
+                    CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title)
+                else:
+                    CleanName = None
+
+                controlValueDict = {'Location'   : unicode_song_path}
+
+                newValueDict = { 'TrackID' : f.mb_trackid,
+                              #'ReleaseID' : f.mb_albumid,
                               'ArtistName' : f_artist,
                               'AlbumTitle' : f.album,
                               'TrackNumber': f.track,
@@ -110,224 +142,169 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                               'TrackTitle' : f.title,
                               'BitRate'    : f.bitrate,
                               'Format'     : f.format,
-                              'Location'   : unicode_song_path }
+                              'CleanName'  : CleanName
+                              }
+                              
+                #song_list.append(song_dict)
+                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone()
+                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
+                if not check_exist_song:
+                    #This is a new track
+                    if f_artist:
+                        new_artists.append(f_artist)
+                    myDB.upsert("have", newValueDict, controlValueDict)
+                    new_song_count+=1
+                else:
+                    if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                        #Important track metadata has been modified, need to run matcher again
+                        if f_artist and f_artist != check_exist_song['ArtistName']:
+                            new_artists.append(f_artist)
+                        elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored":
+                            new_artists.append(f_artist)
+                        else:
+                            continue
+
+                        newValueDict['Matched'] = None
+                        myDB.upsert("have", newValueDict, controlValueDict)
+                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
+                        myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
+                        new_song_count+=1
+                    else:
+                        #This track information hasn't changed
+                        if f_artist and check_exist_song['Matched'] != "Ignored":
+                            new_artists.append(f_artist)
+
+                file_count+=1
 
-                song_list.append(song_dict)
 
     # Now we start track matching
-    total_number_of_songs = len(song_list)
+    logger.info("%s new/modified songs found and added to the database" % new_song_count)
-    logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"])
+    total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0]
+    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
     
     # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
     # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
     
-    song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
+    ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
+    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
     
     # We'll use this to give a % completion, just because the track matching might take a while
     song_count = 0
+    latest_artist = []
     
     for song in song_list:
 
+        latest_artist.append(song['ArtistName'])
+        if song_count == 0:
+            logger.info("Now matching songs by %s" % song['ArtistName'])
+        elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0:
+            logger.info("Now matching songs by %s" % song['ArtistName'])
+        
+        #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle']
         song_count += 1
         completion_percentage = float(song_count)/total_number_of_songs * 100
         
         if completion_percentage%10 == 0:
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
         
-        # If the track has a trackid & releaseid (beets: albumid) that the most surefire way
+        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
-        # of identifying a track to a specific release so we'll use that first
+        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
-        if song['TrackID'] and song['ReleaseID']:
+        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
+        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
-            # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet
-            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
-            
-            # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table
-            if not track:
-                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
-    
-            if track:
-                # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'ReleaseID' : track['ReleaseID'] }
-                
-                # Insert it into the Headphones hybrid release (ReleaseID == AlbumID)                   
-                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
-                                           'ReleaseID' : track['AlbumID'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-                                 
-                # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-                
-                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
-                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
-                
-                # Matched. Move on to the next one:
-                continue
-    
-        # If we can't find it with TrackID & ReleaseID, next most specific will be 
-        # releaseid + tracktitle, although perhaps less reliable due to a higher 
-        # likelihood of variations in the song title (e.g. feat. artists)
-        if song['ReleaseID'] and song['TrackTitle']:
-    
-            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
-                
-            if track:
-                # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table)
-                # So use both TrackID & ReleaseID as the control values
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'ReleaseID' : track['ReleaseID'] }
-                                     
-                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
-                                           'ReleaseID' : track['AlbumID'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-                                 
-                # Update both tables here as well
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-                
-                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
-                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
-                
-                # Done
-                continue
-                
-        # Next most specific will be the opposite: a TrackID and an AlbumTitle
-        # TrackIDs span multiple releases so if something is on an official album
-        # and a compilation, for example, this will match it to the right one
-        # However - there may be multiple matches here
-        if song['TrackID'] and song['AlbumTitle']:
-    
-            # Even though there might be multiple matches, we just need to grab one to confirm a match
-            track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
-                
-            if track:
-                # Don't need the hybridControlValueDict here since ReleaseID is not unique
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'AlbumTitle' : track['AlbumTitle'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-
-                continue   
-        
-        # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably 
-        # even more unreliable than the previous queries, and might span multiple releases)
         if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
             
-            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-                
             if track:
                 controlValueDict = { 'ArtistName' : track['ArtistName'],
                                      'AlbumTitle' : track['AlbumTitle'],
                                      'TrackTitle' : track['TrackTitle'] }                           
-                                     
                 newValueDict = { 'Location' : song['Location'],
                                  'BitRate'  : song['BitRate'],
                                  'Format'   : song['Format'] }
-
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
                 myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                continue
+                controlValueDict2 = { 'Location' : song['Location']}
-        
+                newValueDict2 = { 'Matched' : track['AlbumID']}
-        # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc)
+                myDB.upsert("have", newValueDict2, controlValueDict2)
-        # This is more reliable than the former but requires some string manipulation so we'll do it only
+            else:
-        # if we can't find a match with the original data
+                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
-            
-            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
-            
-            track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
-            
-            if not track:
-                track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
-    
                 if track:
                     controlValueDict = { 'CleanName' : track['CleanName']}                         
-                                     
                     newValueDict = { 'Location' : song['Location'],
                                      'BitRate'  : song['BitRate'],
                                      'Format'   : song['Format'] }               
-
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
                     myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                continue     
+                    controlValueDict2 = { 'Location' : song['Location']}
+                    newValueDict2 = { 'Matched' : track['AlbumID']}
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
+                else:
+                    controlValueDict2 = { 'Location' : song['Location']}
+                    newValueDict2 = { 'Matched' : "Failed"}
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
   
-        # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable
-        # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track
-        # with more specific information exists in the library, it'll overwrite these values
-        if song['TrackID']:
-    
-            track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone()
-            
-            if not track:
-                track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone()
-    
-            if track:
-                controlValueDict = { 'TrackID' : track['TrackID'] }
 
+            alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()  
+            if alltrack:
+                controlValueDict = { 'ArtistName' : alltrack['ArtistName'],
+                                     'AlbumTitle' : alltrack['AlbumTitle'],
+                                     'TrackTitle' : alltrack['TrackTitle'] }
                 newValueDict = { 'Location' : song['Location'],
                                  'BitRate'  : song['BitRate'],
                                  'Format'   : song['Format'] }
-
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                continue          
+                controlValueDict2 = { 'Location' : song['Location']}
-        
+                newValueDict2 = { 'Matched' : alltrack['AlbumID']}
-        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
+                myDB.upsert("have", newValueDict2, controlValueDict2)
-        if song['ArtistName']:
-            new_artists.append(song['ArtistName'])
             else:
-            continue
+                alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
+                if alltrack:
+                    controlValueDict = { 'CleanName' : alltrack['CleanName']}                                         
+                    newValueDict = { 'Location' : song['Location'],
+                                     'BitRate'  : song['BitRate'],
+                                     'Format'   : song['Format'] }               
+                    myDB.upsert("alltracks", newValueDict, controlValueDict)
 
-        # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
+                    controlValueDict2 = { 'Location' : song['Location']}
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+                    newValueDict2 = { 'Matched' : alltrack['AlbumID']}
-            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
                 else:
-            continue
+                    controlValueDict2 = { 'Location' : song['Location']}
+                    newValueDict2 = { 'Matched' : "Failed"}
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
+        else:
+            controlValueDict2 = { 'Location' : song['Location']}
+            newValueDict2 = { 'Matched' : "Failed"}
+            myDB.upsert("have", newValueDict2, controlValueDict2)
         
-        myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
 
     logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
 
     
     if not append:
+        logger.info('Updating scanned artist track counts')
+        
         # Clean up the new artist list
         unique_artists = {}.fromkeys(new_artists).keys()
         current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
 
-        artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
+        #There was a bug where artists with special characters (-,') would show up in new artists.     
+        artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]]
+        artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]]
 
         # Update track counts
-        logger.info('Updating current artist track counts')
     
-        for artist in current_artists:
+        for artist in artists_checked:
             # Have tracks are selected from tracks table and not all tracks because of duplicates
             # We update the track count upon an album switch to compliment this
-            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
+            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
+            #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases 
+            # (can fix by getting rid of second len statement)
+            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
             
         logger.info('Found %i new artists' % len(artist_list))
     
@@ -337,9 +314,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                 importer.artistlist_to_mbids(artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
-                myDB.action('DELETE from newartists')
+                #myDB.action('DELETE from newartists')
                 for artist in artist_list:
-                    myDB.action('INSERT into newartists VALUES (?)', [artist])
+                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
         
         if headphones.DETECT_BITRATE:
             headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
@@ -348,6 +325,42 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
         
-        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName]))
+        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName]))
         myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
 
+    update_album_status()
+    logger.info('Library scan complete')
+
+    #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
+def update_album_status(AlbumID=None):
+    myDB = db.DBConnection()
+    logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+    if AlbumID:
+        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+    else:
+        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
+    for album in album_status_updater:
+        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']])
+        total_tracks = 0
+        have_tracks = 0
+        for track in track_counter:
+            total_tracks+=1
+            if track['Location']:
+                have_tracks+=1
+        if total_tracks != 0:
+            album_completion = float(float(have_tracks) / float(total_tracks)) * 100
+        else:
+            album_completion = 0
+            logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
+
+        if album_completion >= headphones.ALBUM_COMPLETION_PCT:
+            new_album_status = "Downloaded"
+        else:
+            if album['Status'] == "Skipped" or album['Status'] == "Downloaded":
+                new_album_status = "Skipped"
+            else:
+                new_album_status = album['Status']
+        myDB.upsert("albums", {'Status'   : new_album_status}, {'AlbumID'   : album['AlbumID']})
+        if new_album_status != album['Status']:
+            logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
+    logger.info('Album status update complete')

headphones/mb.py

@@ -19,7 +19,7 @@ import time
 import threading
 
 import headphones
-from headphones import logger, db
+from headphones import logger, db, helpers
 from headphones.helpers import multikeysort, replace_all
 
 import lib.musicbrainzngs as musicbrainzngs
@@ -332,7 +332,9 @@ def getRelease(releaseid, include_artist_info=True):
         
         return release
 
-def get_all_releases(rgid,includeExtras=False):
+def get_new_releases(rgid,includeExtras=False,forcefull=False):
+
+    myDB = db.DBConnection()
     results = []
     try:
         limit = 100
@@ -352,8 +354,29 @@ def get_all_releases(rgid,includeExtras=False):
     if not results or len(results) == 0:
         return False
 
+    #Clean all references to releases in dB that are no longer referenced in musicbrainz
+    release_list = []
+    force_repackage1 = 0
+    if len(results) != 0:
+        for release_mark in results:
+            release_list.append(unicode(release_mark['id']))
+            release_title = release_mark['title']
+        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
+        if remove_missing_releases:
+            for items in remove_missing_releases:
+                if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
+                    # Remove all from albums/tracks that aren't in release
+                    myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
+                    logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
+                    force_repackage1 = 1
+    else:
+        logger.info("Error pulling data from MusicBrainz:  Maintaining dB")
+
+    num_new_releases = 0
 
-    releases = []
     for releasedata in results:
         #releasedata.get will return None if it doesn't have a status
         #all official releases should have the Official status included
@@ -361,6 +384,14 @@ def get_all_releases(rgid,includeExtras=False):
             continue
         
         release = {}
+        rel_id_check = releasedata['id']
+        artistid = unicode(releasedata['artist-credit'][0]['artist']['id'])
+
+        album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
+        if not album_checker or forcefull:
+            #DELETE all references to this release since we're updating it anyway.
+            myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
+            myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
             release['AlbumTitle'] = unicode(releasedata['title'])
             release['AlbumID'] = unicode(rgid)
             release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
@@ -383,23 +414,100 @@ def get_all_releases(rgid,includeExtras=False):
             release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
             #assuming that the list will contain media and that the format will be consistent
             try:
-            release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format'])
+                additional_medium=''
+                for position in releasedata['medium-list']:
+                    if position['format'] == releasedata['medium-list'][0]['format']:
+                        medium_count = int(position['position'])
+                    else:
+                        additional_medium = additional_medium+' + '+position['format']
+                if medium_count == 1:
+                    disc_number = ''
+                else:
+                    disc_number = str(medium_count)+'x'
+                packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium
+                release['ReleaseFormat'] = unicode(packaged_medium)
             except:
                 release['ReleaseFormat'] = u'Unknown'
      
             release['Tracks'] = getTracksFromRelease(releasedata)
-        releases.append(release)
 
-    return releases
+        # What we're doing here now is first updating the allalbums & alltracks table to the most
+        # current info, then moving the appropriate release into the album table and its associated
+        # tracks into the tracks table
+            controlValueDict = {"ReleaseID" : release['ReleaseID']}
+
+            newValueDict = {"ArtistID":         release['ArtistID'],
+                            "ArtistName":       release['ArtistName'],
+                            "AlbumTitle":       release['AlbumTitle'],
+                            "AlbumID":          release['AlbumID'],
+                            "AlbumASIN":        release['AlbumASIN'],
+                            "ReleaseDate":      release['ReleaseDate'],
+                            "Type":             release['Type'],
+                            "ReleaseCountry":   release['ReleaseCountry'],
+                            "ReleaseFormat":    release['ReleaseFormat']
+                        }
+
+            myDB.upsert("allalbums", newValueDict, controlValueDict)
+            
+            for track in release['Tracks']:
+
+                cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
+        
+                controlValueDict = {"TrackID":      track['id'],
+                                    "ReleaseID":    release['ReleaseID']}
+
+                newValueDict = {"ArtistID":         release['ArtistID'],
+                                "ArtistName":       release['ArtistName'],
+                                "AlbumTitle":       release['AlbumTitle'],
+                                "AlbumID":          release['AlbumID'],
+                                "AlbumASIN":        release['AlbumASIN'],
+                                "TrackTitle":       track['title'],
+                                "TrackDuration":    track['duration'],
+                                "TrackNumber":      track['number'],
+                                "CleanName":        cleanname
+                            }
+                            
+                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
+            
+                if not match:
+                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
+                #if not match:
+                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
+                if match:
+                    newValueDict['Location'] = match['Location']
+                    newValueDict['BitRate'] = match['BitRate']
+                    newValueDict['Format'] = match['Format']
+                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location']))
+                                
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+            num_new_releases = num_new_releases + 1
+            #print releasedata['title']
+            #print num_new_releases
+            if album_checker:
+                logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+            else:
+                logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+    if force_repackage1 == 1:
+        num_new_releases = -1
+        logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title))
+    else:
+        num_new_releases = num_new_releases
+
+    return num_new_releases
 
 def getTracksFromRelease(release):
     totalTracks = 1
     tracks = []
     for medium in release['medium-list']:
         for track in medium['track-list']:
+            try:
+                track_title = unicode(track['title'])
+            except:
+                track_title = unicode(track['recording']['title'])
             tracks.append({
                     'number':        totalTracks,
-                    'title':         unicode(track['recording']['title']),
+                    'title':         track_title,
                     'id':            unicode(track['recording']['id']),
                     'url':           u"http://musicbrainz.org/track/" + track['recording']['id'],
                     'duration':      int(track['length']) if 'length' in track else 0

headphones/music_encoder.py

@@ -224,9 +224,9 @@ def command(encoder,musicSource,musicDest,albumPath):
         opts = []
         if headphones.ADVANCEDENCODER =='':
             if headphones.ENCODEROUTPUTFORMAT=='ogg':
-                opts.extend(['-acodec libvorbis'])
+                opts.extend(['-acodec', 'libvorbis'])
             if headphones.ENCODEROUTPUTFORMAT=='m4a':
-                opts.extend(['-strict experimental'])
+                opts.extend(['-strict', 'experimental'])
             if headphones.ENCODERVBRCBR=='cbr':
                 opts.extend(['-ar', str(headphones.SAMPLINGFREQUENCY), '-ab', str(headphones.BITRATE) + 'k'])
             elif headphones.ENCODERVBRCBR=='vbr':
@@ -244,7 +244,16 @@ def command(encoder,musicSource,musicDest,albumPath):
     logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
     logger.debug(subprocess.list2cmdline(cmd))
 
-    p = subprocess.Popen(cmd, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # stop windows opening the cmd
+    startupinfo = None
+    if headphones.SYS_PLATFORM == "win32":
+        startupinfo = subprocess.STARTUPINFO()
+        try:
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        except AttributeError:
+            startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
+
+    p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     stdout, stderr = p.communicate(headphones.ENCODER)
 

headphones/postprocessor.py

@@ -43,6 +43,7 @@ def checkFolder():
         
             if album['FolderName']:
                 
+                if album['Kind'] == 'nzb':
                     # We're now checking sab config options after sending to determine renaming - but we'll keep the
                     # iterations in just in case we can't read the config for some reason
 
@@ -52,19 +53,21 @@ def checkFolder():
                                                 sab_replace_spaces(sab_replace_dots(album['FolderName']))
                                         ]
                     
-                torrent_album_path = os.path.join(headphones.DOWNLOAD_TORRENT_DIR, album['FolderName']).encode(headphones.SYS_ENCODING,'replace')
-
                     for nzb_folder_name in nzb_album_possibilities:
                         
                         nzb_album_path = os.path.join(headphones.DOWNLOAD_DIR, nzb_folder_name).encode(headphones.SYS_ENCODING, 'replace')
     
                         if os.path.exists(nzb_album_path):
                             logger.debug('Found %s in NZB download folder. Verifying....' % album['FolderName'])
-                        verify(album['AlbumID'], nzb_album_path, album['Kind'])
+                            verify(album['AlbumID'], nzb_album_path, 'nzb')
+                            
+                if album['Kind'] == 'torrent':
+
+                    torrent_album_path = os.path.join(headphones.DOWNLOAD_TORRENT_DIR, album['FolderName']).encode(headphones.SYS_ENCODING,'replace')
     
                     if os.path.exists(torrent_album_path):
                         logger.debug('Found %s in torrent download folder. Verifying....' % album['FolderName'])
-                    verify(album['AlbumID'], torrent_album_path, album['Kind'])
+                        verify(album['AlbumID'], torrent_album_path, 'torrent')
 
 def verify(albumid, albumpath, Kind=None, forced=False):
 
@@ -325,7 +328,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
     logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle']))
     # Check to see if we're preserving the torrent dir
     if headphones.KEEP_TORRENT_FILES and Kind=="torrent":
-        new_folder = os.path.join(albumpath, 'headphones-modified').encode(headphones.SYS_ENCODING, 'replace')
+        new_folder = os.path.join(albumpath, 'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace'))
         logger.info("Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding")
         try:
             shutil.copytree(albumpath, new_folder)
@@ -392,6 +395,8 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
     else:
         albumpaths = [albumpath]
         
+    updateFilePermissions(albumpaths)
+        
     myDB = db.DBConnection()
     myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
     myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid])
@@ -464,6 +469,9 @@ def addAlbumArt(artwork, albumpath, release):
 
     album_art_name = album_art_name.replace('?','_').replace(':', '_').encode(headphones.SYS_ENCODING, 'replace')
 
+    if headphones.FILE_UNDERSCORES:
+        album_art_name = album_art_name.replace(' ', '_')
+
     if album_art_name.startswith('.'):
         album_art_name = album_art_name.replace(0, '_')
 
@@ -493,6 +501,10 @@ def moveFiles(albumpath, release, tracks):
         
     artist = release['ArtistName'].replace('/', '_')
     album = release['AlbumTitle'].replace('/', '_')
+    if headphones.FILE_UNDERSCORES:
+        artist = artist.replace(' ', '_')
+        album = album.replace(' ', '_')
+
     releasetype = release['Type'].replace('/', '_')
 
     if release['ArtistName'].startswith('The '):
@@ -521,7 +533,7 @@ def moveFiles(albumpath, release, tracks):
             }
             
     folder = helpers.replace_all(headphones.FOLDER_FORMAT.strip(), values)
-    folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_')
+    folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_').replace('|','_')
     
     if folder.endswith('.'):
         folder = folder.replace(folder[len(folder)-1], '_')
@@ -831,6 +843,10 @@ def renameFiles(albumpath, downloaded_track_list, release):
         
         
         new_file_name = new_file_name.replace('?','_').replace(':', '_').encode(headphones.SYS_ENCODING, 'replace')
+        new_file_name = new_file_name.replace('*','_')
+
+        if headphones.FILE_UNDERSCORES:
+            new_file_name = new_file_name.replace(' ', '_')
 
         if new_file_name.startswith('.'):
             new_file_name = new_file_name.replace(0, '_')
@@ -848,6 +864,19 @@ def renameFiles(albumpath, downloaded_track_list, release):
             logger.error('Error renaming file: %s. Error: %s' % (downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), e))
             continue
             
+def updateFilePermissions(albumpaths):
+
+    for folder in albumpaths:
+        logger.info("Updating file permissions in " + folder.decode(headphones.SYS_ENCODING, 'replace'))
+        for r,d,f in os.walk(folder):
+            for files in f:
+                full_path = os.path.join(r, files)
+                try:
+                    os.chmod(full_path, int(headphones.FILE_PERMISSIONS, 8))
+                except:
+                    logger.error("Could not change permissions for file: " + full_path.decode(headphones.SYS_ENCODING, 'replace'))
+                    continue
+
 def renameUnprocessedFolder(albumpath):
     
     i = 0

headphones/searcher.py

@@ -41,6 +41,10 @@ import lib.bencode as bencode
 import headphones.searcher_rutracker as rutrackersearch
 rutracker = rutrackersearch.Rutracker()
 
+# Persistent What.cd API object
+gazelle = None
+
+
 class NewzbinDownloader(urllib.FancyURLopener):
 
     def __init__(self):
@@ -114,7 +118,7 @@ def searchforalbum(albumid=None, new=False, lossless=False):
 
         for result in results:
             foundNZB = "none"
-            if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
+            if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
                 if result['Status'] == "Wanted Lossless":
                     foundNZB = searchNZB(result['AlbumID'], new, losslessOnly=True)
                 else:
@@ -131,12 +135,14 @@ def searchforalbum(albumid=None, new=False, lossless=False):
 
         foundNZB = "none"
         
-        if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
+        if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
             foundNZB = searchNZB(albumid, new, lossless)
 
         if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none":
             searchTorrent(albumid, new, lossless)
 
+    logger.info('Search for Wanted albums complete')
+    
 def searchNZB(albumid=None, new=False, losslessOnly=False):
 
     myDB = db.DBConnection()
@@ -433,6 +439,65 @@ def searchNZB(albumid=None, new=False, losslessOnly=False):
                         except Exception, e:
                             logger.error(u"An unknown error occurred trying to parse the feed: %s" % e)
 
+
+        if headphones.OMGWTFNZBS:
+
+            provider = "omgwtfnzbs"
+
+            if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
+                categories = "22"
+            elif headphones.PREFERRED_QUALITY:
+                categories = "22,7"
+            else:
+                 categories = "7"
+
+            if albums['Type'] == 'Other':
+                categories = "29"
+                logger.info("Album type is audiobook/spokenword. Searching all music categories")
+
+            params = {  "user": headphones.OMGWTFNZBS_UID,
+                        "api": headphones.OMGWTFNZBS_APIKEY,
+                        "catid": categories,
+                        "retention": headphones.USENET_RETENTION,
+                        "search": term
+                        }
+
+            searchURL = 'http://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params)
+
+            # Add a user-agent
+            request = urllib2.Request(searchURL)
+            request.add_header('User-Agent', 'headphones/0.0 +https://github.com/rembo10/headphones')
+            opener = urllib2.build_opener()
+
+            logger.info(u'Parsing results from <a href="%s">omgwtfnzbs</a>' % searchURL)
+
+            try:
+                data = opener.open(request).read()
+            except Exception, e:
+                logger.warn('Error fetching data from omgwtfnzbs: %s' % e)
+                data = False
+
+            if data:
+
+                d = json.loads(data)
+
+                if 'notice' in data: 
+                    logger.info(u"No results returned from omgwtfnzbs: %s" % d['notice'])
+                    pass
+
+                else:
+                    for item in d:
+                        try:
+                            url = item['getnzb']
+                            title = item['release']
+                            size = int(item['sizebytes'])
+
+                            resultlist.append((title, size, url, provider))
+                            logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
+
+                        except Exception, e:
+                            logger.error(u"An unknown error occurred trying to parse the results: %s" % e)
+
         # attempt to verify that this isn't a substring result
         # when looking for "Foo - Foo" we don't want "Foobar"
         # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
@@ -735,6 +800,7 @@ def preprocess(resultlist):
 
 
 def searchTorrent(albumid=None, new=False, losslessOnly=False):
+    global gazelle  # persistent what.cd api object to reduce number of login attempts
 
     myDB = db.DBConnection()
 
@@ -939,7 +1005,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                         try:
                             title = item.title
                             desc_match = re.search(r"Size: (\d+)<", item.description)
-                            size = desc_match.group(1)
+                            size = int(desc_match.group(1))
                             url = item.link
                             resultlist.append((title, size, url, provider))
                             logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
@@ -1004,10 +1070,10 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
             bitrate = None
             bitrate_string = bitrate
 
-            if headphones.PREFERRED_QUALITY == 2 or losslessOnly:  # Lossless Only mode
+            if headphones.PREFERRED_QUALITY == 3 or losslessOnly:  # Lossless Only mode
                 search_formats = [gazelleformat.FLAC]
                 maxsize = 10000000000
-            elif headphones.PREFERRED_QUALITY == 3:  # Preferred quality mode
+            elif headphones.PREFERRED_QUALITY == 2:  # Preferred quality mode
                 search_formats = [None]  # should return all
                 bitrate = headphones.PREFERRED_BITRATE
                 if bitrate:
@@ -1024,13 +1090,16 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                 search_formats = [gazelleformat.MP3]
                 maxsize = 300000000
 
+            if not gazelle or not gazelle.logged_in():
                 try:
+                    logger.info(u"Attempting to log in to What.cd...")
                     gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD)
+                    gazelle._login()
                 except Exception, e:
                     gazelle = None
-                logger.warn(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e)))
+                    logger.error(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e)))
 
-            if gazelle:
+            if gazelle and gazelle.logged_in():
                 logger.info(u"Searching %s..." % provider)
                 all_torrents = []
                 for search_format in search_formats:
@@ -1073,11 +1142,22 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                                        provider))
 
         # Pirate Bay
-        if headphones.PIRATEBAY and headphones.TORRENT_DOWNLOADER == 0:
+        if headphones.PIRATEBAY:
-            logger.warn("Cannot search Pirate Bay with Blackhole option set")
-        if headphones.PIRATEBAY and headphones.TORRENT_DOWNLOADER != 0:
             provider = "The Pirate Bay"    
+            if headphones.PIRATEBAY_PROXY_URL:
+                #Might need to clean up the user submitted url
+                pirate_proxy = headphones.PIRATEBAY_PROXY_URL
+                
+                if not pirate_proxy.startswith('http'):
+                    pirate_proxy = 'http://' + pirate_proxy
+                if pirate_proxy.endswith('/'):
+                    pirate_proxy = pirate_proxy[:-1]
+                    
+                providerurl = url_fix(pirate_proxy + "/search/" + term + "/0/99/")
+                
+            else:
                 providerurl = url_fix("http://thepiratebay.sx/search/" + term + "/0/99/")
+                
             if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
                 category = '104'          #flac
                 maxsize = 10000000000
@@ -1117,9 +1197,15 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                             title = ''.join(item.find("a", {"class" : "detLink"}))
                             seeds = int(''.join(item.find("td", {"align" : "right"})))
                             url = item.findAll("a")[3]['href']
+                            if headphones.TORRENT_DOWNLOADER == 0:
+                                tor_hash = re.findall("urn:btih:(.*?)&", url)
+                                if len(tor_hash) > 0:
+                                    url = "http://torrage.com/torrent/"+str(tor_hash[0]).upper()+".torrent"
+                                else:
+                                    url = None
                             formatted_size = re.search('Size (.*),', unicode(item)).group(1).replace(u'\xa0', ' ')
                             size = helpers.piratesize(formatted_size)
-                            if size < maxsize and minimumseeders < seeds:
+                            if size < maxsize and minimumseeders < seeds and url != None:
                                 resultlist.append((title, size, url, provider))
                                 logger.info('Found %s. Size: %s' % (title, formatted_size))
                             else:
@@ -1193,6 +1279,9 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                                         rightformat = False
                             except Exception, e:
                                 rightformat = False
+                            for findterm in term.split(" "):
+                                if not findterm in title:
+                                    rightformat = False
                             if rightformat == True and size < maxsize and minimumseeders < seeds:
                                 resultlist.append((title, size, url, provider))
                                 logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
@@ -1427,6 +1516,11 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                         file_or_url = bestqual[2]
 
                     torrentid = transmission.addTorrent(file_or_url)
+                    
+                    if not torrentid:
+                        logger.error("Error sending torrent to Transmission. Are you sure it's running?")
+                        return
+                        
                     torrent_folder_name = transmission.getTorrentFolder(torrentid)
                     logger.info('Torrent folder name: %s' % torrent_folder_name)
 
@@ -1449,7 +1543,7 @@ def preprocesstorrent(resultlist, pre_sorted_list=False):
     for result in resultlist:
 
         # get outta here if rutracker or piratebay
-        if result[3] == 'rutracker.org' or result[3] == 'The Pirate Bay':
+        if result[3] == 'rutracker.org':
             return True, result
 
         try:

headphones/searcher_rutracker.py

@@ -124,7 +124,7 @@ class Rutracker():
                 
             # Seeders
 
-            for link in soup.find_all('td', attrs={'class' : 'row4 seedmed'}): 
+            for link in soup.find_all('b', attrs={'class' : 'seedmed'}):
                 seeder = link.get_text()
                 seeders.append(seeder)
             

headphones/transmission.py

@@ -20,6 +20,7 @@ import urllib2
 import lib.simplejson as json
 import base64
 import time
+import re
 
 # This is just a simple script to send torrents to transmission. The
 # intention is to turn this into a class where we can check the state
@@ -33,6 +34,8 @@ def addTorrent(link):
     
     response = torrentAction(method,arguments)
 
+    if not response:
+        return False
         
     if response['result'] == 'success':
         name = response['arguments']['torrent-added']['name']
@@ -82,7 +85,24 @@ def torrentAction(method, arguments):
     if host.endswith('/'):
         host = host[:-1]
 
+    # Either the host ends with a port, or some directory, or rpc
+    # If it ends with /rpc we don't have to do anything
+    # If it ends with a port we add /transmission/rpc
+    # anything else we just add rpc
+    if not host.endswith('/rpc'):
+        # Check if it ends in a port number
+        i = host.rfind(':')
+        if i >= 0:
+            possible_port = host[i+1:]
+            try:
+                port = int(possible_port)
                 host = host + "/transmission/rpc"
+            except ValueError:
+                host = host + "/rpc"
+        else:
+            logger.error('Transmission port missing')
+            return
+
     request = urllib2.Request(host)
     if username and password:
         base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')

headphones/updater.py

@@ -17,17 +17,16 @@ import headphones
 
 from headphones import logger, db, importer
 
-def dbUpdate():
+def dbUpdate(forcefull=False):
 
     myDB = db.DBConnection()
 
     activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
-
     logger.info('Starting update for %i active artists' % len(activeartists))
     
     for artist in activeartists:
     
         artistid = artist[0]
-        importer.addArtisttoDB(artistid)
+        importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull)
         
-    logger.info('Update complete')
+    logger.info('Active artist update complete')

headphones/webserve.py

@@ -24,11 +24,14 @@ from mako import exceptions
 
 import time
 import threading
+import string
+import json
+from operator import itemgetter
 
 import headphones
 
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync
+from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers
-from headphones.helpers import checked, radio,today
+from headphones.helpers import checked, radio,today, cleanName
 
 import lib.simplejson as simplejson
 
@@ -105,7 +108,7 @@ class WebInterface(object):
     def albumPage(self, AlbumID):
         myDB = db.DBConnection()
         album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-        tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [AlbumID])
+        tracks = myDB.select('SELECT * from tracks WHERE AlbumID=? ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID])
         description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=?', [AlbumID]).fetchone()
         title = album['ArtistName'] + ' - ' + album['AlbumTitle']
         return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description)
@@ -150,7 +153,7 @@ class WebInterface(object):
         newValueDict = {'IncludeExtras': 1,
                         'Extras':        extras}
         myDB.upsert("artists", newValueDict, controlValueDict)
-        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True]).start()
+        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start()
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     getExtras.exposed = True
 
@@ -163,6 +166,8 @@ class WebInterface(object):
         for album in extraalbums:
             myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
             myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
+            myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
+            myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     removeExtras.exposed = True
 
@@ -187,11 +192,15 @@ class WebInterface(object):
     def deleteArtist(self, ArtistID):
         logger.info(u"Deleting all traces of artist: " + ArtistID)
         myDB = db.DBConnection()
+        namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID])
+        for name in namecheck:
+            artistname=name['ArtistName']
         myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID])
+        myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname])
         myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID])
         raise cherrypy.HTTPRedirect("home")
     deleteArtist.exposed = True
@@ -211,7 +220,7 @@ class WebInterface(object):
     deleteEmptyArtists.exposed = True
 
     def refreshArtist(self, ArtistID):
-        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start()
+        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start()
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     refreshArtist.exposed=True
 
@@ -238,8 +247,15 @@ class WebInterface(object):
             raise cherrypy.HTTPRedirect("upcoming")
     markAlbums.exposed = True
 
-    def addArtists(self, **args):
+    def addArtists(self, action=None, **args):
+        if action == "add":
             threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start()
+        if action == "ignore":
+            myDB = db.DBConnection()
+            for artist in args:
+                myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist])
+                myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist])
+                logger.info("Artist %s removed from new artist list and set to ignored" % artist)
         raise cherrypy.HTTPRedirect("home")
     addArtists.exposed = True
 
@@ -274,6 +290,8 @@ class WebInterface(object):
         myDB = db.DBConnection()
         myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID])
         myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID])
+        myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID])
+        myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID])
         if ArtistID:
             raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
         else:
@@ -334,6 +352,193 @@ class WebInterface(object):
         return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists)
     manageNew.exposed = True
 
+    def manageUnmatched(self):
+        myDB = db.DBConnection()
+        have_album_dictionary = []
+        headphones_album_dictionary = []
+        unmatched_albums = []
+        have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName')
+        for albums in have_albums:
+            #Have to skip over manually matched tracks
+            if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
+                original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle'])
+            # else:
+            #     original_clean = None
+                if original_clean == albums['CleanName']:
+                    have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }  
+                    have_album_dictionary.append(have_dict)
+        headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName')
+        for albums in headphones_albums:
+            headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }  
+            headphones_album_dictionary.append(headphones_dict) 
+        #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]]
+
+        check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary])
+        unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check]
+
+
+        return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums)
+    manageUnmatched.exposed = True
+
+    def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None):
+        myDB = db.DBConnection()
+     
+        if action == "ignoreArtist":
+            artist = existing_artist
+            myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched = "Failed"', [artist])
+        
+        elif action == "ignoreAlbum":
+            artist = existing_artist
+            album = existing_album
+            myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched = "Failed"', (artist, album))
+        
+        elif action == "matchArtist":
+            existing_artist_clean = helpers.cleanName(existing_artist).lower()
+            new_artist_clean = helpers.cleanName(new_artist).lower()
+            if new_artist_clean != existing_artist_clean:
+                have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist])
+                update_count = 0
+                for entry in have_tracks:
+                    old_clean_filename = entry['CleanName']
+                    if old_clean_filename.startswith(existing_artist_clean):
+                        new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1)
+                        myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename])
+                        controlValueDict = {"CleanName": new_clean_filename}
+                        newValueDict = {"Location" : entry['Location'],
+                                        "BitRate" : entry['BitRate'],
+                                        "Format" : entry['Format']
+                                        }
+                        #Attempt to match tracks with new CleanName
+                        match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_alltracks:
+                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                        match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_tracks:
+                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename])
+                            update_count+=1
+                    #This was throwing errors and I don't know why, but it seems to be working fine.
+                    #else:
+                        #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist)
+                logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist))
+                if update_count > 0:
+                    librarysync.update_album_status()
+            else:
+                logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist)
+
+        elif action == "matchAlbum":
+            existing_artist_clean = helpers.cleanName(existing_artist).lower()
+            new_artist_clean = helpers.cleanName(new_artist).lower()
+            existing_album_clean = helpers.cleanName(existing_album).lower()
+            new_album_clean = helpers.cleanName(new_album).lower()
+            existing_clean_string = existing_artist_clean+" "+existing_album_clean
+            new_clean_string = new_artist_clean+" "+new_album_clean
+            if existing_clean_string != new_clean_string:
+                have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album))
+                update_count = 0
+                for entry in have_tracks:
+                    old_clean_filename = entry['CleanName']
+                    if old_clean_filename.startswith(existing_clean_string):
+                        new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1)
+                        myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename])
+                        controlValueDict = {"CleanName": new_clean_filename}
+                        newValueDict = {"Location" : entry['Location'],
+                                        "BitRate" : entry['BitRate'],
+                                        "Format" : entry['Format']
+                                        }
+                        #Attempt to match tracks with new CleanName
+                        match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_alltracks:
+                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                        match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_tracks:
+                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename])
+                            album_id = match_tracks['AlbumID']
+                            update_count+=1
+                    #This was throwing errors and I don't know why, but it seems to be working fine.
+                    #else:
+                        #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string))
+                logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (update_count, new_artist, new_album))
+                if update_count > 0:
+                    librarysync.update_album_status(album_id)
+            else:
+                logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album))
+                
+    markUnmatched.exposed = True
+
+    def manageManual(self):
+        myDB = db.DBConnection()
+        manual_albums = []
+        manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
+        for albums in manualalbums:
+            if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
+                original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle'])
+                if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean:
+                    if albums['Matched'] == "Ignored":
+                        album_status = "Ignored"
+                    elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean:
+                        album_status = "Matched"
+                    manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status }
+                    if manual_dict not in manual_albums: 
+                        manual_albums.append(manual_dict)
+        manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle'))
+
+        return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted)
+    manageManual.exposed = True
+
+    def markManual(self, action=None, existing_artist=None, existing_album=None):
+        myDB = db.DBConnection()
+        if action == "unignoreArtist":
+            artist = existing_artist
+            myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND Matched="Ignored"', [artist])
+            logger.info("Artist: %s successfully restored to unmatched list" % artist)
+
+        elif action == "unignoreAlbum":
+            artist = existing_artist
+            album = existing_album
+            myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album))
+            logger.info("Album: %s successfully restored to unmatched list" % album)
+
+        elif action == "unmatchArtist":
+            artist = existing_artist
+            update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', [artist])
+            update_count = 0
+            for tracks in update_clean:
+                original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower()
+                album = tracks['AlbumTitle']
+                track_title = tracks['TrackTitle']
+                if tracks['CleanName'] != original_clean:
+                    myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title))
+                    update_count+=1
+            if update_count > 0:
+                librarysync.update_album_status()
+            logger.info("Artist: %s successfully restored to unmatched list" % artist)
+
+        elif action == "unmatchAlbum":
+            artist = existing_artist
+            album = existing_album
+            update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', (artist, album))
+            update_count = 0
+            for tracks in update_clean:
+                original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower()
+                track_title = tracks['TrackTitle']
+                if tracks['CleanName'] != original_clean:
+                    album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', [tracks['CleanName']]).fetchone()
+                    if album_id_check:
+                        album_id = album_id_check[0]
+                    myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title))
+                    update_count+=1
+            if update_count > 0:
+                librarysync.update_album_status(album_id)
+            logger.info("Album: %s successfully restored to unmatched list" % album)
+
+    markManual.exposed = True
+
     def markArtists(self, action=None, **args):
         myDB = db.DBConnection()
         artistsToAdd = []
@@ -342,6 +547,8 @@ class WebInterface(object):
                 myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
                 myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
                 myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
+                myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID])
+                myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID])
                 myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID])
             elif action == 'pause':
                 controlValueDict = {'ArtistID': ArtistID}
@@ -397,10 +604,16 @@ class WebInterface(object):
 
     def forceUpdate(self):
         from headphones import updater
-        threading.Thread(target=updater.dbUpdate).start()
+        threading.Thread(target=updater.dbUpdate, args=[False]).start()
         raise cherrypy.HTTPRedirect("home")
     forceUpdate.exposed = True
 
+    def forceFullUpdate(self):
+        from headphones import updater
+        threading.Thread(target=updater.dbUpdate, args=[True]).start()
+        raise cherrypy.HTTPRedirect("home")
+    forceFullUpdate.exposed = True
+
     def forceSearch(self):
         from headphones import searcher
         threading.Thread(target=searcher.searchforalbum).start()
@@ -533,6 +746,20 @@ class WebInterface(object):
         return s
     getArtists_json.exposed=True
 
+    def getAlbumsByArtist_json(self, artist=None):
+        myDB = db.DBConnection()
+        album_json = {}
+        counter = 0
+        album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist])
+        for album in album_list:
+            album_json[counter] = album['AlbumTitle']
+            counter+=1
+        json_albums = json.dumps(album_json)
+        
+        cherrypy.response.headers['Content-type'] = 'application/json'
+        return json_albums
+    getAlbumsByArtist_json.exposed=True
+
     def clearhistory(self, type=None):
         myDB = db.DBConnection()
         if type == 'all':
@@ -554,6 +781,26 @@ class WebInterface(object):
 
     generateAPI.exposed = True
 
+    def forceScan(self, keepmatched=None):
+        myDB = db.DBConnection()
+        #########################################
+        #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE
+        myDB.select('DELETE from Have')
+        logger.info('Removed all entries in local library database')
+        myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL')
+        myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL')
+        logger.info('All tracks in library unmatched')
+        myDB.action('UPDATE artists SET HaveTracks=NULL')
+        logger.info('Reset track counts for all artists')
+        myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"')
+        logger.info('Marking all unwanted albums as Skipped')
+        try:
+            threading.Thread(target=librarysync.libraryScan).start()
+        except Exception, e:
+            logger.error('Unable to complete the scan: %s' % e)
+        raise cherrypy.HTTPRedirect("home")
+    forceScan.exposed = True
+
     def config(self):
 
         interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/')
@@ -565,9 +812,14 @@ class WebInterface(object):
                     "http_port" : headphones.HTTP_PORT,
                     "http_pass" : headphones.HTTP_PASSWORD,
                     "launch_browser" : checked(headphones.LAUNCH_BROWSER),
+                    "enable_https" : checked(headphones.ENABLE_HTTPS),
+                    "https_cert" : headphones.HTTPS_CERT,
+                    "https_key" : headphones.HTTPS_KEY,
                     "api_enabled" : checked(headphones.API_ENABLED),
                     "api_key" : headphones.API_KEY,
                     "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL,
+                    "update_db_interval" : headphones.UPDATE_DB_INTERVAL,
+                    "mb_ignore_age" : headphones.MB_IGNORE_AGE,
                     "nzb_search_interval" : headphones.SEARCH_INTERVAL,
                     "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL,
                     "sab_host" : headphones.SAB_HOST,
@@ -607,6 +859,9 @@ class WebInterface(object):
                     "use_nzbsrus" : checked(headphones.NZBSRUS),
                     "nzbsrus_uid" : headphones.NZBSRUS_UID,
                     "nzbsrus_apikey" : headphones.NZBSRUS_APIKEY,
+                    "use_omgwtfnzbs" : checked(headphones.OMGWTFNZBS),
+                    "omgwtfnzbs_uid" : headphones.OMGWTFNZBS_UID,
+                    "omgwtfnzbs_apikey" : headphones.OMGWTFNZBS_APIKEY,
                     "preferred_words" : headphones.PREFERRED_WORDS,
                     "ignored_words" : headphones.IGNORED_WORDS,
                     "required_words" : headphones.REQUIRED_WORDS,
@@ -616,6 +871,7 @@ class WebInterface(object):
                     "use_isohunt" : checked(headphones.ISOHUNT),
                     "use_kat" : checked(headphones.KAT),
                     "use_piratebay" : checked(headphones.PIRATEBAY),
+                    "piratebay_proxy_url" : headphones.PIRATEBAY_PROXY_URL,
                     "use_mininova" : checked(headphones.MININOVA),
                     "use_waffles" : checked(headphones.WAFFLES),
                     "waffles_uid" : headphones.WAFFLES_UID,
@@ -647,6 +903,7 @@ class WebInterface(object):
                     "lossless_dest_dir" : headphones.LOSSLESS_DESTINATION_DIR,
                     "folder_format" : headphones.FOLDER_FORMAT,
                     "file_format" : headphones.FILE_FORMAT,
+                    "file_underscores" : checked(headphones.FILE_UNDERSCORES),
                     "include_extras" : checked(headphones.INCLUDE_EXTRAS),
                     "autowant_upcoming" : checked(headphones.AUTOWANT_UPCOMING),
                     "autowant_all" : checked(headphones.AUTOWANT_ALL),
@@ -693,6 +950,8 @@ class WebInterface(object):
                     "hpuser": headphones.HPUSER,
                     "hppass": headphones.HPPASS,
                     "cache_sizemb": headphones.CACHE_SIZEMB,
+                    "file_permissions": headphones.FILE_PERMISSIONS,
+                    "folder_permissions": headphones.FOLDER_PERMISSIONS
                 }
 
         # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8
@@ -713,29 +972,35 @@ class WebInterface(object):
     config.exposed = True
 
     def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None,
-        download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None,
+        download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None,
         sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, 
         utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, 
-        use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, 
+        use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, omgwtfnzbs=0, omgwtfnzbs_uid=None, omgwtfnzbs_apikey=None, 
         preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None,
-        numberofseeders=None, use_piratebay=0, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None,
+        numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None,
         rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0,
-        destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0,
+        destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0,
         remix=0, spokenword=0, audiobook=0, autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None,
         bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0,
         delete_lossless_files=0, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None,
         xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, synoindex_enabled=False,
         pushover_enabled=0, pushover_onsnatch=0, pushover_keys=None, pushover_priority=0, mirror=None, customhost=None, customport=None,
-        customsleep=None, hpuser=None, hppass=None, preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, preferred_bitrate_allow_lossless=0, cache_sizemb=None, **kwargs):
+        customsleep=None, hpuser=None, hppass=None, preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, preferred_bitrate_allow_lossless=0, cache_sizemb=None, 
+        enable_https=0, https_cert=None, https_key=None, file_permissions=None, folder_permissions=None, **kwargs):
 
         headphones.HTTP_HOST = http_host
         headphones.HTTP_PORT = http_port
         headphones.HTTP_USERNAME = http_username
         headphones.HTTP_PASSWORD = http_password
         headphones.LAUNCH_BROWSER = launch_browser
+        headphones.ENABLE_HTTPS = enable_https
+        headphones.HTTPS_CERT = https_cert
+        headphones.HTTPS_KEY = https_key
         headphones.API_ENABLED = api_enabled
         headphones.API_KEY = api_key
         headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval
+        headphones.UPDATE_DB_INTERVAL = update_db_interval
+        headphones.MB_IGNORE_AGE = mb_ignore_age
         headphones.SEARCH_INTERVAL = nzb_search_interval
         headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval
         headphones.SAB_HOST = sab_host
@@ -769,6 +1034,9 @@ class WebInterface(object):
         headphones.NZBSRUS = nzbsrus
         headphones.NZBSRUS_UID = nzbsrus_uid
         headphones.NZBSRUS_APIKEY = nzbsrus_apikey
+        headphones.OMGWTFNZBS = omgwtfnzbs
+        headphones.OMGWTFNZBS_UID = omgwtfnzbs_uid
+        headphones.OMGWTFNZBS_APIKEY = omgwtfnzbs_apikey
         headphones.PREFERRED_WORDS = preferred_words
         headphones.IGNORED_WORDS = ignored_words
         headphones.REQUIRED_WORDS = required_words
@@ -778,6 +1046,7 @@ class WebInterface(object):
         headphones.ISOHUNT = use_isohunt
         headphones.KAT = use_kat
         headphones.PIRATEBAY = use_piratebay
+        headphones.PIRATEBAY_PROXY_URL = piratebay_proxy_url
         headphones.MININOVA = use_mininova
         headphones.WAFFLES = waffles
         headphones.WAFFLES_UID = waffles_uid
@@ -806,6 +1075,7 @@ class WebInterface(object):
         headphones.LOSSLESS_DESTINATION_DIR = lossless_destination_dir
         headphones.FOLDER_FORMAT = folder_format
         headphones.FILE_FORMAT = file_format
+        headphones.FILE_UNDERSCORES = file_underscores
         headphones.INCLUDE_EXTRAS = include_extras
         headphones.AUTOWANT_UPCOMING = autowant_upcoming
         headphones.AUTOWANT_ALL = autowant_all
@@ -851,6 +1121,8 @@ class WebInterface(object):
         headphones.HPUSER = hpuser
         headphones.HPPASS = hppass
         headphones.CACHE_SIZEMB = int(cache_sizemb)
+        headphones.FILE_PERMISSIONS = file_permissions
+        headphones.FOLDER_PERMISSIONS = folder_permissions
 
         # Handle the variable config options. Note - keys with False values aren't getting passed
 

headphones/webstart.py

@@ -20,12 +20,31 @@ import cherrypy
 
 import headphones
 
+from headphones import logger
 from headphones.webserve import WebInterface
+from headphones.helpers import create_https_certificates
 
 def initialize(options={}):
 
+    #HTTPS stuff stolen from sickbeard
+    enable_https = options['enable_https']
+    https_cert = options['https_cert']
+    https_key = options['https_key']
 
-    cherrypy.config.update({
+    if enable_https:
+        # If either the HTTPS certificate or key do not exist, make some self-signed ones.
+        if not (https_cert and os.path.exists(https_cert)) or not (https_key and os.path.exists(https_key)):
+            if not create_https_certificates(https_cert, https_key):
+                logger.warn(u"Unable to create cert/key files, disabling HTTPS")
+                headphones.ENABLE_HTTPS = False
+                enable_https = False
+
+        if not (os.path.exists(https_cert) and os.path.exists(https_key)):
+            logger.warn(u"Disabled HTTPS because of missing CERT and KEY files")
+            headphones.ENABLE_HTTPS = False
+            enable_https = False
+        
+    options_dict = {
                 'log.screen':           False,
                 'server.thread_pool':   10,
                 'server.socket_port':   options['http_port'],
@@ -34,7 +53,17 @@ def initialize(options={}):
                 'tools.encode.on' : True,
                 'tools.encode.encoding' : 'utf-8',
                 'tools.decode.on' : True,
-        })
+        }
+        
+    if enable_https:
+        options_dict['server.ssl_certificate'] = https_cert
+        options_dict['server.ssl_private_key'] = https_key
+        protocol = "https"
+    else:
+        protocol = "http"
+    
+    logger.info(u"Starting Headphones on " + protocol + "://" + str(options['http_host']) + ":" + str(options['http_port']) + "/")
+    cherrypy.config.update(options_dict)
 
     conf = {
         '/': {

lib/beets/mediafile.py

@@ -572,9 +572,23 @@ class ImageField(object):
 
             # No cover found.
             return None
+	elif obj.type == 'flac':
+            if 'metadata_block_picture' not in obj.mgfile:
+                return None
+
+            for data in obj.mgfile['metadata_block_picture']:
+                try:
+                    pic = lib.mutagen.flac.Picture(data)
+                    break
+                except TypeError:
+                    pass
+            else:
+                return None
+
+            return pic.data
 
         else:
-            # Here we're assuming everything but MP3 and MPEG-4 uses
+	    # Here we're assuming everything but MP3, FLAC and MPEG-4 use
             # the Xiph/Vorbis Comments standard. This may not be valid.
             # http://wiki.xiph.org/VorbisComment#Cover_art
 
@@ -624,6 +638,13 @@ class ImageField(object):
                 cover = lib.mutagen.mp4.MP4Cover(val, self._mp4kind(val))
                 obj.mgfile['covr'] = [cover]
 	
+        elif obj.type == 'flac':
+            if val is None:
+                pic = lib.mutagen.flac.Picture()
+                pic.data = val
+                pic.mime = self._mime(val)
+                obj.mgfile['metadata_block_picture'] = [pic.write()]
+
         else:
             # Again, assuming Vorbis Comments standard.
 

lib/certgen.py

@@ -0,0 +1,82 @@
+# -*- coding: latin-1 -*-
+#
+# Copyright (C) Martin Sjögren and AB Strakt 2001, All rights reserved
+# Copyright (C) Jean-Paul Calderone 2008, All rights reserved
+# This file is licenced under the GNU LESSER GENERAL PUBLIC LICENSE Version 2.1 or later (aka LGPL v2.1)
+# Please see LGPL2.1.txt for more information
+"""
+Certificate generation module.
+"""
+
+from OpenSSL import crypto
+import time
+
+TYPE_RSA = crypto.TYPE_RSA
+TYPE_DSA = crypto.TYPE_DSA
+
+serial = int(time.time())
+
+
+def createKeyPair(type, bits):
+    """
+    Create a public/private key pair.
+
+    Arguments: type - Key type, must be one of TYPE_RSA and TYPE_DSA
+               bits - Number of bits to use in the key
+    Returns:   The public/private key pair in a PKey object
+    """
+    pkey = crypto.PKey()
+    pkey.generate_key(type, bits)
+    return pkey
+
+def createCertRequest(pkey, digest="md5", **name):
+    """
+    Create a certificate request.
+
+    Arguments: pkey   - The key to associate with the request
+               digest - Digestion method to use for signing, default is md5
+               **name - The name of the subject of the request, possible
+                        arguments are:
+                          C     - Country name
+                          ST    - State or province name
+                          L     - Locality name
+                          O     - Organization name
+                          OU    - Organizational unit name
+                          CN    - Common name
+                          emailAddress - E-mail address
+    Returns:   The certificate request in an X509Req object
+    """
+    req = crypto.X509Req()
+    subj = req.get_subject()
+
+    for (key,value) in name.items():
+        setattr(subj, key, value)
+
+    req.set_pubkey(pkey)
+    req.sign(pkey, digest)
+    return req
+
+def createCertificate(req, (issuerCert, issuerKey), serial, (notBefore, notAfter), digest="md5"):
+    """
+    Generate a certificate given a certificate request.
+
+    Arguments: req        - Certificate reqeust to use
+               issuerCert - The certificate of the issuer
+               issuerKey  - The private key of the issuer
+               serial     - Serial number for the certificate
+               notBefore  - Timestamp (relative to now) when the certificate
+                            starts being valid
+               notAfter   - Timestamp (relative to now) when the certificate
+                            stops being valid
+               digest     - Digest method to use for signing, default is md5
+    Returns:   The signed certificate in an X509 object
+    """
+    cert = crypto.X509()
+    cert.set_serial_number(serial)
+    cert.gmtime_adj_notBefore(notBefore)
+    cert.gmtime_adj_notAfter(notAfter)
+    cert.set_issuer(issuerCert.get_subject())
+    cert.set_subject(req.get_subject())
+    cert.set_pubkey(req.get_pubkey())
+    cert.sign(issuerKey, digest)
+    return cert

lib/pygazelle/api.py

@@ -5,18 +5,21 @@
 #
 # Loosely based on the API implementation from 'whatbetter', by Zachary Denton
 # See https://github.com/zacharydenton/whatbetter
+from HTMLParser import HTMLParser
 
+import sys
 import json
 import time
 import lib.requests as requests
 
-from user import User
+from .user import User
-from artist import Artist
+from .artist import Artist
-from tag import Tag
+from .tag import Tag
-from request import Request
+from .request import Request
-from torrent_group import TorrentGroup
+from .torrent_group import TorrentGroup
-from torrent import Torrent
+from .torrent import Torrent
-from category import Category
+from .category import Category
+from .inbox import Mailbox
 
 class LoginException(Exception):
     pass
@@ -40,13 +43,15 @@ class GazelleAPI(object):
 
 
     def __init__(self, username=None, password=None):
-        self.session = requests.session(headers=self.default_headers)
+        self.session = requests.session()
+        self.session.headers = self.default_headers
         self.username = username
         self.password = password
         self.authkey = None
         self.passkey = None
         self.userid = None
         self.logged_in_user = None
+        self.default_timeout = 30
         self.cached_users = {}
         self.cached_artists = {}
         self.cached_tags = {}
@@ -55,36 +60,72 @@ class GazelleAPI(object):
         self.cached_requests = {}
         self.cached_categories = {}
         self.site = "https://what.cd/"
-        self.rate_limit = 2.0 # seconds between requests
+        self.past_request_timestamps = []
-        self._login()
+
+    def wait_for_rate_limit(self):
+        # maximum is 5 requests within 10 secs
+        time_frame = 10
+        max_reqs = 5
+
+        slice_point = 0
+
+        while len(self.past_request_timestamps) >= max_reqs:
+            for i, timestamp in enumerate(self.past_request_timestamps):
+                if timestamp < time.time() - time_frame:
+                    slice_point = i + 1
+                else:
+                    break
+
+            if slice_point:
+                self.past_request_timestamps = self.past_request_timestamps[slice_point:]
+            else:
+                time.sleep(0.1)
+
+    def logged_in(self):
+        return self.logged_in_user is not None and self.logged_in_user.id == self.userid
 
     def _login(self):
         """
         Private method.
         Logs in user and gets authkey from server.
         """
+
+        if self.logged_in():
+            return
+
+        self.wait_for_rate_limit()
+
         loginpage = 'https://what.cd/login.php'
         data = {'username': self.username,
                 'password': self.password}
-        r = self.session.post(loginpage, data=data)
+        r = self.session.post(loginpage, data=data, timeout=self.default_timeout)
+        self.past_request_timestamps.append(time.time())
         if r.status_code != 200:
-            raise LoginException("Login error, http code %s" % r.status_code)
+            raise LoginException("Login returned status code %s" % r.status_code)
-        accountinfo = self.request('index')
+
+        try:
+            accountinfo = self.request('index', autologin=False)
+        except RequestException as e:
+            raise LoginException("Login probably incorrect")
+        if not accountinfo or 'id' not in accountinfo:
+            raise LoginException("Login probably incorrect")
         self.userid = accountinfo['id']
         self.authkey = accountinfo['authkey']
         self.passkey = accountinfo['passkey']
         self.logged_in_user = User(self.userid, self)
         self.logged_in_user.set_index_data(accountinfo)
 
-    def request(self, action, **kwargs):
+    def request(self, action, autologin=True, **kwargs):
         """
         Makes an AJAX request at a given action.
         Pass an action and relevant arguments for that action.
         """
-
+        def make_request(action, **kwargs):
             ajaxpage = 'ajax.php'
             content = self.unparsed_request(ajaxpage, action, **kwargs)
             try:
+                if not isinstance(content, text_type):
+                    content = content.decode('utf-8')
                 parsed = json.loads(content)
                 if parsed['status'] != 'success':
                     raise RequestException
@@ -92,21 +133,29 @@ class GazelleAPI(object):
             except ValueError:
                 raise RequestException
 
-    def unparsed_request(self, page, action, **kwargs):
+        try:
+            return make_request(action, **kwargs)
+        except Exception as e:
+            if autologin and not self.logged_in():
+                self._login()
+                return make_request(action, **kwargs)
+            else:
+                raise e
+
+    def unparsed_request(self, sitepage, action, **kwargs):
         """
         Makes a generic HTTP request at a given page with a given action.
         Also pass relevant arguments for that action.
         """
-        while time.time() - self.last_request < self.rate_limit:
+        self.wait_for_rate_limit()
-            time.sleep(0.1)
 
-        url = "%s/%s" % (self.site, page)
+        url = "%s%s" % (self.site, sitepage)
         params = {'action': action}
         if self.authkey:
             params['auth'] = self.authkey
         params.update(kwargs)
-        r = self.session.get(url, params=params, allow_redirects=False)
+        r = self.session.get(url, params=params, allow_redirects=False, timeout=self.default_timeout)
-        self.last_request = time.time()
+        self.past_request_timestamps.append(time.time())
         return r.content
 
     def get_user(self, id):
@@ -141,18 +190,37 @@ class GazelleAPI(object):
 
         return found_users
 
-    def get_artist(self, id, name=None):
+    def get_inbox(self, page='1', sort='unread'):
+        """
+        Returns the inbox Mailbox for the logged in user
+        """
+        return Mailbox(self, 'inbox', page, sort)
+        
+    def get_sentbox(self, page='1', sort='unread'):
+        """
+        Returns the sentbox Mailbox for the logged in user
+        """
+        return Mailbox(self, 'sentbox', page, sort)
+
+    def get_artist(self, id=None, name=None):
         """
         Returns an Artist for the passed ID, associated with this API object. You'll need to call Artist.update_data()
         if the artist hasn't already been cached. This is done on demand to reduce unnecessary API calls.
         """
+        if id:
             id = int(id)
             if id in self.cached_artists.keys():
                 artist = self.cached_artists[id]
             else:
                 artist = Artist(id, self)
             if name:
-            artist.name = name
+                artist.name = HTMLParser().unescape(name)
+        elif name:
+            artist = Artist(-1, self)
+            artist.name = HTMLParser().unescape(name)
+        else:
+            raise Exception("You must specify either an ID or a Name to get an artist.")
+
         return artist
 
     def get_tag(self, name):
@@ -189,7 +257,7 @@ class GazelleAPI(object):
 
     def get_torrent(self, id):
         """
-        Returns a TorrentGroup for the passed ID, associated with this API object.
+        Returns a Torrent for the passed ID, associated with this API object.
         """
         id = int(id)
         if id in self.cached_torrents.keys():
@@ -197,6 +265,24 @@ class GazelleAPI(object):
         else:
             return Torrent(id, self)
 
+    def get_torrent_from_info_hash(self, info_hash):
+        """
+        Returns a Torrent for the passed info hash (if one exists), associated with this API object.
+        """
+        try:
+            response = self.request(action='torrent', hash=info_hash.upper())
+        except RequestException:
+            return None
+
+        id = int(response['torrent']['id'])
+        if id in self.cached_torrents.keys():
+            torrent = self.cached_torrents[id]
+        else:
+            torrent = Torrent(id, self)
+
+        torrent.set_torrent_complete_data(response)
+        return torrent
+
     def get_category(self, id, name=None):
         """
         Returns a Category for the passed ID, associated with this API object.
@@ -210,6 +296,45 @@ class GazelleAPI(object):
             cat.name = name
         return cat
 
+    def get_top_10(self, type="torrents", limit=25):
+        """
+        Lists the top <limit> items of <type>. Type can be "torrents", "tags", or "users". Limit MUST be
+        10, 25, or 100...it can't just be an arbitrary number (unfortunately). Results are organized into a list of hashes.
+        Each hash contains the results for a specific time frame, like 'day', or 'week'. In the hash, the 'results' key
+        contains a list of objects appropriate to the passed <type>.
+        """
+
+        response = self.request(action='top10', type=type, limit=limit)
+        top_items = []
+        if not response:
+            raise RequestException
+        for category in response:
+            results = []
+            if type == "torrents":
+                for item in category['results']:
+                    torrent = self.get_torrent(item['torrentId'])
+                    torrent.set_torrent_top_10_data(item)
+                    results.append(torrent)
+            elif type == "tags":
+                for item in category['results']:
+                    tag = self.get_tag(item['name'])
+                    results.append(tag)
+            elif type == "users":
+                for item in category['results']:
+                    user = self.get_user(item['id'])
+                    results.append(user)
+            else:
+                raise Exception("%s is an invalid type argument for GazelleAPI.get_top_ten()" % type)
+
+            top_items.append({
+                "caption": category['caption'],
+                "tag": category['tag'],
+                "limit": category['limit'],
+                "results": results
+            })
+
+        return top_items
+
     def search_torrents(self, **kwargs):
         """
         Searches based on the args you pass and returns torrent groups filled with torrents.
@@ -281,3 +406,8 @@ class GazelleAPI(object):
             id=id, authkey=self.logged_in_user.authkey, torrent_pass=self.logged_in_user.passkey)
         with open(dest, 'w+') as dest_file:
             dest_file.write(file_data)
+
+if sys.version_info[0] == 3:
+    text_type = str
+else:
+    text_type = unicode

lib/pygazelle/artist.py

@@ -1,4 +1,4 @@
-
+from HTMLParser import HTMLParser
 
 class InvalidArtistException(Exception):
     pass
@@ -26,15 +26,28 @@ class Artist(object):
         self.parent_api.cached_artists[self.id] = self # add self to cache of known Artist objects
 
     def update_data(self):
+        if self.id > 0:
             response = self.parent_api.request(action='artist', id=self.id)
+        elif self.name:
+            self.name = HTMLParser().unescape(self.name)
+            try:
+                response = self.parent_api.request(action='artist', artistname=self.name)
+            except Exception:
+                self.name = self.name.split(" & ")[0]
+                response = self.parent_api.request(action='artist', artistname=self.name)
+        else:
+            raise InvalidArtistException("Neither ID or Artist Name is valid, can't update data.")
         self.set_data(response)
 
     def set_data(self, artist_json_response):
-        if self.id != artist_json_response['id']:
+        if self.id > 0 and self.id != artist_json_response['id']:
             raise InvalidArtistException("Tried to update an artists's information from an 'artist' API call with a different id." +
                                " Should be %s, got %s" % (self.id, artist_json_response['id']) )
+        elif self.name:
+            self.id = artist_json_response['id']
+            self.parent_api.cached_artists[self.id] = self
 
-        self.name = artist_json_response['name']
+        self.name = HTMLParser().unescape(artist_json_response['name'])
         self.notifications_enabled = artist_json_response['notificationsEnabled']
         self.has_bookmarked = artist_json_response['hasBookmarked']
         self.image = artist_json_response['image']

lib/pygazelle/inbox.py

@@ -0,0 +1,107 @@
+class MailboxMessage(object):
+    def __init__(self, api, message):
+        self.id = message['convId']
+        self.conv = Conversation(api, self.id)
+        self.subject = message['subject']
+        self.unread = message['unread']
+        self.sticky = message['sticky']
+        self.fwd_id = message['forwardedId']
+        self.fwd_name = message['forwardedName']
+        self.sender_id = message['senderId']
+        self.username = message['username']
+        self.donor = message['donor']
+        self.warned = message['warned']
+        self.enabled = message['enabled']
+        self.date = message['date']
+
+    def __repr__(self):
+        return "MailboxMessage ID %s - %s %s %s" % (self.id, self.subject, self.sender_id, self.username)
+
+
+class ConversationMessage(object):
+    def __init__(self, msg_resp):
+        self.id = msg_resp['messageId']
+        self.sender_id = msg_resp['senderId']
+        self.sender_name = msg_resp['senderName']
+        self.sent_date = msg_resp['sentDate']
+        self.bb_body = msg_resp['bbBody']
+        self.body = msg_resp['body']
+
+    def __repr__(self):
+        return "ConversationMessage ID %s - %s %s" % (self.id, self.sender_name, self.sent_date)
+
+
+class Conversation(object):
+    def __init__(self, api, conv_id):
+        self.id = conv_id
+        self.parent_api = api
+        self.subject = None
+        self.sticky = None
+        self.messages = []
+
+    def __repr__(self):
+        return "Conversation ID %s - %s" % (self.id, self.subject)
+
+    def set_conv_data(self, conv_resp):
+        assert self.id == conv_resp['convId']
+        self.subject = conv_resp['subject']
+        self.sticky = conv_resp['sticky']
+        self.messages = [ConversationMessage(m) for m in conv_resp['messages']]
+
+    def update_conv_data(self):
+        response = self.parent_api.request(action='inbox',
+                                           type='viewconv', id=self.id)
+        self.set_conv_data(response)
+
+
+class Mailbox(object):
+    """
+    This class represents the logged in user's inbox/sentbox
+    """
+    def __init__(self, parent_api, boxtype='inbox', page='1', sort='unread'): 
+        self.parent_api = parent_api
+        self.boxtype = boxtype 
+        self.current_page = page
+        self.total_pages = None
+        self.sort = sort
+        self.messages = None
+
+    def set_mbox_data(self, mbox_resp):
+        """
+        Takes parsed JSON response from 'inbox' action on api
+        and updates the available subset of mailbox information.
+        """
+        self.current_page = mbox_resp['currentPage']
+        self.total_pages = mbox_resp['pages']
+        self.messages = \
+          [MailboxMessage(self.parent_api, m) for m in mbox_resp['messages']]
+
+    def update_mbox_data(self):
+        response = self.parent_api.request(action='inbox',
+                     type=self.boxtype, page=self.current_page, sort=self.sort)
+        self.set_mbox_data(response)
+
+    def next_page(self):
+        if not self.total_pages:
+            raise ValueError("call update_mbox_data() first")
+        total_pages = int(self.total_pages)
+        cur_page = int(self.current_page)
+        if cur_page < total_pages:
+            return Mailbox(self.parent_api, self.boxtype,
+                           str(cur_page + 1), self.sort)
+        raise ValueError("Already at page %d/%d" % (cur_page, total_pages))
+
+    def prev_page(self):
+        if not self.total_pages:
+            raise ValueError("call update_mbox_data() first")
+        total_pages = int(self.total_pages)
+        cur_page = int(self.current_page)
+        if cur_page > 1:
+            return Mailbox(self.parent_api, self.boxtype,
+                           str(cur_page - 1), self.sort)
+        raise ValueError("Already at page %d/%d" % (cur_page, total_pages))
+
+    def __repr__(self):
+        return "Mailbox: %s %s Page %s/%s" \
+                 % (self.boxtype, self.sort,
+                    self.current_page, self.total_pages)

lib/pygazelle/torrent.py

@@ -1,3 +1,4 @@
+from HTMLParser import HTMLParser
 import re
 
 class InvalidTorrentException(Exception):
@@ -35,6 +36,40 @@ class Torrent(object):
 
         self.parent_api.cached_torrents[self.id] = self
 
+    def set_torrent_complete_data(self, torrent_json_response):
+        if self.id != torrent_json_response['torrent']['id']:
+            raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." +
+                                       " Should be %s, got %s" % (self.id, torrent_json_response['id']) )
+
+        self.group = self.parent_api.get_torrent_group(torrent_json_response['group']['id'])
+        had_complete_list = self.group.has_complete_torrent_list
+        self.group.set_group_data(torrent_json_response)
+        self.group.has_complete_torrent_list = had_complete_list
+
+        self.media = torrent_json_response['torrent']['media']
+        self.format = torrent_json_response['torrent']['format']
+        self.encoding = torrent_json_response['torrent']['encoding']
+        self.remaster_year = torrent_json_response['torrent']['remasterYear']
+        self.remastered = torrent_json_response['torrent']['remastered']
+        self.remaster_title = torrent_json_response['torrent']['remasterTitle']
+        self.remaster_record_label = torrent_json_response['torrent']['remasterRecordLabel']
+        self.scene = torrent_json_response['torrent']['scene']
+        self.has_log = torrent_json_response['torrent']['hasLog']
+        self.has_cue = torrent_json_response['torrent']['hasCue']
+        self.log_score = torrent_json_response['torrent']['logScore']
+        self.file_count = torrent_json_response['torrent']['fileCount']
+        self.free_torrent = torrent_json_response['torrent']['freeTorrent']
+        self.size = torrent_json_response['torrent']['size']
+        self.leechers = torrent_json_response['torrent']['leechers']
+        self.seeders = torrent_json_response['torrent']['seeders']
+        self.snatched = torrent_json_response['torrent']['snatched']
+        self.time = torrent_json_response['torrent']['time']
+        self.description = torrent_json_response['torrent']['description']
+        self.file_list = [ re.match("(.+){{{(\d+)}}}", item).groups()
+                           for item in torrent_json_response['torrent']['fileList'].split("|||") ] # tuple ( filename, filesize )
+        self.file_path = torrent_json_response['torrent']['filePath']
+        self.user = self.parent_api.get_user(torrent_json_response['torrent']['userId'])
+
     def set_torrent_artist_data(self, artist_torrent_json_response):
         if self.id != artist_torrent_json_response['id']:
             raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." +
@@ -118,6 +153,26 @@ class Torrent(object):
         self.free_torrent = search_torrent_json_response['isFreeleech'] or search_torrent_json_response['isPersonalFreeleech']
         self.time = search_torrent_json_response['time']
 
+    def set_torrent_top_10_data(self, top_10_json_response):
+        if self.id != top_10_json_response['torrentId']:
+            raise InvalidTorrentException("Tried to update a Torrent's information from a 'browse'/search API call with a different id." +
+                                          " Should be %s, got %s" % (self.id, top_10_json_response['torrentId']) )
+
+        # TODO: Add conditionals to handle torrents that aren't music
+        self.group = self.parent_api.get_torrent_group(top_10_json_response['groupId'])
+        self.group.name = top_10_json_response['groupName']
+        if not self.group.music_info and top_10_json_response['artist']:
+            self.group.music_info = {'artists': [self.parent_api.get_artist(name=HTMLParser().unescape(top_10_json_response['artist']))]}
+        self.remaster_title = top_10_json_response['remasterTitle']
+        self.media = top_10_json_response['media']
+        self.format = top_10_json_response['format']
+        self.encoding = top_10_json_response['encoding']
+        self.has_log = top_10_json_response['hasLog']
+        self.has_cue = top_10_json_response['hasCue']
+        self.scene = top_10_json_response['scene']
+        self.seeders = top_10_json_response['seeders']
+        self.leechers = top_10_json_response['leechers']
+        self.snatched = top_10_json_response['snatched']
 
 
     def __repr__(self):

lib/pygazelle/torrent_group.py

@@ -1,4 +1,4 @@
-from torrent import Torrent
+from .torrent import Torrent
 
 class InvalidTorrentGroupException(Exception):
     pass
@@ -62,6 +62,7 @@ class TorrentGroup(object):
         self.music_info['with'] = [ self.parent_api.get_artist(artist['id'], artist['name'])
                                        for artist in self.music_info['with'] ]
 
+        if 'torrents' in torrent_group_json_response:
             self.torrents = []
             for torrent_dict in torrent_group_json_response['torrents']:
                 torrent_dict['groupId'] = self.id
@@ -69,6 +70,9 @@ class TorrentGroup(object):
                 torrent.set_torrent_group_data(torrent_dict)
                 self.torrents.append(torrent)
             self.has_complete_torrent_list = True
+        elif 'torrent' in torrent_group_json_response:
+            torrent = self.parent_api.get_torrent(torrent_group_json_response['torrent']['id'])
+            self.torrents.append(torrent)
 
     def set_artist_group_data(self, artist_group_json_response):
         """

lib/pygazelle/user.py

@@ -67,9 +67,9 @@ class User(object):
         Takes parsed JSON response from 'user' action on api, and updates relevant user information.
         To avoid problems, only pass in user data from an API call that used this user's ID as an argument.
         """
-        if self.id != user_json_response['id']:
+        if self.username and self.username != user_json_response['username']:
-            raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different id." +
+            raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different username." +
-                               " Should be %s, got %s" % (self.id, user_json_response['id']) )
+                                       " Should be %s, got %s" % (self.username, user_json_response['username']) )
 
         self.username = user_json_response['username']
         self.avatar = user_json_response['avatar']