Merge branch 'develop'

data/interfaces/default/album.html

@@ -21,7 +21,7 @@
 				<div id="dialog" title="Choose an Alternate Release" style="display:none" class="configtable">
 					<div class="links">
 						<%
-							alternate_albums = myDB.select("SELECT * from allalbums WHERE AlbumID=?", [album['AlbumID']])
+							alternate_albums = myDB.select("SELECT * from allalbums WHERE AlbumID=? ORDER BY ReleaseDate ASC", [album['AlbumID']])
 						%>
 						%if not alternate_albums:
 							<p>No alternate releases found. Try refreshing the artist (if the artist is being refreshed, please wait until it's finished)</p>
@@ -32,9 +32,9 @@
 									track_count = len(myDB.select("SELECT * from alltracks WHERE ReleaseID=?", [alternate_album['ReleaseID']]))
 									have_track_count = len(myDB.select("SELECT * from alltracks WHERE ReleaseID=? AND Location IS NOT NULL", [alternate_album['ReleaseID']]))
 									if alternate_album['AlbumID'] == alternate_album['ReleaseID']:
-										alternate_album_name = "Headphones Default Release [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
+										alternate_album_name = "Headphones Default Release (" + str(alternate_album['ReleaseDate']) + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
 									else:
-										alternate_album_name = alternate_album['AlbumTitle'] + " (" + alternate_album['ReleaseCountry'] + ", " + alternate_album['ReleaseFormat'] + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
+										alternate_album_name = alternate_album['AlbumTitle'] + " (" + alternate_album['ReleaseCountry'] + ", " + str(alternate_album['ReleaseDate']) + ", " + alternate_album['ReleaseFormat'] + ") [" + str(have_track_count) + "/" + str(track_count) + " tracks]"
 									
 								%>
 								<a href="#" onclick="doAjaxCall('switchAlbum?AlbumID=${album['AlbumID']}&ReleaseID=${alternate_album['ReleaseID']}', $(this), 'table');" data-success="Switched release to: ${alternate_album_name}">${alternate_album_name}</a><br>
@@ -138,7 +138,7 @@
 					</tr>
 				%endfor
 				<%
-				unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])			
+				unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched = "Failed" ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']])
 				%>
 				%if unmatched:
 					 %for track in unmatched:

data/interfaces/default/artist.html

@@ -93,7 +93,7 @@
 
 				myDB = db.DBConnection()					
 				totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']]))
-				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]))
+				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']]))
 				
 				try:
 					percent = (havetracks*100.0)/totaltracks

data/interfaces/default/config.html

@@ -95,7 +95,15 @@
                                 </div>
                                 <div class="row">
                                     <label>Library Scan Interval</label>
-                                    <input type="text" name="libraryscan_interval" value="${config['libraryscan_interval']}" size="4">mins
+                                    <input type="text" name="libraryscan_interval" value="${config['libraryscan_interval']}" size="4">hours
+                                </div>
+                                <div class="row">
+                                    <label>MusicBrainz Update Interval</label>
+                                    <input type="text" name="update_db_interval" value="${config['update_db_interval']}" size="4">hours
+                                </div>
+                                <div class="row">
+                                    <label>Ignore Album Updates</label>
+                                    <input type="text" name="mb_ignore_age" value="${config['mb_ignore_age']}" size="4">days
                                 </div>
                             </fieldset>
                         </td>
@@ -354,6 +362,22 @@
                                     </div>
                                 </div>
                             </fieldset>
+                            <fieldset>
+                                <legend>omgwtfnzbs</legend>
+                                <div class="row checkbox">
+                                    <input id="useomgwtfnzbs" type="checkbox" name="omgwtfnzbs" onclick="initConfigCheckbox($(this));" value="1" ${config['use_omgwtfnzbs']} /><label>Use omgwtfnzbs</label>
+                                </div>
+                                <div class="config">
+                                    <div class="row">
+                                        <label>omgwtfnzbs UID</label>
+                                        <input type="text" name="omgwtfnzbs_uid" value="${config['omgwtfnzbs_uid']}" size="10">
+                                    </div>
+                                    <div class="row">
+                                        <label>omgwtfnzbs API Key</label>
+                                        <input type="text" name="omgwtfnzbs_apikey" value="${config['omgwtfnzbs_apikey']}" size="10">
+                                    </div>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -1259,6 +1283,7 @@
                 initConfigCheckbox("#usenewznab");
                 initConfigCheckbox("#usenzbsrus");
                 initConfigCheckbox("#usenzbsorg");
+                initConfigCheckbox("#useomgwtfnzbs");
                 initConfigCheckbox("#usepiratebay");
                 initConfigCheckbox("#usewaffles");
                 initConfigCheckbox("#userutracker");

data/interfaces/default/js/script.js

@@ -315,6 +315,7 @@ function doAjaxCall(url,elem,reload,form) {
 	  				console.log('refresh'); refreshTable();
 	  			}
 	  			if ( reload == "tabs") 	refreshTab();
+	  			if ( reload == "page") 	location.reload();
 	  			if ( form ) {
 	  				// Change the option to 'choose...'
 	  				$(formID + " select").children('option[disabled=disabled]').attr('selected','selected');

data/interfaces/default/manage.html

@@ -22,6 +22,7 @@
 			%if not headphones.ADD_ARTISTS:
 			<a class="menu_link_edit" href="manageNew">Manage New Artists</a>
 			%endif
+			<a class="menu_link_edit" href="manageUnmatched">Manage Unmatched</a>
 		</div>
 	</div>	
 </%def>
@@ -35,6 +36,7 @@
 			<li><a href="#tabs-1">Scan Music Library</a></li>
 			<li><a href="#tabs-2">Imports</a></li>
 			<li><a href="#tabs-3">Force Actions</a></li>
+			<li><a href="#tabs-4">Force Legacy</a></li>
 		</ul>
 		<div id="tabs-1" class="configtable">
 			<fieldset>
@@ -119,7 +121,7 @@
 			<legend>Force Search</legend>
 			<div class="links">
 				<a href="#" onclick="doAjaxCall('forceSearch',$(this))" data-success="Checking for wanted albums successful" data-error="Error checking wanted albums"><span class="ui-icon ui-icon-search"></span>Force Check for Wanted Albums</a>
-				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists</a>
+				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists [Fast]</a>
 				<a href="#" onclick="doAjaxCall('forcePostProcess',$(this))" data-success="Post-Processor is being loaded" data-error="Error during Post-Processing"><span class="ui-icon ui-icon-wrench"></span>Force Post-Process Albums in Download Folder</a>
 				<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><span class="ui-icon ui-icon-refresh"></span>Check for Headphones Updates</a>
 				<a href="#" id="delete_empty_artists"><span class="ui-icon ui-icon-trash"></span>Delete empty Artists</a>
@@ -139,6 +141,26 @@
 		</fieldset>
 		
 	</div>
+
+
+	<div id="tabs-4" class="configtable">
+		<fieldset>
+			<legend>Force Legacy</legend>
+				<p>Please note that these functions will take a significant amount of time to complete.</p>
+			<div class="links">
+				<a href="#" onclick="doAjaxCall('forceFullUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists [Comprehensive]</a> 
+				<BR>
+				<a href="#" onclick="doAjaxCall('forceScan',$(this))" data-success="Library scan successful" data-error="Error forcing library scan"><span class="ui-icon ui-icon-refresh"></span>Force Re-scan Library [Comprehensive]</a> 
+				<BR>
+				<small>*Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched".</small>
+
+		</div>
+		</fieldset>
+		
+	</div>
+
+
+
 </div>
 </%def>
 <%def name="javascriptIncludes()">

data/interfaces/default/managealbums.html

@@ -57,7 +57,7 @@
 
 				myDB = db.DBConnection()					
 				totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']]))
-				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]))
+				havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']]))
 				
 				try:
 					percent = (havetracks*100.0)/totaltracks

data/interfaces/default/managemanual.html

@@ -0,0 +1,121 @@
+<%inherit file="base.html" />
+<%!
+	import headphones
+	from headphones import db, helpers
+	myDB = db.DBConnection()
+%>
+
+<%def name="headerIncludes()">
+	<div id="subhead_container">
+		<div id="subhead_menu">
+		</div>		
+	</div>	
+	<a href="manageUnmatched" class="back">&laquo; Back to Unmatched Albums</a>
+</%def>
+
+
+<%def name="body()">
+	<div class="table_wrapper">
+	<div id="manageheader" class="title">
+		<h1 class="clearfix"><img src="interfaces/default/images/icon_manage.png" alt="manage"/>Manage Manually Changed Albums</h1>
+	</div>
+
+	<table class="display" id="artist_table">
+		<thead>
+			<tr>
+				<th id="artist">Local Artist</th>
+				<th id="album">Local Album</th>
+				<th id="status">Previous Action</th>
+			</tr>
+		</thead>
+		<tbody>
+		<% count_albums=0 %>
+		%for album in manualalbums:
+			<tr class="gradeZ">
+			<%
+				old_artist_clean = album['ArtistName'].replace('&','%26').replace('+', '%2B').replace("'","%27")
+				old_album_clean = album['AlbumTitle'].replace('&','%26').replace('+', '%2B').replace("'","%27")
+			%>
+			<td id="artist">${album['ArtistName']}<BR>
+				<button id="restore_artist${count_albums}" onClick="restore_Artist(this.id)">(<-) Restore Artist</button>
+				<div id="restore_artist_dialog${count_albums}" title="Restore Artist" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to restore Local Artist: ${album['ArtistName']} to unmatched?</td></tr>
+					<tr><td align="right"><BR>
+					%if album['AlbumStatus'] == "Ignored":
+						<button href="#" onclick="doAjaxCall('markManual?action=unignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Restore Artist</button>
+					%elif album['AlbumStatus'] == "Matched":
+						<button href="#" onclick="doAjaxCall('markManual?action=unmatchArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully restored ${album['ArtistName']} to unmatched">Restore Artist</button>
+					%endif
+					</td></tr>
+					</table>
+				</div>
+			</td>
+			<td id="album">${album['AlbumTitle']}<BR>
+				<button id="restore_album${count_albums}" onClick="restore_Album(this.id)">(<-) Restore Album</button>
+				<div id="restore_album_dialog${count_albums}" title="Restore Album" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to restore Local Album: ${album['AlbumTitle']} to unmatched?</td></tr>
+					<tr><td align="right"><BR>
+					%if album['AlbumStatus'] == "Ignored":
+						<button href="#" onclick="doAjaxCall('markManual?action=unignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully restored ${album['AlbumTitle']} to unmatched">Restore Album</button>
+					%elif album['AlbumStatus'] == "Matched":
+						<button href="#" onclick="doAjaxCall('markManual?action=unmatchAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully restored ${album['AlbumTitle']} to unmatched">Restore Album</button>
+					%endif
+					</td></tr>
+					</table>
+				</div>
+
+			</td>
+			<td id="status">${album['AlbumStatus']}
+
+			</td>
+			</tr>
+		<% count_albums+=1 %>
+		%endfor
+		</tbody>
+	</table>
+	</div>
+</%def>
+
+<%def name="headIncludes()">
+	<link rel="stylesheet" href="interfaces/default/css/data_table.css">
+</%def>
+
+<%def name="javascriptIncludes()">
+	<script src="js/libs/jquery.dataTables.min.js"></script>
+	<script>
+	$(document).ready(function()
+	{
+		$('#artist_table').dataTable(
+			{
+				"bStateSave": true,
+				"bPaginate": true,
+				"oLanguage": {
+					"sSearch": "",
+	                "sLengthMenu":"Show _MENU_ albums per page",
+						"sInfo":"Showing _START_ to _END_ of _TOTAL_ albums",
+						"sInfoEmpty":"Showing 0 to 0 of 0 albums",
+						"sInfoFiltered":"(filtered from _MAX_ total albums)",
+						"sEmptyTable": " ",
+	                    },
+	            "sPaginationType": "full_numbers",
+	         });
+
+initActions();
+	});
+
+	function restore_Artist(clicked_id) {
+		n=clicked_id.replace("restore_artist","");
+		$("#restore_artist_dialog"+n).dialog();
+		return false;
+	}
+
+	function restore_Album(clicked_id) {
+		n=clicked_id.replace("restore_album","");
+		$("#restore_album_dialog"+n).dialog();
+		return false;
+	}
+
+	</script>
+</%def>

data/interfaces/default/managenew.html

@@ -20,7 +20,10 @@
 	</div>
 	<form action="addArtists" method="get">
 	<div id="markalbum">
-		Add selected artists
+	    <select name="action">
+        	<option value="add">(+) ADD Selected Artists</option>
+        	<option value="ignore">(-) IGNORE Selected Artists</option>
+       	</select>
 		<input type="submit" value="Go">
 	</div>
 	<table class="display" id="artist_table">

data/interfaces/default/manageunmatched.html

@@ -0,0 +1,259 @@
+<%inherit file="base.html" />
+<%!
+	import headphones
+	import json
+	from headphones import db, helpers
+	myDB = db.DBConnection()
+	artist_json = {}
+	counter = 0
+	artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE")
+	for artist in artist_list:
+		artist_json[counter] = artist['ArtistName'] 
+		counter+=1
+	json_artists = json.dumps(artist_json)
+%>
+
+<%def name="headerIncludes()">
+	<div id="subhead_container">
+		<div id="subhead_menu">
+		<a class="menu_link_edit" href="manageManual">Manage Manually Matched & Ignored</a>
+		</div>		
+	</div>	
+	<a href="manage" class="back">&laquo; Back to manage overview</a>
+</%def>
+
+
+<%def name="body()">
+	<div class="table_wrapper">
+	<div id="manageheader" class="title">
+		<h1 class="clearfix"><img src="interfaces/default/images/icon_manage.png" alt="manage"/>Manage Unmatched Albums</h1>
+	</div>
+
+	<table class="display" id="artist_table">
+		<thead>
+			<tr>
+				<th id="artist">Local Artist</th>
+				<th id="album">Local Album</th>
+			</tr>
+		</thead>
+		<tbody>
+		<% count_albums=0 %>
+		%for album in unmatchedalbums:
+			<tr class="gradeZ">
+				<% 
+					old_artist_clean = album['ArtistName'].replace('&','%26').replace("'","%27")
+					old_album_clean = album['AlbumTitle'].replace('&','%26').replace("'","%27")
+					old_artist_js = album['ArtistName'].replace("'","\\'").replace('"','\\"')
+					old_album_js = album['AlbumTitle'].replace("'","\\'").replace('"','\\"')
+				%>
+				<td id="artist">${album['ArtistName']}<BR>
+				<button id="ignore_artists${count_albums}" onClick="ignore_Artist(this.id)">(-) Ignore Artist</button>
+				<div id="ignore_artist_dialog${count_albums}" title="Ignore Artist" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to ignore Local Artist: ${album['ArtistName']} from future matching?</td></tr>
+					<tr><td align="right"><BR>
+					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreArtist&existing_artist=${old_artist_clean}', $(this), 'page');" data-success="Successfully ignored ${album['ArtistName']} from future matching">Ignore Artist</button>
+					</td></tr>
+					</table>
+				</div>
+
+				<button id="match_artists${count_albums}" onClick="load_Artist(this.id)">(->) Match Artist</button>
+				<div id="artist_dialog${count_albums}" title="Match Artist" style="display:none">
+					<table width=400>
+					<tr><td>Local Artist:</td><td>${album['ArtistName']}</td></tr>
+						<tr><td>Match Artist</td><td>
+						<select id="artist_options${count_albums}" name="new_artist">
+						</select>
+						</td></tr>
+					<tr><td></td><td align="right"><BR>
+					<button href="#" onclick="artist_matcher(${count_albums}, '${old_artist_js}')">Match Artist</button>
+					</td></tr>
+					</table>
+				</div>
+
+				</td>
+				<td id="album">${album['AlbumTitle']}<BR>
+				<button id="ignore_albums${count_albums}" onClick="ignore_Album(this.id)">(-) Ignore Album</button>
+				<div id="ignore_album_dialog${count_albums}" title="Ignore Album" style="display:none">
+					<table>
+					<tr><td>Are you sure you want to ignore Local Album: ${album['AlbumTitle']} from future matching?</td></tr>
+					<tr><td align="right"><BR>
+					<button href="#" onclick="doAjaxCall('markUnmatched?action=ignoreAlbum&existing_artist=${old_artist_clean}&existing_album=${old_album_clean}', $(this), 'page');" data-success="Successfully ignored ${album['AlbumTitle']} from future matching">Ignore Album</button>
+					</td></tr>
+					</table>
+				</div>
+
+				<button id="match_albums${count_albums}" onClick="load_AlbumArtist(this.id)">(->) Match Album</button>
+				<div id="album_dialog${count_albums}" title="Match Album" style="display:none">
+					<table width=650>
+					<tr><td>Local Artist:</td><td>${album['ArtistName']}</td></tr>
+					<tr><td>Local Album:</td><td>${album['AlbumTitle']}</td></tr>
+					<tr><td>Match Artist</td><td>
+					<select id="album_artist_options${count_albums}" name="new_artist">
+					</select>
+					</td></tr>
+					<tr><td>Match Album</td><td>
+					<select id="album_options${count_albums}" name="new_album">
+					</select>
+					</td></tr>
+					<tr><td></td><td align="right"><BR>
+					<button href="#" onclick="album_matcher(${count_albums}, '${old_artist_js}', '${old_album_js}')">Match Album</button>
+					</td></tr>
+					</table>
+				</div>
+
+				</td>
+			</tr>
+		<% count_albums+=1 %>
+		%endfor
+		</tbody>
+	</table>
+	</div>
+</%def>
+
+<%def name="headIncludes()">
+	<link rel="stylesheet" href="interfaces/default/css/data_table.css">
+</%def>
+
+<%def name="javascriptIncludes()">
+	<script src="js/libs/jquery.dataTables.min.js"></script>
+	<script>
+	$(document).ready(function()
+	{
+		$('#artist_table').dataTable(
+			{
+				"bStateSave": true,
+				"bPaginate": true,
+				"oLanguage": {
+					"sSearch": "",
+	                "sLengthMenu":"Show _MENU_ albums per page",
+						"sInfo":"Showing _START_ to _END_ of _TOTAL_ albums",
+						"sInfoEmpty":"Showing 0 to 0 of 0 albums",
+						"sInfoFiltered":"(filtered from _MAX_ total albums)",
+						"sEmptyTable": " ",
+	                    },
+	            "sPaginationType": "full_numbers",
+	         });
+
+initActions();
+	});
+
+	function ignore_Artist(clicked_id) {
+		n=clicked_id.replace("ignore_artists","");
+		$("#ignore_artist_dialog"+n).dialog();
+		return false;
+	}
+
+	function ignore_Album(clicked_id) {
+		n=clicked_id.replace("ignore_albums","");
+		$("#ignore_album_dialog"+n).dialog();
+		return false;
+	}
+
+
+	function load_Artist(clicked_id) {
+		n=clicked_id.replace("match_artists","");
+		var d = $("#artist_dialog"+n).dialog();
+    	d.dialog("option", "width", 450);
+    	d.dialog("option", "position", "center");
+		$('#artist_options'+n).html('');
+		$.each(${json_artists}, function(key, value) {
+    		$('#artist_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    		}));
+		});
+		change_artist(n)
+		return false;
+	}
+
+	function change_artist(n) {
+		selected_artist = $("#artist_options"+n).find("option:selected").text();
+		selected_artist_clean = selected_artist.replace('&', '%26').replace('+', '%2B');
+		$("#artist_options"+n).change(function() {
+			selected_artist = $("#artist_options"+n).find("option:selected").text();
+			selected_artist_clean = selected_artist.replace('&', '%26').replace('+', '%2B');
+			});
+	}
+
+	function artist_matcher(n, existing_artist) {
+		var existing_artist = existing_artist.toString();
+		var existing_artist_clean = existing_artist.replace('&', '%26').replace('+', '%2B');
+		$('#match_artists'+n).attr('data-success', 'Successfully matched '+existing_artist+' with '+selected_artist);
+		doAjaxCall('markUnmatched?action=matchArtist&existing_artist='+existing_artist_clean+'&new_artist='+selected_artist_clean, $('#match_artists'+n), 'page'); 
+		}
+
+
+	function load_AlbumArtist(clicked_id) {
+		n=clicked_id.replace("match_albums","");
+		var d = $("#album_dialog"+n).dialog();
+    	d.dialog("option", "width", 700);
+    	d.dialog("option", "position", "center");
+    	$('#album_artist_options'+n).html('');
+		$.each(${json_artists}, function(key, value) {
+    		$('#album_artist_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    		}));
+		});
+		load_json_albums(n)
+		return false;
+	}
+
+
+	function load_json_albums(n) {
+		selected_album_artist = $("#album_artist_options"+n).find("option:selected").text();
+		selected_album_artist_clean = selected_album_artist.replace('&', '%26').replace('+', '%2B');
+		$('#album_options'+n).html('')
+		$.getJSON("getAlbumsByArtist_json?artist="+selected_album_artist_clean, function( data ) {
+			$.each( data, function( key, value ) {
+			    $('#album_options'+n).append($("<option/>", {
+        		value: value,
+        		text: value
+    			}));
+			});
+		selected_album= $("#album_options"+n).find("option:selected").text();
+		selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+		});
+		change_json_albums(n)
+		change_album(n)
+	}
+
+
+	function change_json_albums(n) {
+		$("#album_artist_options"+n).change(function(){
+			selected_album_artist = $("#album_artist_options"+n).find("option:selected").text();
+			selected_album_artist_clean = selected_album_artist.replace('&', '%26').replace('+', '%2B');
+			$('#album_options'+n).html('')
+			$.getJSON("getAlbumsByArtist_json?artist="+selected_album_artist_clean, function( data ) {
+				$.each( data, function( key, value ) {
+				    $('#album_options'+n).append($("<option/>", {
+	        		value: value,
+	        		text: value
+	    			}));
+				});
+				selected_album= $("#album_options"+n).find("option:selected").text();
+				selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+			});
+		change_album(n)
+	    });
+	}
+
+	function change_album(n) {
+		$("#album_options"+n).change(function() {
+			selected_album= $("#album_options"+n).find("option:selected").text();
+			selected_album_clean = selected_album.replace('&', '%26').replace('+', '%2B');
+			});
+	}
+
+	function album_matcher(n, existing_artist, existing_album) {
+		var existing_artist = existing_artist.toString();
+		var existing_artist_clean = existing_artist.replace('&', '%26').replace('+', '%2B');
+		var existing_album = existing_album.toString();
+		var existing_album_clean = existing_album.replace('&', '%26').replace('+', '%2B');
+		$('#match_albums'+n).attr('data-success', 'Successfully matched '+existing_album+' with '+selected_album);
+		doAjaxCall('markUnmatched?action=matchAlbum&existing_artist='+existing_artist_clean+'&new_artist='+selected_album_artist_clean+'&existing_album='+existing_album_clean+'&new_album='+selected_album_clean, $('#match_albums'+n), 'page'); 
+	}
+
+	</script>
+</%def>

headphones/__init__.py

@@ -130,6 +130,8 @@ SEARCH_INTERVAL = 360
 LIBRARYSCAN = False
 LIBRARYSCAN_INTERVAL = 300
 DOWNLOAD_SCAN_INTERVAL = 5
+UPDATE_DB_INTERVAL = 24
+MB_IGNORE_AGE = 365
 
 SAB_HOST = None
 SAB_USERNAME = None
@@ -166,6 +168,10 @@ NZBSRUS = False
 NZBSRUS_UID = None
 NZBSRUS_APIKEY = None
 
+OMGWTFNZBS = False
+OMGWTFNZBS_UID = None
+OMGWTFNZBS_APIKEY = None
+
 PREFERRED_WORDS = None
 IGNORED_WORDS = None
 REQUIRED_WORDS = None
@@ -302,11 +308,12 @@ def initialize():
                 ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \
                 TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, PIRATEBAY_PROXY_URL, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \
                 RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \
-                LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
+                LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
                 NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \
                 UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \
-                NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \
+                NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \
-                LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \
+		NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \
+                INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \
                 MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, \
                 PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \
                 MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \
@@ -326,6 +333,7 @@ def initialize():
         CheckSection('Newznab')
         CheckSection('NZBsorg')
         CheckSection('NZBsRus')
+        CheckSection('omgwtfnzbs')
         CheckSection('Waffles')
         CheckSection('Rutracker')
         CheckSection('What.cd')
@@ -405,6 +413,8 @@ def initialize():
         LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1))
         LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300)
         DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5)
+        UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24)
+        MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365)
 
         TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '')
         NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10')
@@ -465,6 +475,10 @@ def initialize():
         NZBSRUS_UID = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_uid', '')
         NZBSRUS_APIKEY = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_apikey', '')
 
+        OMGWTFNZBS = bool(check_setting_int(CFG, 'omgwtfnzbs', 'omgwtfnzbs', 0))
+        OMGWTFNZBS_UID = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_uid', '')
+        OMGWTFNZBS_APIKEY = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_apikey', '')
+
         PREFERRED_WORDS = check_setting_str(CFG, 'General', 'preferred_words', '')
         IGNORED_WORDS = check_setting_str(CFG, 'General', 'ignored_words', '')
         REQUIRED_WORDS = check_setting_str(CFG, 'General', 'required_words', '')
@@ -792,6 +806,8 @@ def config_write():
     new_config['General']['libraryscan'] = int(LIBRARYSCAN)
     new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL
     new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL
+    new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL
+    new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE
 
     new_config['SABnzbd'] = {}
     new_config['SABnzbd']['sab_host'] = SAB_HOST
@@ -842,6 +858,11 @@ def config_write():
     new_config['NZBsRus']['nzbsrus_uid'] = NZBSRUS_UID
     new_config['NZBsRus']['nzbsrus_apikey'] = NZBSRUS_APIKEY
 
+    new_config['omgwtfnzbs'] = {}
+    new_config['omgwtfnzbs']['omgwtfnzbs'] = int(OMGWTFNZBS)
+    new_config['omgwtfnzbs']['omgwtfnzbs_uid'] = OMGWTFNZBS_UID
+    new_config['omgwtfnzbs']['omgwtfnzbs_apikey'] = OMGWTFNZBS_APIKEY
+
     new_config['General']['preferred_words'] = PREFERRED_WORDS
     new_config['General']['ignored_words'] = IGNORED_WORDS
     new_config['General']['required_words'] = REQUIRED_WORDS
@@ -917,9 +938,9 @@ def start():
         # Start our scheduled background tasks
         from headphones import updater, searcher, librarysync, postprocessor
 
-        SCHED.add_interval_job(updater.dbUpdate, hours=24)
+        SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL)
         SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL)
-        SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True})
+        SCHED.add_interval_job(librarysync.libraryScan, hours=LIBRARYSCAN_INTERVAL, kwargs={'cron':True})
 
         if CHECK_GITHUB:
             SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL)
@@ -953,6 +974,19 @@ def dbcheck():
     c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))')
     c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)')
     c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)')
+    #Below creates indices to speed up Active Artist updating
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)')
+    #Below creates indices to speed up library scanning & matching
+    c.execute('CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)')
+    c.execute('CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)')
 
     try:
         c.execute('SELECT IncludeExtras from artists')
@@ -1118,6 +1152,7 @@ def dbcheck():
     except sqlite3.OperationalError:
         c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL')
 
+
     conn.commit()
     c.close()
 

headphones/api.py

@@ -123,8 +123,9 @@ class Api(object):
     
         artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"')
         albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC')
+        description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"')
         
-        self.data = { 'artist': artist, 'albums': albums }
+        self.data = { 'artist': artist, 'albums': albums, 'description' : description }
         return
     
     def _getAlbum(self, **kwargs):

headphones/importer.py

@@ -15,6 +15,7 @@
 
 from lib.pyItunes import *
 import time
+import threading
 import os
 from lib.beets.mediafile import MediaFile
 
@@ -108,7 +109,7 @@ def addArtistIDListToDB(artistidlist):
     for artistid in artistidlist:
         addArtisttoDB(artistid)
 
-def addArtisttoDB(artistid, extrasonly=False):
+def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
     
     # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums
     from headphones import cache
@@ -189,65 +190,160 @@ def addArtisttoDB(artistid, extrasonly=False):
         db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
         includeExtras = db_artist['IncludeExtras']
     except IndexError:
-        includeExtras = False  
+        includeExtras = False
+
+    #Clean all references to release group in dB that are no longer referenced in musicbrainz
+    group_list = []
+    force_repackage = 0
+    #Don't nuke the database if there's a MusicBrainz error
+    if len(artist['releasegroups']) != 0 and not extrasonly:
+        for groups in artist['releasegroups']:
+            group_list.append(groups['id'])
+        remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
+        for items in remove_missing_groups_from_albums:
+            if items['AlbumID'] not in group_list:
+                # Remove all from albums/tracks that aren't in release groups
+                myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
+                myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
+                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID']))
+                force_repackage = 1
+    else:
+        logger.info("[%s] Error pulling data from MusicBrainz:  Maintaining dB" % artist['artist_name'])
+    
+    # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
+
 
     for rg in artist['releasegroups']:
         
-        logger.info("Now adding/updating: " + rg['title'])
+        al_title = rg['title']
-        
+        today = helpers.today()
         rgid = rg['id']
-        
+        skip_log = 0
-        # check if the album already exists
+        #Make a user configurable variable to skip update of albums with release dates older than this date (in days)
-        rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
+        pause_delta = headphones.MB_IGNORE_AGE
-                    
-        releases = mb.get_all_releases(rgid,includeExtras)
-        if releases == []:
-            logger.info('No official releases in release group %s' % rg['title'])
-            continue
-        if not releases:
-            errors = True
-            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
-            continue
-        
-        # This will be used later to build a hybrid release     
-        fullreleaselist = []
 
-        for release in releases:
+        if not forcefull:
-        # What we're doing here now is first updating the allalbums & alltracks table to the most
+            check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone()
-        # current info, then moving the appropriate release into the album table and its associated
+            if check_release_date:
-        # tracks into the tracks table
+                if check_release_date[0] is None:
-            controlValueDict = {"ReleaseID" : release['ReleaseID']}
+                    logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title']))
+                    new_releases = mb.get_new_releases(rgid,includeExtras,True)          
+                else:
+                    if len(check_release_date[0]) == 10:
+                        release_date = check_release_date[0]
+                    elif len(check_release_date[0]) == 7:
+                        release_date = check_release_date[0]+"-31"
+                    elif len(check_release_date[0]) == 4:
+                        release_date = check_release_date[0]+"-12-31"
+                    else:
+                        release_date = today
+                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                        logger.info("[%s] Now updating: %s (Release Date <%s Days) " % (artist['artist_name'], rg['title'], pause_delta))
+                        new_releases = mb.get_new_releases(rgid,includeExtras,True)
+                    else:
+                        logger.info("[%s] Skipping: %s (Release Date >%s Days)" % (artist['artist_name'], rg['title'], pause_delta))
+                        skip_log = 1
+                        new_releases = 0
+            else:
+                logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title']))
+                new_releases = mb.get_new_releases(rgid,includeExtras)
 
-            newValueDict = {"ArtistID":         release['ArtistID'],
+            if force_repackage == 1:
-                            "ArtistName":       release['ArtistName'],
+                new_releases = -1
-                            "AlbumTitle":       release['AlbumTitle'],
+                logger.info('[%s] Forcing repackage of %s (Release Group Removed)' % (artist['artist_name'], al_title))
-                            "AlbumID":          release['AlbumID'],
+            else:
-                            "AlbumASIN":        release['AlbumASIN'],
+                new_releases = new_releases
-                            "ReleaseDate":      release['ReleaseDate'],
+        else:
-                            "Type":             release['Type'],
+            logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist['artist_name'], rg['title']))
-                            "ReleaseCountry":   release['ReleaseCountry'],
+            new_releases = mb.get_new_releases(rgid,includeExtras,forcefull)
-                            "ReleaseFormat":    release['ReleaseFormat']
+        
+        #What this does is adds new releases per artist to the allalbums + alltracks databases
+        #new_releases = mb.get_new_releases(rgid,includeExtras)
+        #print al_title
+        #print new_releases
+
+        if new_releases != 0:
+            #Dump existing hybrid release since we're repackaging/replacing it
+            myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']])
+            myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']])
+            # This will be used later to build a hybrid release     
+            fullreleaselist = []
+            #Search for releases within a release group
+            find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
+            # Build the dictionary for the fullreleaselist
+            for items in find_hybrid_releases:
+                if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing
+                    hybrid_release_id = items['ReleaseID']
+                    newValueDict = {"ArtistID":         items['ArtistID'],
+                        "ArtistName":       items['ArtistName'],
+                        "AlbumTitle":       items['AlbumTitle'],
+                        "AlbumID":          items['AlbumID'],
+                        "AlbumASIN":        items['AlbumASIN'],
+                        "ReleaseDate":      items['ReleaseDate'],
+                        "Type":             items['Type'],
+                        "ReleaseCountry":   items['ReleaseCountry'],
+                        "ReleaseFormat":    items['ReleaseFormat']
+                    }
+                    find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
+                    totalTracks = 1
+                    hybrid_track_array = []
+                    for hybrid_tracks in find_hybrid_tracks:
+                        hybrid_track_array.append({
+                            'number':        hybrid_tracks['TrackNumber'],
+                            'title':         hybrid_tracks['TrackTitle'],
+                            'id':            hybrid_tracks['TrackID'],
+                            #'url':           hybrid_tracks['TrackURL'],
+                            'duration':      hybrid_tracks['TrackDuration']
+                            })
+                        totalTracks += 1  
+                    newValueDict['ReleaseID'] = hybrid_release_id
+                    newValueDict['Tracks'] = hybrid_track_array
+                    fullreleaselist.append(newValueDict)
+
+                #print fullreleaselist
+
+            # Basically just do the same thing again for the hybrid release
+            # This may end up being called with an empty fullreleaselist
+            try:
+                hybridrelease = getHybridRelease(fullreleaselist)
+                logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title']))
+            except Exception, e:
+                errors = True
+                logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e))
+                continue
+            
+            # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
+            # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
+            # The hybrid won't have a country or a format
+            controlValueDict = {"ReleaseID":  rg['id']}
+
+            newValueDict = {"ArtistID":         artistid,
+                            "ArtistName":       artist['artist_name'],
+                            "AlbumTitle":       rg['title'],
+                            "AlbumID":          rg['id'],
+                            "AlbumASIN":        hybridrelease['AlbumASIN'],
+                            "ReleaseDate":      hybridrelease['ReleaseDate'],
+                            "Type":             rg['type']
                         }
-
+                        
             myDB.upsert("allalbums", newValueDict, controlValueDict)
             
-            # Build the dictionary for the fullreleaselist
+            for track in hybridrelease['Tracks']:
-            newValueDict['ReleaseID'] = release['ReleaseID']
-            newValueDict['Tracks'] = release['Tracks']
-            fullreleaselist.append(newValueDict)
-            
-            for track in release['Tracks']:
 
                 cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
         
                 controlValueDict = {"TrackID":      track['id'],
-                                    "ReleaseID":    release['ReleaseID']}
+                                    "ReleaseID":    rg['id']}
 
-                newValueDict = {"ArtistID":         release['ArtistID'],
+                newValueDict = {"ArtistID":         artistid,
-                                "ArtistName":       release['ArtistName'],
+                                "ArtistName":       artist['artist_name'],
-                                "AlbumTitle":       release['AlbumTitle'],
+                                "AlbumTitle":       rg['title'],
-                                "AlbumID":          release['AlbumID'],
+                                "AlbumASIN":        hybridrelease['AlbumASIN'],
-                                "AlbumASIN":        release['AlbumASIN'],
+                                "AlbumID":          rg['id'],
                                 "TrackTitle":       track['title'],
                                 "TrackDuration":    track['duration'],
                                 "TrackNumber":      track['number'],
@@ -258,184 +354,117 @@ def addArtisttoDB(artistid, extrasonly=False):
             
                 if not match:
                     match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
-                if not match:
+                #if not match:
-                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
+                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
                 if match:
                     newValueDict['Location'] = match['Location']
                     newValueDict['BitRate'] = match['BitRate']
                     newValueDict['Format'] = match['Format']
-                    myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location']))
                                 
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
-
-        # Basically just do the same thing again for the hybrid release
-        # This may end up being called with an empty fullreleaselist
-        try:
-            hybridrelease = getHybridRelease(fullreleaselist)
-        except Exception, e:
-            errors = True
-            logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
-            continue
-        
-        # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
-        # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
-        # The hybrid won't have a country or a format
-        controlValueDict = {"ReleaseID":  rg['id']}
-
-        newValueDict = {"ArtistID":         artistid,
-                        "ArtistName":       artist['artist_name'],
-                        "AlbumTitle":       rg['title'],
-                        "AlbumID":          rg['id'],
-                        "AlbumASIN":        hybridrelease['AlbumASIN'],
-                        "ReleaseDate":      hybridrelease['ReleaseDate'],
-                        "Type":             rg['type']
-                    }
-                    
-        myDB.upsert("allalbums", newValueDict, controlValueDict)
-        
-        for track in hybridrelease['Tracks']:
-
-            cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
-    
-            controlValueDict = {"TrackID":      track['id'],
-                                "ReleaseID":    rg['id']}
-
-            newValueDict = {"ArtistID":         artistid,
-                            "ArtistName":       artist['artist_name'],
-                            "AlbumTitle":       rg['title'],
-                            "AlbumASIN":        hybridrelease['AlbumASIN'],
-                            "AlbumID":          rg['id'],
-                            "TrackTitle":       track['title'],
-                            "TrackDuration":    track['duration'],
-                            "TrackNumber":      track['number'],
-                            "CleanName":        cleanname
-                        }
-                        
-            match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
-        
-            if not match:
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
-            if not match:
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
-            if match:
-                newValueDict['Location'] = match['Location']
-                newValueDict['BitRate'] = match['BitRate']
-                newValueDict['Format'] = match['Format']
-                myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
-                            
-            myDB.upsert("alltracks", newValueDict, controlValueDict)
-        
-        # Delete matched tracks from the have table
-        myDB.action('DELETE from have WHERE Matched="True"')
-        
-        # If there's no release in the main albums tables, add the default (hybrid)
-        # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
-        if not rg_exists:
-            releaseid = rg['id']
-        elif rg_exists and not rg_exists['ReleaseID']:
-            # Need to do some importing here - to transition the old format of using the release group
-            # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID
-            # so we'll need to move over the locations, bitrates & formats from the tracks table to the new
-            # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups
-            logger.info("Copying current track information to alternate releases")
-            tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall()
-            for track in tracks:
-                if track['Location']:
-                    controlValueDict = {"TrackID":  track['TrackID']}
-                    newValueDict = {"Location":     track['Location'],
-                                    "BitRate":      track['BitRate'],
-                                    "Format":       track['Format'],
-                                    }
-                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-            releaseid = rg['id']
-        else:
-            releaseid = rg_exists['ReleaseID']
-        
-        album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()
-
-        controlValueDict = {"AlbumID":  rg['id']}
-
-        newValueDict = {"ArtistID":         album['ArtistID'],
-                        "ArtistName":       album['ArtistName'],
-                        "AlbumTitle":       album['AlbumTitle'],
-                        "ReleaseID":        album['ReleaseID'],
-                        "AlbumASIN":        album['AlbumASIN'],
-                        "ReleaseDate":      album['ReleaseDate'],
-                        "Type":             album['Type'],
-                        "ReleaseCountry":   album['ReleaseCountry'],
-                        "ReleaseFormat":    album['ReleaseFormat']
-                    }
             
-        if not rg_exists:
+            # Delete matched tracks from the have table
+            #myDB.action('DELETE from have WHERE Matched="True"')
             
-            today = helpers.today()
+            # If there's no release in the main albums tables, add the default (hybrid)
-            
+            # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
-            newValueDict['DateAdded']= today
+            # check if the album already exists
-                            
+            rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
-            if headphones.AUTOWANT_ALL:
+            if not rg_exists:
-                newValueDict['Status'] = "Wanted"
+                releaseid = rg['id']
-            elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
-                newValueDict['Status'] = "Wanted"
-            # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
-            # The first test just makes sure we have year-month-day
-            elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
-                newValueDict['Status'] = "Wanted"
             else:
-                newValueDict['Status'] = "Skipped"
+                releaseid = rg_exists['ReleaseID']
-        
+            
-        myDB.upsert("albums", newValueDict, controlValueDict)
+            album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()
 
-        myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']])
+            controlValueDict = {"AlbumID":  rg['id']}
-        tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
 
-        # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
+            newValueDict = {"ArtistID":         album['ArtistID'],
-        total_track_count = len(tracks)
+                            "ArtistName":       album['ArtistName'],
-        
+                            "AlbumTitle":       album['AlbumTitle'],
-        for track in tracks:
+                            "ReleaseID":        album['ReleaseID'],
-        
+                            "AlbumASIN":        album['AlbumASIN'],
-            controlValueDict = {"TrackID":  track['TrackID'],
+                            "ReleaseDate":      album['ReleaseDate'],
-                                "AlbumID":  rg['id']}
+                            "Type":             album['Type'],
-
+                            "ReleaseCountry":   album['ReleaseCountry'],
-            newValueDict = {"ArtistID":     track['ArtistID'],
+                            "ReleaseFormat":    album['ReleaseFormat']
-                        "ArtistName":       track['ArtistName'],
-                        "AlbumTitle":       track['AlbumTitle'],
-                        "AlbumASIN":        track['AlbumASIN'],
-                        "ReleaseID":        track['ReleaseID'],
-                        "TrackTitle":       track['TrackTitle'],
-                        "TrackDuration":    track['TrackDuration'],
-                        "TrackNumber":      track['TrackNumber'],
-                        "CleanName":        track['CleanName'],
-                        "Location":         track['Location'],
-                        "Format":           track['Format'],
-                        "BitRate":          track['BitRate']
                         }
-                        
+                
-            myDB.upsert("tracks", newValueDict, controlValueDict)
+            if not rg_exists:
+                
+                today = helpers.today()
+                
+                newValueDict['DateAdded']= today
+                                
+                if headphones.AUTOWANT_ALL:
+                    newValueDict['Status'] = "Wanted"
+                elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
+                    newValueDict['Status'] = "Wanted"
+                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
+                # The first test just makes sure we have year-month-day
+                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
+                    newValueDict['Status'] = "Wanted"
+                else:
+                    newValueDict['Status'] = "Skipped"
+            
+            myDB.upsert("albums", newValueDict, controlValueDict) 
 
-        # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
+            tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
-        have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
+
-        marked_as_downloaded = False
+            # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
-        
+            total_track_count = len(tracks)
-        if rg_exists:
+            
-            if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
+            for track in tracks:
-                myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+            
-                marked_as_downloaded = True
+                controlValueDict = {"TrackID":  track['TrackID'],
+                                    "AlbumID":  rg['id']}
+
+                newValueDict = {"ArtistID":     track['ArtistID'],
+                            "ArtistName":       track['ArtistName'],
+                            "AlbumTitle":       track['AlbumTitle'],
+                            "AlbumASIN":        track['AlbumASIN'],
+                            "ReleaseID":        track['ReleaseID'],
+                            "TrackTitle":       track['TrackTitle'],
+                            "TrackDuration":    track['TrackDuration'],
+                            "TrackNumber":      track['TrackNumber'],
+                            "CleanName":        track['CleanName'],
+                            "Location":         track['Location'],
+                            "Format":           track['Format'],
+                            "BitRate":          track['BitRate']
+                            }
+                            
+                myDB.upsert("tracks", newValueDict, controlValueDict) 
+
+            # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
+            have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
+            marked_as_downloaded = False
+            
+            if rg_exists:
+                if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
+                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+                    marked_as_downloaded = True
+            else:
+                if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
+                    myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+                    marked_as_downloaded = True
+
+            logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
+            cache.getThumb(AlbumID=rg['id'])
+            
+            #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
+            if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:    
+                from headphones import searcher
+                searcher.searchforalbum(albumid=rg['id'])
         else:
-            if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
+            if skip_log == 0:
-                myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
+                logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
-                marked_as_downloaded = True
-
-        logger.info(u"Seeing if we need album art for " + rg['title'])
-        cache.getThumb(AlbumID=rg['id'])
-        
-        #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
-        if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:    
-            from headphones import searcher
-            searcher.searchforalbum(albumid=rg['id'])
 
     latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone()
     totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid]))
-    havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
+    #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']]))
+    havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist['artist_name']]))
 
     controlValueDict = {"ArtistID":     artistid}
     
@@ -456,13 +485,15 @@ def addArtisttoDB(artistid, extrasonly=False):
     
     myDB.upsert("artists", newValueDict, controlValueDict)
     
-    logger.info(u"Seeing if we need album art for: " + artist['artist_name'])
+    logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
     cache.getThumb(ArtistID=artistid)
     
     if errors:
-        logger.info("Finished updating artist: " + artist['artist_name'] + " but with errors, so not marking it as updated in the database")
+        logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name']))
     else:
-        logger.info(u"Updating complete for: " + artist['artist_name'])
+        myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']])
+        logger.info(u"Updating complete for: %s" % artist['artist_name'])
+
     
 def addReleaseById(rid):
     
@@ -562,14 +593,14 @@ def addReleaseById(rid):
             if not match:
                 match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone()
             
-            if not match:
+            #if not match:
-                match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
+                #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
                     
             if match:
                 newValueDict['Location'] = match['Location']
                 newValueDict['BitRate'] = match['BitRate']
                 newValueDict['Format'] = match['Format']
-                myDB.action('DELETE from have WHERE Location=?', [match['Location']])
+                #myDB.action('DELETE from have WHERE Location=?', [match['Location']])
         
             myDB.upsert("tracks", newValueDict, controlValueDict)
                 

headphones/librarysync.py

@@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer
 # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName
 def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False):
 
+
     if cron and not headphones.LIBRARYSCAN:
         return
         
@@ -43,23 +44,39 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
         return
 
     myDB = db.DBConnection()
+    new_artists = []
+
+    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
     
     if not append:
         # Clean up bad filepaths
-        tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL')
+        tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL')
-    
+
         for track in tracks:
-            if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)):
+            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
-                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']])
+            if not os.path.isfile(encoded_track_string):
+                myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
+                myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']])
+    
+        del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
 
-        myDB.action('DELETE from have')
+        for track in del_have_tracks:
+            encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
+            if not os.path.isfile(encoded_track_string):
+                if track['ArtistName']:
+                    #Make sure deleted files get accounted for when updating artist track counts
+                    new_artists.append(track['ArtistName'])
+                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
+                logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace'))
+           ###############myDB.action('DELETE from have')
 
-    logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
-
-    new_artists = []
     bitrates = []
     
     song_list = []
+    new_song_count = 0
+    file_count = 0
+
+    latest_subdirectory = []
     
     for r,d,f in os.walk(dir):
         #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one
@@ -68,9 +85,17 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
             if directory.startswith("."):
                 d.remove(directory)
         for files in f:
+
             # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
             if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
 
+                subdirectory = r.replace(dir,'')
+                latest_subdirectory.append(subdirectory)
+                if file_count == 0 and r.replace(dir,'') !='':
+                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+                elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0:
+                    logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace')))
+
                 song = os.path.join(r, files)
 
                 # We need the unicode path to use for logging, inserting into database
@@ -99,8 +124,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                 # Add the song to our song list - 
                 # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
 
-                song_dict = { 'TrackID' : f.mb_trackid,
+                if f_artist and f.album and f.title:
-                              'ReleaseID' : f.mb_albumid,
+                    CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title)
+                else:
+                    CleanName = None
+
+                controlValueDict = {'Location'   : unicode_song_path}
+
+                newValueDict = { 'TrackID' : f.mb_trackid,
+                              #'ReleaseID' : f.mb_albumid,
                               'ArtistName' : f_artist,
                               'AlbumTitle' : f.album,
                               'TrackNumber': f.track,
@@ -110,224 +142,169 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                               'TrackTitle' : f.title,
                               'BitRate'    : f.bitrate,
                               'Format'     : f.format,
-                              'Location'   : unicode_song_path }
+                              'CleanName'  : CleanName
+                              }
                               
-                song_list.append(song_dict)
+                #song_list.append(song_dict)
+                check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone()
+                #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed.
+                if not check_exist_song:
+                    #This is a new track
+                    if f_artist:
+                        new_artists.append(f_artist)
+                    myDB.upsert("have", newValueDict, controlValueDict)
+                    new_song_count+=1
+                else:
+                    if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title:
+                        #Important track metadata has been modified, need to run matcher again
+                        if f_artist and f_artist != check_exist_song['ArtistName']:
+                            new_artists.append(f_artist)
+                        elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored":
+                            new_artists.append(f_artist)
+                        else:
+                            continue
+
+                        newValueDict['Matched'] = None
+                        myDB.upsert("have", newValueDict, controlValueDict)
+                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
+                        myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path])
+                        new_song_count+=1
+                    else:
+                        #This track information hasn't changed
+                        if f_artist and check_exist_song['Matched'] != "Ignored":
+                            new_artists.append(f_artist)
+
+                file_count+=1
+
 
     # Now we start track matching
-    total_number_of_songs = len(song_list)
+    logger.info("%s new/modified songs found and added to the database" % new_song_count)
-    logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
+    song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"])
+    total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0]
+    logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....")
     
     # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
     # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
     
-    song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
+    ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
+    song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle'])
     
     # We'll use this to give a % completion, just because the track matching might take a while
     song_count = 0
+    latest_artist = []
     
     for song in song_list:
+
+        latest_artist.append(song['ArtistName'])
+        if song_count == 0:
+            logger.info("Now matching songs by %s" % song['ArtistName'])
+        elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0:
+            logger.info("Now matching songs by %s" % song['ArtistName'])
         
+        #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle']
         song_count += 1
         completion_percentage = float(song_count)/total_number_of_songs * 100
         
         if completion_percentage%10 == 0:
             logger.info("Track matching is " + str(completion_percentage) + "% complete")
         
-        # If the track has a trackid & releaseid (beets: albumid) that the most surefire way
+        #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED.  WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT
-        # of identifying a track to a specific release so we'll use that first
+        #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME.  ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION,
-        if song['TrackID'] and song['ReleaseID']:
+        #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT.  FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
+        #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
 
-            # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet
-            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
-            
-            # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table
-            if not track:
-                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
-    
-            if track:
-                # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'ReleaseID' : track['ReleaseID'] }
-                
-                # Insert it into the Headphones hybrid release (ReleaseID == AlbumID)                   
-                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
-                                           'ReleaseID' : track['AlbumID'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-                                 
-                # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-                
-                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
-                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
-                
-                # Matched. Move on to the next one:
-                continue
-    
-        # If we can't find it with TrackID & ReleaseID, next most specific will be 
-        # releaseid + tracktitle, although perhaps less reliable due to a higher 
-        # likelihood of variations in the song title (e.g. feat. artists)
-        if song['ReleaseID'] and song['TrackTitle']:
-    
-            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
-                
-            if track:
-                # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table)
-                # So use both TrackID & ReleaseID as the control values
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'ReleaseID' : track['ReleaseID'] }
-                                     
-                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
-                                           'ReleaseID' : track['AlbumID'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-                                 
-                # Update both tables here as well
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-                
-                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
-                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
-                
-                # Done
-                continue
-                
-        # Next most specific will be the opposite: a TrackID and an AlbumTitle
-        # TrackIDs span multiple releases so if something is on an official album
-        # and a compilation, for example, this will match it to the right one
-        # However - there may be multiple matches here
-        if song['TrackID'] and song['AlbumTitle']:
-    
-            # Even though there might be multiple matches, we just need to grab one to confirm a match
-            track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
-                
-            if track:
-                # Don't need the hybridControlValueDict here since ReleaseID is not unique
-                controlValueDict = { 'TrackID'   : track['TrackID'],
-                                     'AlbumTitle' : track['AlbumTitle'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
-
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
-
-                continue   
-        
-        # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably 
-        # even more unreliable than the previous queries, and might span multiple releases)
         if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
             
-            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-    
-            if not track:
-                track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
-                
             if track:
                 controlValueDict = { 'ArtistName' : track['ArtistName'],
                                      'AlbumTitle' : track['AlbumTitle'],
-                                     'TrackTitle' : track['TrackTitle'] }
+                                     'TrackTitle' : track['TrackTitle'] }                           
-                                     
                 newValueDict = { 'Location' : song['Location'],
                                  'BitRate'  : song['BitRate'],
                                  'Format'   : song['Format'] }
-
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
                 myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                continue
+                controlValueDict2 = { 'Location' : song['Location']}
-        
+                newValueDict2 = { 'Matched' : track['AlbumID']}
-        # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc)
+                myDB.upsert("have", newValueDict2, controlValueDict2)
-        # This is more reliable than the former but requires some string manipulation so we'll do it only
+            else:
-        # if we can't find a match with the original data
+                track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+                if track:
-            
+                    controlValueDict = { 'CleanName' : track['CleanName']}                         
-            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
+                    newValueDict = { 'Location' : song['Location'],
-            
+                                     'BitRate'  : song['BitRate'],
-            track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
+                                     'Format'   : song['Format'] }               
-            
+                    myDB.upsert("tracks", newValueDict, controlValueDict)
-            if not track:
+
-                track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
+                    controlValueDict2 = { 'Location' : song['Location']}
-    
+                    newValueDict2 = { 'Matched' : track['AlbumID']}
-            if track:
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                controlValueDict = { 'CleanName' : track['CleanName'] }
+                else:
-                                     
+                    controlValueDict2 = { 'Location' : song['Location']}
+                    newValueDict2 = { 'Matched' : "Failed"}
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
+  
+
+            alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()  
+            if alltrack:
+                controlValueDict = { 'ArtistName' : alltrack['ArtistName'],
+                                     'AlbumTitle' : alltrack['AlbumTitle'],
+                                     'TrackTitle' : alltrack['TrackTitle'] }
                 newValueDict = { 'Location' : song['Location'],
                                  'BitRate'  : song['BitRate'],
                                  'Format'   : song['Format'] }
-
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
-                myDB.upsert("tracks", newValueDict, controlValueDict)
 
-                continue     
+                controlValueDict2 = { 'Location' : song['Location']}
-        
+                newValueDict2 = { 'Matched' : alltrack['AlbumID']}
-        # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable
+                myDB.upsert("have", newValueDict2, controlValueDict2)
-        # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track
+            else:
-        # with more specific information exists in the library, it'll overwrite these values
+                alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
-        if song['TrackID']:
+                if alltrack:
-    
+                    controlValueDict = { 'CleanName' : alltrack['CleanName']}                                         
-            track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone()
+                    newValueDict = { 'Location' : song['Location'],
-            
+                                     'BitRate'  : song['BitRate'],
-            if not track:
+                                     'Format'   : song['Format'] }               
-                track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone()
+                    myDB.upsert("alltracks", newValueDict, controlValueDict)
-    
-            if track:
-                controlValueDict = { 'TrackID' : track['TrackID'] }
-                                     
-                newValueDict = { 'Location' : song['Location'],
-                                 'BitRate'  : song['BitRate'],
-                                 'Format'   : song['Format'] }
 
-                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                    controlValueDict2 = { 'Location' : song['Location']}
-                myDB.upsert("tracks", newValueDict, controlValueDict)
+                    newValueDict2 = { 'Matched' : alltrack['AlbumID']}
-
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
-                continue          
+                else:
-        
+                    controlValueDict2 = { 'Location' : song['Location']}
-        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
+                    newValueDict2 = { 'Matched' : "Failed"}
-        if song['ArtistName']:
+                    myDB.upsert("have", newValueDict2, controlValueDict2)
-            new_artists.append(song['ArtistName'])
         else:
-            continue
+            controlValueDict2 = { 'Location' : song['Location']}
+            newValueDict2 = { 'Matched' : "Failed"}
+            myDB.upsert("have", newValueDict2, controlValueDict2)
         
-        # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
+        #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
-        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
-            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
-        else:
-            continue
-        
-        myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
 
     logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
-    
+
     
     if not append:
+        logger.info('Updating scanned artist track counts')
+        
         # Clean up the new artist list
         unique_artists = {}.fromkeys(new_artists).keys()
         current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
-        
+
-        artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
+        #There was a bug where artists with special characters (-,') would show up in new artists.     
-        
+        artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]]
+        artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]]
+
         # Update track counts
-        logger.info('Updating current artist track counts')
     
-        for artist in current_artists:
+        for artist in artists_checked:
             # Have tracks are selected from tracks table and not all tracks because of duplicates
             # We update the track count upon an album switch to compliment this
-            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
+            havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist]))
-            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])
+            #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases 
+            # (can fix by getting rid of second len statement)
+            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
             
         logger.info('Found %i new artists' % len(artist_list))
     
@@ -337,9 +314,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                 importer.artistlist_to_mbids(artist_list)
             else:
                 logger.info('To add these artists, go to Manage->Manage New Artists')
-                myDB.action('DELETE from newartists')
+                #myDB.action('DELETE from newartists')
                 for artist in artist_list:
-                    myDB.action('INSERT into newartists VALUES (?)', [artist])
+                    myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
         
         if headphones.DETECT_BITRATE:
             headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000
@@ -348,6 +325,42 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
         # If we're appending a new album to the database, update the artists total track counts
         logger.info('Updating artist track counts')
         
-        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName]))
+        havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName]))
         myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
-    
+
+    update_album_status()
+    logger.info('Library scan complete')
+
+    #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
+def update_album_status(AlbumID=None):
+    myDB = db.DBConnection()
+    logger.info('Counting matched tracks to mark albums as skipped/downloaded')
+    if AlbumID:
+        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
+    else:
+        album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
+    for album in album_status_updater:
+        track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']])
+        total_tracks = 0
+        have_tracks = 0
+        for track in track_counter:
+            total_tracks+=1
+            if track['Location']:
+                have_tracks+=1
+        if total_tracks != 0:
+            album_completion = float(float(have_tracks) / float(total_tracks)) * 100
+        else:
+            album_completion = 0
+            logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
+
+        if album_completion >= headphones.ALBUM_COMPLETION_PCT:
+            new_album_status = "Downloaded"
+        else:
+            if album['Status'] == "Skipped" or album['Status'] == "Downloaded":
+                new_album_status = "Skipped"
+            else:
+                new_album_status = album['Status']
+        myDB.upsert("albums", {'Status'   : new_album_status}, {'AlbumID'   : album['AlbumID']})
+        if new_album_status != album['Status']:
+            logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
+    logger.info('Album status update complete')

headphones/mb.py

@@ -19,7 +19,7 @@ import time
 import threading
 
 import headphones
-from headphones import logger, db
+from headphones import logger, db, helpers
 from headphones.helpers import multikeysort, replace_all
 
 import lib.musicbrainzngs as musicbrainzngs
@@ -332,7 +332,9 @@ def getRelease(releaseid, include_artist_info=True):
         
         return release
 
-def get_all_releases(rgid,includeExtras=False):
+def get_new_releases(rgid,includeExtras=False,forcefull=False):
+
+    myDB = db.DBConnection()
     results = []
     try:
         limit = 100
@@ -352,8 +354,29 @@ def get_all_releases(rgid,includeExtras=False):
     if not results or len(results) == 0:
         return False
 
-        
+    #Clean all references to releases in dB that are no longer referenced in musicbrainz
-    releases = []
+    release_list = []
+    force_repackage1 = 0
+    if len(results) != 0:
+        for release_mark in results:
+            release_list.append(unicode(release_mark['id']))
+            release_title = release_mark['title']
+        remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
+        if remove_missing_releases:
+            for items in remove_missing_releases:
+                if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
+                    # Remove all from albums/tracks that aren't in release
+                    myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
+                    myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
+                    logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
+                    force_repackage1 = 1
+    else:
+        logger.info("Error pulling data from MusicBrainz:  Maintaining dB")
+
+    num_new_releases = 0
+
     for releasedata in results:
         #releasedata.get will return None if it doesn't have a status
         #all official releases should have the Official status included
@@ -361,45 +384,130 @@ def get_all_releases(rgid,includeExtras=False):
             continue
         
         release = {}
-        release['AlbumTitle'] = unicode(releasedata['title'])
+        rel_id_check = releasedata['id']
-        release['AlbumID'] = unicode(rgid)
+        artistid = unicode(releasedata['artist-credit'][0]['artist']['id'])
-        release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
+
-        release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None      
+        album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
-        release['ReleaseID'] = releasedata['id']
+        if not album_checker or forcefull:
-        if 'release-group' not in releasedata:
+            #DELETE all references to this release since we're updating it anyway.
-            raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
+            myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check])
-        release['Type'] = unicode(releasedata['release-group']['type'])
+            myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check])
+            release['AlbumTitle'] = unicode(releasedata['title'])
+            release['AlbumID'] = unicode(rgid)
+            release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
+            release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None      
+            release['ReleaseID'] = releasedata['id']
+            if 'release-group' not in releasedata:
+                raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
+            release['Type'] = unicode(releasedata['release-group']['type'])
 
 
-        #making the assumption that the most important artist will be first in the list
+            #making the assumption that the most important artist will be first in the list
-        if 'artist-credit' in releasedata:
+            if 'artist-credit' in releasedata:
-            release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
+                release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
-            release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
+                release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
-        else:
+            else:
-            logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
+                logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
-            return False
+                return False
-                
+                    
 
-        release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
+            release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
-        #assuming that the list will contain media and that the format will be consistent
+            #assuming that the list will contain media and that the format will be consistent
-        try:
+            try:
-            release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format'])
+                additional_medium=''
-        except:
+                for position in releasedata['medium-list']:
-            release['ReleaseFormat'] = u'Unknown'
+                    if position['format'] == releasedata['medium-list'][0]['format']:
- 
+                        medium_count = int(position['position'])
-        release['Tracks'] = getTracksFromRelease(releasedata)
+                    else:
-        releases.append(release)
+                        additional_medium = additional_medium+' + '+position['format']
+                if medium_count == 1:
+                    disc_number = ''
+                else:
+                    disc_number = str(medium_count)+'x'
+                packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium
+                release['ReleaseFormat'] = unicode(packaged_medium)
+            except:
+                release['ReleaseFormat'] = u'Unknown'
+     
+            release['Tracks'] = getTracksFromRelease(releasedata)
 
-    return releases
+        # What we're doing here now is first updating the allalbums & alltracks table to the most
+        # current info, then moving the appropriate release into the album table and its associated
+        # tracks into the tracks table
+            controlValueDict = {"ReleaseID" : release['ReleaseID']}
+
+            newValueDict = {"ArtistID":         release['ArtistID'],
+                            "ArtistName":       release['ArtistName'],
+                            "AlbumTitle":       release['AlbumTitle'],
+                            "AlbumID":          release['AlbumID'],
+                            "AlbumASIN":        release['AlbumASIN'],
+                            "ReleaseDate":      release['ReleaseDate'],
+                            "Type":             release['Type'],
+                            "ReleaseCountry":   release['ReleaseCountry'],
+                            "ReleaseFormat":    release['ReleaseFormat']
+                        }
+
+            myDB.upsert("allalbums", newValueDict, controlValueDict)
+            
+            for track in release['Tracks']:
+
+                cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
+        
+                controlValueDict = {"TrackID":      track['id'],
+                                    "ReleaseID":    release['ReleaseID']}
+
+                newValueDict = {"ArtistID":         release['ArtistID'],
+                                "ArtistName":       release['ArtistName'],
+                                "AlbumTitle":       release['AlbumTitle'],
+                                "AlbumID":          release['AlbumID'],
+                                "AlbumASIN":        release['AlbumASIN'],
+                                "TrackTitle":       track['title'],
+                                "TrackDuration":    track['duration'],
+                                "TrackNumber":      track['number'],
+                                "CleanName":        cleanname
+                            }
+                            
+                match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
+            
+                if not match:
+                    match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
+                #if not match:
+                    #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()         
+                if match:
+                    newValueDict['Location'] = match['Location']
+                    newValueDict['BitRate'] = match['BitRate']
+                    newValueDict['Format'] = match['Format']
+                    #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
+                    myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location']))
+                                
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+            num_new_releases = num_new_releases + 1
+            #print releasedata['title']
+            #print num_new_releases
+            if album_checker:
+                logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+            else:
+                logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check))
+    if force_repackage1 == 1:
+        num_new_releases = -1
+        logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title))
+    else:
+        num_new_releases = num_new_releases
+
+    return num_new_releases
 
 def getTracksFromRelease(release):
     totalTracks = 1
     tracks = []
     for medium in release['medium-list']:
         for track in medium['track-list']:
+            try:
+                track_title = unicode(track['title'])
+            except:
+                track_title = unicode(track['recording']['title'])
             tracks.append({
                     'number':        totalTracks,
-                    'title':         unicode(track['recording']['title']),
+                    'title':         track_title,
                     'id':            unicode(track['recording']['id']),
                     'url':           u"http://musicbrainz.org/track/" + track['recording']['id'],
                     'duration':      int(track['length']) if 'length' in track else 0

headphones/music_encoder.py

@@ -244,7 +244,16 @@ def command(encoder,musicSource,musicDest,albumPath):
     logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
     logger.debug(subprocess.list2cmdline(cmd))
 
-    p = subprocess.Popen(cmd, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # stop windows opening the cmd
+    startupinfo = None
+    if headphones.SYS_PLATFORM == "win32":
+        startupinfo = subprocess.STARTUPINFO()
+        try:
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        except AttributeError:
+            startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
+
+    p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     stdout, stderr = p.communicate(headphones.ENCODER)
 

headphones/postprocessor.py

@@ -533,7 +533,7 @@ def moveFiles(albumpath, release, tracks):
             }
             
     folder = helpers.replace_all(headphones.FOLDER_FORMAT.strip(), values)
-    folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_')
+    folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_').replace('|','_')
     
     if folder.endswith('.'):
         folder = folder.replace(folder[len(folder)-1], '_')
@@ -843,6 +843,7 @@ def renameFiles(albumpath, downloaded_track_list, release):
         
         
         new_file_name = new_file_name.replace('?','_').replace(':', '_').encode(headphones.SYS_ENCODING, 'replace')
+        new_file_name = new_file_name.replace('*','_')
 
         if headphones.FILE_UNDERSCORES:
             new_file_name = new_file_name.replace(' ', '_')

headphones/searcher.py

@@ -118,7 +118,7 @@ def searchforalbum(albumid=None, new=False, lossless=False):
 
         for result in results:
             foundNZB = "none"
-            if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
+            if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
                 if result['Status'] == "Wanted Lossless":
                     foundNZB = searchNZB(result['AlbumID'], new, losslessOnly=True)
                 else:
@@ -135,12 +135,14 @@ def searchforalbum(albumid=None, new=False, lossless=False):
 
         foundNZB = "none"
         
-        if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
+        if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST):
             foundNZB = searchNZB(albumid, new, lossless)
 
         if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none":
             searchTorrent(albumid, new, lossless)
 
+    logger.info('Search for Wanted albums complete')
+    
 def searchNZB(albumid=None, new=False, losslessOnly=False):
 
     myDB = db.DBConnection()
@@ -437,6 +439,65 @@ def searchNZB(albumid=None, new=False, losslessOnly=False):
                         except Exception, e:
                             logger.error(u"An unknown error occurred trying to parse the feed: %s" % e)
 
+
+        if headphones.OMGWTFNZBS:
+
+            provider = "omgwtfnzbs"
+
+            if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
+                categories = "22"
+            elif headphones.PREFERRED_QUALITY:
+                categories = "22,7"
+            else:
+                 categories = "7"
+
+            if albums['Type'] == 'Other':
+                categories = "29"
+                logger.info("Album type is audiobook/spokenword. Searching all music categories")
+
+            params = {  "user": headphones.OMGWTFNZBS_UID,
+                        "api": headphones.OMGWTFNZBS_APIKEY,
+                        "catid": categories,
+                        "retention": headphones.USENET_RETENTION,
+                        "search": term
+                        }
+
+            searchURL = 'http://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params)
+
+            # Add a user-agent
+            request = urllib2.Request(searchURL)
+            request.add_header('User-Agent', 'headphones/0.0 +https://github.com/rembo10/headphones')
+            opener = urllib2.build_opener()
+
+            logger.info(u'Parsing results from <a href="%s">omgwtfnzbs</a>' % searchURL)
+
+            try:
+                data = opener.open(request).read()
+            except Exception, e:
+                logger.warn('Error fetching data from omgwtfnzbs: %s' % e)
+                data = False
+
+            if data:
+
+                d = json.loads(data)
+
+                if 'notice' in data: 
+                    logger.info(u"No results returned from omgwtfnzbs: %s" % d['notice'])
+                    pass
+
+                else:
+                    for item in d:
+                        try:
+                            url = item['getnzb']
+                            title = item['release']
+                            size = int(item['sizebytes'])
+
+                            resultlist.append((title, size, url, provider))
+                            logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
+
+                        except Exception, e:
+                            logger.error(u"An unknown error occurred trying to parse the results: %s" % e)
+
         # attempt to verify that this isn't a substring result
         # when looking for "Foo - Foo" we don't want "Foobar"
         # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
@@ -944,7 +1005,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                         try:
                             title = item.title
                             desc_match = re.search(r"Size: (\d+)<", item.description)
-                            size = desc_match.group(1)
+                            size = int(desc_match.group(1))
                             url = item.link
                             resultlist.append((title, size, url, provider))
                             logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
@@ -1009,11 +1070,11 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
             bitrate = None
             bitrate_string = bitrate
 
-            if headphones.PREFERRED_QUALITY == 2 or losslessOnly:  # Lossless Only mode
+            if headphones.PREFERRED_QUALITY == 3 or losslessOnly:  # Lossless Only mode
                 search_formats = [gazelleformat.FLAC]
                 maxsize = 10000000000
-            elif headphones.PREFERRED_QUALITY == 3:  # Preferred quality mode
+            elif headphones.PREFERRED_QUALITY == 2:  # Preferred quality mode
-                search_formats=[None]  # should return all
+                search_formats = [None]  # should return all
                 bitrate = headphones.PREFERRED_BITRATE
                 if bitrate:
                     for encoding_string in gazelleencoding.ALL_ENCODINGS:

headphones/transmission.py

@@ -20,6 +20,7 @@ import urllib2
 import lib.simplejson as json
 import base64
 import time
+import re
 
 # This is just a simple script to send torrents to transmission. The
 # intention is to turn this into a class where we can check the state
@@ -83,8 +84,13 @@ def torrentAction(method, arguments):
 
     if host.endswith('/'):
         host = host[:-1]
-	
+
-    host = host + "/transmission/rpc"
+    if not host.endswith('/rpc'):
+        if host.endswith(':\d{2,6}'):
+            host = host + "/transmission/rpc"
+        else:
+            host = host + "/rpc"
+
     request = urllib2.Request(host)
     if username and password:
         base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')

headphones/updater.py

@@ -17,17 +17,16 @@ import headphones
 
 from headphones import logger, db, importer
 
-def dbUpdate():
+def dbUpdate(forcefull=False):
 
     myDB = db.DBConnection()
 
     activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
-
     logger.info('Starting update for %i active artists' % len(activeartists))
     
     for artist in activeartists:
     
         artistid = artist[0]
-        importer.addArtisttoDB(artistid)
+        importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull)
         
-    logger.info('Update complete')
+    logger.info('Active artist update complete')

headphones/webserve.py

@@ -24,11 +24,14 @@ from mako import exceptions
 
 import time
 import threading
+import string
+import json
+from operator import itemgetter
 
 import headphones
 
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync
+from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers
-from headphones.helpers import checked, radio,today
+from headphones.helpers import checked, radio,today, cleanName
 
 import lib.simplejson as simplejson
 
@@ -105,7 +108,7 @@ class WebInterface(object):
     def albumPage(self, AlbumID):
         myDB = db.DBConnection()
         album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-        tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [AlbumID])
+        tracks = myDB.select('SELECT * from tracks WHERE AlbumID=? ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID])
         description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=?', [AlbumID]).fetchone()
         title = album['ArtistName'] + ' - ' + album['AlbumTitle']
         return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description)
@@ -150,7 +153,7 @@ class WebInterface(object):
         newValueDict = {'IncludeExtras': 1,
                         'Extras':        extras}
         myDB.upsert("artists", newValueDict, controlValueDict)
-        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True]).start()
+        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start()
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     getExtras.exposed = True
 
@@ -163,6 +166,8 @@ class WebInterface(object):
         for album in extraalbums:
             myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
             myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
+            myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
+            myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']])
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     removeExtras.exposed = True
 
@@ -187,11 +192,15 @@ class WebInterface(object):
     def deleteArtist(self, ArtistID):
         logger.info(u"Deleting all traces of artist: " + ArtistID)
         myDB = db.DBConnection()
+        namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID])
+        for name in namecheck:
+            artistname=name['ArtistName']
         myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID])
         myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID])
+        myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname])
         myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID])
         raise cherrypy.HTTPRedirect("home")
     deleteArtist.exposed = True
@@ -211,7 +220,7 @@ class WebInterface(object):
     deleteEmptyArtists.exposed = True
 
     def refreshArtist(self, ArtistID):
-        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start()
+        threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start()
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
     refreshArtist.exposed=True
 
@@ -238,8 +247,15 @@ class WebInterface(object):
             raise cherrypy.HTTPRedirect("upcoming")
     markAlbums.exposed = True
 
-    def addArtists(self, **args):
+    def addArtists(self, action=None, **args):
-        threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start()
+        if action == "add":
+            threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start()
+        if action == "ignore":
+            myDB = db.DBConnection()
+            for artist in args:
+                myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist])
+                myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist])
+                logger.info("Artist %s removed from new artist list and set to ignored" % artist)
         raise cherrypy.HTTPRedirect("home")
     addArtists.exposed = True
 
@@ -274,6 +290,8 @@ class WebInterface(object):
         myDB = db.DBConnection()
         myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID])
         myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID])
+        myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID])
+        myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID])
         if ArtistID:
             raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
         else:
@@ -334,6 +352,193 @@ class WebInterface(object):
         return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists)
     manageNew.exposed = True
 
+    def manageUnmatched(self):
+        myDB = db.DBConnection()
+        have_album_dictionary = []
+        headphones_album_dictionary = []
+        unmatched_albums = []
+        have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName')
+        for albums in have_albums:
+            #Have to skip over manually matched tracks
+            if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
+                original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle'])
+            # else:
+            #     original_clean = None
+                if original_clean == albums['CleanName']:
+                    have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }  
+                    have_album_dictionary.append(have_dict)
+        headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName')
+        for albums in headphones_albums:
+            headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }  
+            headphones_album_dictionary.append(headphones_dict) 
+        #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]]
+
+        check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary])
+        unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check]
+
+
+        return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums)
+    manageUnmatched.exposed = True
+
+    def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None):
+        myDB = db.DBConnection()
+     
+        if action == "ignoreArtist":
+            artist = existing_artist
+            myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched = "Failed"', [artist])
+        
+        elif action == "ignoreAlbum":
+            artist = existing_artist
+            album = existing_album
+            myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched = "Failed"', (artist, album))
+        
+        elif action == "matchArtist":
+            existing_artist_clean = helpers.cleanName(existing_artist).lower()
+            new_artist_clean = helpers.cleanName(new_artist).lower()
+            if new_artist_clean != existing_artist_clean:
+                have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist])
+                update_count = 0
+                for entry in have_tracks:
+                    old_clean_filename = entry['CleanName']
+                    if old_clean_filename.startswith(existing_artist_clean):
+                        new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1)
+                        myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename])
+                        controlValueDict = {"CleanName": new_clean_filename}
+                        newValueDict = {"Location" : entry['Location'],
+                                        "BitRate" : entry['BitRate'],
+                                        "Format" : entry['Format']
+                                        }
+                        #Attempt to match tracks with new CleanName
+                        match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_alltracks:
+                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                        match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_tracks:
+                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename])
+                            update_count+=1
+                    #This was throwing errors and I don't know why, but it seems to be working fine.
+                    #else:
+                        #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist)
+                logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist))
+                if update_count > 0:
+                    librarysync.update_album_status()
+            else:
+                logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist)
+
+        elif action == "matchAlbum":
+            existing_artist_clean = helpers.cleanName(existing_artist).lower()
+            new_artist_clean = helpers.cleanName(new_artist).lower()
+            existing_album_clean = helpers.cleanName(existing_album).lower()
+            new_album_clean = helpers.cleanName(new_album).lower()
+            existing_clean_string = existing_artist_clean+" "+existing_album_clean
+            new_clean_string = new_artist_clean+" "+new_album_clean
+            if existing_clean_string != new_clean_string:
+                have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album))
+                update_count = 0
+                for entry in have_tracks:
+                    old_clean_filename = entry['CleanName']
+                    if old_clean_filename.startswith(existing_clean_string):
+                        new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1)
+                        myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename])
+                        controlValueDict = {"CleanName": new_clean_filename}
+                        newValueDict = {"Location" : entry['Location'],
+                                        "BitRate" : entry['BitRate'],
+                                        "Format" : entry['Format']
+                                        }
+                        #Attempt to match tracks with new CleanName
+                        match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_alltracks:
+                            myDB.upsert("alltracks", newValueDict, controlValueDict)
+                        match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone()
+                        if match_tracks:
+                            myDB.upsert("tracks", newValueDict, controlValueDict)
+                            myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename])
+                            album_id = match_tracks['AlbumID']
+                            update_count+=1
+                    #This was throwing errors and I don't know why, but it seems to be working fine.
+                    #else:
+                        #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string))
+                logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (update_count, new_artist, new_album))
+                if update_count > 0:
+                    librarysync.update_album_status(album_id)
+            else:
+                logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album))
+                
+    markUnmatched.exposed = True
+
+    def manageManual(self):
+        myDB = db.DBConnection()
+        manual_albums = []
+        manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
+        for albums in manualalbums:
+            if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
+                original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle'])
+                if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean:
+                    if albums['Matched'] == "Ignored":
+                        album_status = "Ignored"
+                    elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean:
+                        album_status = "Matched"
+                    manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status }
+                    if manual_dict not in manual_albums: 
+                        manual_albums.append(manual_dict)
+        manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle'))
+
+        return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted)
+    manageManual.exposed = True
+
+    def markManual(self, action=None, existing_artist=None, existing_album=None):
+        myDB = db.DBConnection()
+        if action == "unignoreArtist":
+            artist = existing_artist
+            myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND Matched="Ignored"', [artist])
+            logger.info("Artist: %s successfully restored to unmatched list" % artist)
+
+        elif action == "unignoreAlbum":
+            artist = existing_artist
+            album = existing_album
+            myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album))
+            logger.info("Album: %s successfully restored to unmatched list" % album)
+
+        elif action == "unmatchArtist":
+            artist = existing_artist
+            update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', [artist])
+            update_count = 0
+            for tracks in update_clean:
+                original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower()
+                album = tracks['AlbumTitle']
+                track_title = tracks['TrackTitle']
+                if tracks['CleanName'] != original_clean:
+                    myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title))
+                    update_count+=1
+            if update_count > 0:
+                librarysync.update_album_status()
+            logger.info("Artist: %s successfully restored to unmatched list" % artist)
+
+        elif action == "unmatchAlbum":
+            artist = existing_artist
+            album = existing_album
+            update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', (artist, album))
+            update_count = 0
+            for tracks in update_clean:
+                original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower()
+                track_title = tracks['TrackTitle']
+                if tracks['CleanName'] != original_clean:
+                    album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', [tracks['CleanName']]).fetchone()
+                    if album_id_check:
+                        album_id = album_id_check[0]
+                    myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']])
+                    myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title))
+                    update_count+=1
+            if update_count > 0:
+                librarysync.update_album_status(album_id)
+            logger.info("Album: %s successfully restored to unmatched list" % album)
+
+    markManual.exposed = True
+
     def markArtists(self, action=None, **args):
         myDB = db.DBConnection()
         artistsToAdd = []
@@ -342,6 +547,8 @@ class WebInterface(object):
                 myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
                 myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
                 myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
+                myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID])
+                myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID])
                 myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID])
             elif action == 'pause':
                 controlValueDict = {'ArtistID': ArtistID}
@@ -397,10 +604,16 @@ class WebInterface(object):
 
     def forceUpdate(self):
         from headphones import updater
-        threading.Thread(target=updater.dbUpdate).start()
+        threading.Thread(target=updater.dbUpdate, args=[False]).start()
         raise cherrypy.HTTPRedirect("home")
     forceUpdate.exposed = True
 
+    def forceFullUpdate(self):
+        from headphones import updater
+        threading.Thread(target=updater.dbUpdate, args=[True]).start()
+        raise cherrypy.HTTPRedirect("home")
+    forceFullUpdate.exposed = True
+
     def forceSearch(self):
         from headphones import searcher
         threading.Thread(target=searcher.searchforalbum).start()
@@ -533,6 +746,20 @@ class WebInterface(object):
         return s
     getArtists_json.exposed=True
 
+    def getAlbumsByArtist_json(self, artist=None):
+        myDB = db.DBConnection()
+        album_json = {}
+        counter = 0
+        album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist])
+        for album in album_list:
+            album_json[counter] = album['AlbumTitle']
+            counter+=1
+        json_albums = json.dumps(album_json)
+        
+        cherrypy.response.headers['Content-type'] = 'application/json'
+        return json_albums
+    getAlbumsByArtist_json.exposed=True
+
     def clearhistory(self, type=None):
         myDB = db.DBConnection()
         if type == 'all':
@@ -554,6 +781,26 @@ class WebInterface(object):
 
     generateAPI.exposed = True
 
+    def forceScan(self, keepmatched=None):
+        myDB = db.DBConnection()
+        #########################################
+        #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE
+        myDB.select('DELETE from Have')
+        logger.info('Removed all entries in local library database')
+        myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL')
+        myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL')
+        logger.info('All tracks in library unmatched')
+        myDB.action('UPDATE artists SET HaveTracks=NULL')
+        logger.info('Reset track counts for all artists')
+        myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"')
+        logger.info('Marking all unwanted albums as Skipped')
+        try:
+            threading.Thread(target=librarysync.libraryScan).start()
+        except Exception, e:
+            logger.error('Unable to complete the scan: %s' % e)
+        raise cherrypy.HTTPRedirect("home")
+    forceScan.exposed = True
+
     def config(self):
 
         interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/')
@@ -571,6 +818,8 @@ class WebInterface(object):
                     "api_enabled" : checked(headphones.API_ENABLED),
                     "api_key" : headphones.API_KEY,
                     "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL,
+                    "update_db_interval" : headphones.UPDATE_DB_INTERVAL,
+                    "mb_ignore_age" : headphones.MB_IGNORE_AGE,
                     "nzb_search_interval" : headphones.SEARCH_INTERVAL,
                     "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL,
                     "sab_host" : headphones.SAB_HOST,
@@ -610,6 +859,9 @@ class WebInterface(object):
                     "use_nzbsrus" : checked(headphones.NZBSRUS),
                     "nzbsrus_uid" : headphones.NZBSRUS_UID,
                     "nzbsrus_apikey" : headphones.NZBSRUS_APIKEY,
+                    "use_omgwtfnzbs" : checked(headphones.OMGWTFNZBS),
+                    "omgwtfnzbs_uid" : headphones.OMGWTFNZBS_UID,
+                    "omgwtfnzbs_apikey" : headphones.OMGWTFNZBS_APIKEY,
                     "preferred_words" : headphones.PREFERRED_WORDS,
                     "ignored_words" : headphones.IGNORED_WORDS,
                     "required_words" : headphones.REQUIRED_WORDS,
@@ -720,10 +972,10 @@ class WebInterface(object):
     config.exposed = True
 
     def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None,
-        download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None,
+        download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None,
         sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, 
         utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, 
-        use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, 
+        use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, omgwtfnzbs=0, omgwtfnzbs_uid=None, omgwtfnzbs_apikey=None, 
         preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None,
         numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None,
         rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0,
@@ -747,6 +999,8 @@ class WebInterface(object):
         headphones.API_ENABLED = api_enabled
         headphones.API_KEY = api_key
         headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval
+        headphones.UPDATE_DB_INTERVAL = update_db_interval
+        headphones.MB_IGNORE_AGE = mb_ignore_age
         headphones.SEARCH_INTERVAL = nzb_search_interval
         headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval
         headphones.SAB_HOST = sab_host
@@ -780,6 +1034,9 @@ class WebInterface(object):
         headphones.NZBSRUS = nzbsrus
         headphones.NZBSRUS_UID = nzbsrus_uid
         headphones.NZBSRUS_APIKEY = nzbsrus_apikey
+        headphones.OMGWTFNZBS = omgwtfnzbs
+        headphones.OMGWTFNZBS_UID = omgwtfnzbs_uid
+        headphones.OMGWTFNZBS_APIKEY = omgwtfnzbs_apikey
         headphones.PREFERRED_WORDS = preferred_words
         headphones.IGNORED_WORDS = ignored_words
         headphones.REQUIRED_WORDS = required_words

lib/beets/mediafile.py

@@ -572,9 +572,23 @@ class ImageField(object):
 
             # No cover found.
             return None
+	elif obj.type == 'flac':
+            if 'metadata_block_picture' not in obj.mgfile:
+                return None
+
+            for data in obj.mgfile['metadata_block_picture']:
+                try:
+                    pic = lib.mutagen.flac.Picture(data)
+                    break
+                except TypeError:
+                    pass
+            else:
+                return None
+
+            return pic.data
 
         else:
-            # Here we're assuming everything but MP3 and MPEG-4 uses
+	    # Here we're assuming everything but MP3, FLAC and MPEG-4 use
             # the Xiph/Vorbis Comments standard. This may not be valid.
             # http://wiki.xiph.org/VorbisComment#Cover_art
 
@@ -623,6 +637,13 @@ class ImageField(object):
             else:
                 cover = lib.mutagen.mp4.MP4Cover(val, self._mp4kind(val))
                 obj.mgfile['covr'] = [cover]
+	
+        elif obj.type == 'flac':
+            if val is None:
+                pic = lib.mutagen.flac.Picture()
+                pic.data = val
+                pic.mime = self._mime(val)
+                obj.mgfile['metadata_block_picture'] = [pic.write()]
 
         else:
             # Again, assuming Vorbis Comments standard.

lib/pygazelle/api.py

@@ -51,6 +51,7 @@ class GazelleAPI(object):
         self.passkey = None
         self.userid = None
         self.logged_in_user = None
+        self.default_timeout = 30
         self.cached_users = {}
         self.cached_artists = {}
         self.cached_tags = {}
@@ -97,7 +98,8 @@ class GazelleAPI(object):
         loginpage = 'https://what.cd/login.php'
         data = {'username': self.username,
                 'password': self.password}
-        r = self.session.post(loginpage, data=data)
+        r = self.session.post(loginpage, data=data, timeout=self.default_timeout)
+        self.past_request_timestamps.append(time.time())
         if r.status_code != 200:
             raise LoginException("Login returned status code %s" % r.status_code)
 
@@ -112,7 +114,6 @@ class GazelleAPI(object):
         self.passkey = accountinfo['passkey']
         self.logged_in_user = User(self.userid, self)
         self.logged_in_user.set_index_data(accountinfo)
-        self.past_request_timestamps.append(time.time())
 
     def request(self, action, autologin=True, **kwargs):
         """
@@ -153,7 +154,7 @@ class GazelleAPI(object):
         if self.authkey:
             params['auth'] = self.authkey
         params.update(kwargs)
-        r = self.session.get(url, params=params, allow_redirects=False)
+        r = self.session.get(url, params=params, allow_redirects=False, timeout=self.default_timeout)
         self.past_request_timestamps.append(time.time())
         return r.content