gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-20 07:54:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merged sbusers searcher & post processor changes, unicode error fix, added manage artists page, bug fixes

Browse Source
This commit is contained in:
Remy
2011-08-11 00:56:36 -07:00
parent c2e3b5e460
commit 6f06a740c3
13 changed files with 278 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/css/style.css
+1
View File
@@ -143,6 +143,7 @@ div#main { margin: 0; padding: 80px 0 0 0; }
table#artist_table { background-color: white; width: 100%; padding: 20px; }
table#artist_table th#select { text-align: left; }
table#artist_table th#name { text-align: left; min-width: 200px; }
table#artist_table th#status { text-align: left; min-width: 50px; }
table#artist_table th#album { text-align: left; min-width: 300px; }
data/interfaces/default/artist.html
+1 -1
View File
@@ -74,7 +74,7 @@
%>
<tr class="grade${grade}">
<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></th>
<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></td>
<td id="albumart"><img src="http://ec1.images-amazon.com/images/P/${album['AlbumASIN']}.01.MZZZZZZZ.jpg" height="50" width="50"></td>
<td id="albumname"><a href="albumPage?AlbumID=${album['AlbumID']}">${album['AlbumTitle']}</a></td>
<td id="reldate">${album['ReleaseDate']}</td>
data/interfaces/default/manage.html
+10
View File
@@ -2,8 +2,18 @@
<%!
import headphones
%>
<%def name="headerIncludes()">
<div id="subhead_container">
<ul id="subhead_menu">
<li><a href="manageArtists">Manage Artists</a></li>
</ul>
</div>
</%def>
<%def name="body()">
<div id="paddingheader">
<h1><h1>
</div>
<div class="table_wrapper">
<h1>Scan Music Library</h1><br />
Where do you keep your music?<br /><br />
data/interfaces/default/manageartists.html
+69
View File
@@ -0,0 +1,69 @@
<%inherit file="base.html" />
<%def name="body()">
<div id="paddingheader">
<h1>Manage Artists<h1>
</div>
<form action="markArtists" method="get">
<p class="indented">
<select name="action">
<option value="pause">Pause</option>
<option value="resume">Resume</option>
<option value="refresh">Refresh</option>
<option value="delete">Delete</option>
</select>
selected artists
<input type="submit" value="Go">
</p>
<table class="display" id="artist_table">
<thead>
<tr>
<th id="select"><input type="checkbox" onClick="toggle(this)" /></th>
<th id="name">Artist Name</th>
<th id="status">Status</th>
</tr>
</thead>
<tbody>
%for artist in artists:
<%
if artist['Status'] == 'Paused':
grade = 'X'
elif artist['Status'] == 'Loading':
grade = 'C'
else:
grade = 'Z'
%>
<tr class="grade${grade}">
<td id="select"><input type="checkbox" name="${artist['ArtistID']}" class="checkbox" /></td>
<td id="name"><span title="${artist['ArtistSortName']}"></span><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></td>
<td id="status">${artist['Status']}</td>
</tr>
%endfor
</tbody>
</table>
</form>
</%def>
<%def name="headIncludes()">
<link rel="stylesheet" href="css/data_table.css">
</%def>
<%def name="javascriptIncludes()">
<script src="js/libs/jquery.dataTables.min.js"></script>
<script>
$(document).ready(function()
{
$('#artist_table').dataTable(
{
"aoColumns": [
null,
{ "sType": "title-string"},
null
],
"bStateSave": true,
"bPaginate": false
});
});
</script>
</%def>
data/interfaces/remix/artist.html
+1 -1
View File
@@ -74,7 +74,7 @@
%>
<tr class="grade${grade}">
<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></th>
<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></td>
<td id="albumart"><img src="http://ec1.images-amazon.com/images/P/${album['AlbumASIN']}.01.MZZZZZZZ.jpg" height="50" width="50"></td>
<td id="albumname"><a href="albumPage?AlbumID=${album['AlbumID']}">${album['AlbumTitle']}</a></td>
<td id="reldate">${album['ReleaseDate']}</td>
data/interfaces/remix/manage.html
+10
View File
@@ -2,8 +2,18 @@
<%!
import headphones
%>
<%def name="headerIncludes()">
<div id="subhead_container">
<ul id="subhead_menu">
<li><a href="manageArtists">Manage Artists</a></li>
</ul>
</div>
</%def>
<%def name="body()">
<div id="paddingheader">
<h1><h1>
</div>
<div class="table_wrapper">
<h1>Scan Music Library</h1><br />
Where do you keep your music?<br /><br />
data/interfaces/remix/manageartists.html
+69
View File
@@ -0,0 +1,69 @@
<%inherit file="base.html" />
<%def name="body()">
<div id="paddingheader">
<h1>Manage Artists<h1>
</div>
<form action="markArtists" method="get">
<p class="indented">
<select name="action">
<option value="pause">Pause</option>
<option value="resume">Resume</option>
<option value="refresh">Refresh</option>
<option value="delete">Delete</option>
</select>
selected artists
<input type="submit" value="Go">
</p>
<table class="display" id="artist_table">
<thead>
<tr>
<th id="select"><input type="checkbox" onClick="toggle(this)" /></th>
<th id="name">Artist Name</th>
<th id="status">Status</th>
</tr>
</thead>
<tbody>
%for artist in artists:
<%
if artist['Status'] == 'Paused':
grade = 'X'
elif artist['Status'] == 'Loading':
grade = 'C'
else:
grade = 'Z'
%>
<tr class="grade${grade}">
<td id="select"><input type="checkbox" name="${artist['ArtistID']}" class="checkbox" /></td>
<td id="name"><span title="${artist['ArtistSortName']}"></span><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></td>
<td id="status">${artist['Status']}</td>
</tr>
%endfor
</tbody>
</table>
</form>
</%def>
<%def name="headIncludes()">
<link rel="stylesheet" href="css/data_table.css">
</%def>
<%def name="javascriptIncludes()">
<script src="js/libs/jquery.dataTables.min.js"></script>
<script>
$(document).ready(function()
{
$('#artist_table').dataTable(
{
"aoColumns": [
null,
{ "sType": "title-string"},
null
],
"bStateSave": true,
"bPaginate": false
});
});
</script>
</%def>
headphones/__init__.py
+4 -1
View File
@@ -105,6 +105,7 @@ LASTFM_USERNAME = None
MEDIA_FORMATS = ["mp3", "flac", "aac", "ogg", "ape", "m4a"]
INTERFACE = None
FOLDER_PERMISSIONS = None
def CheckSection(sec):
""" Check if INI section exists, if not create it """
@@ -163,7 +164,7 @@ def initialize():
ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \
LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \
NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE
NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS
if __INITIALIZED__:
return False
@@ -241,6 +242,7 @@ def initialize():
LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '')
INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default')
FOLDER_PERMISSIONS = check_setting_str(CFG, 'General', 'folder_permissions', '0755')
if not LOG_DIR:
LOG_DIR = os.path.join(DATA_DIR, 'logs')
@@ -408,6 +410,7 @@ def config_write():
new_config['General']['lastfm_username'] = LASTFM_USERNAME
new_config['General']['interface'] = INTERFACE
new_config['General']['folder_permissions'] = FOLDER_PERMISSIONS
new_config.write()
headphones/exceptions.py
+26
View File
@@ -0,0 +1,26 @@
def ex(e):
"""
Returns a string from the exception text if it exists.
"""
# sanity check
if not e.args or not e.args[0]:
return ""
e_message = e.args[0]
# if fixStupidEncodings doesn't fix it then maybe it's not a string, in which case we'll try printing it anyway
if not e_message:
try:
e_message = str(e.args[0])
except:
e_message = ""
return e_message
class HeadphonesException(Exception):
"Generic Headphones Exception - should never be thrown, only subclassed"
class NewzbinAPIThrottled(HeadphonesException):
"Newzbin has throttled us, deal with it"
headphones/mb.py
+2 -2
View File
@@ -24,7 +24,7 @@ def findArtist(name, limit=1):
attempt = 0
artistResults = None
chars = set('!?')
chars = set('!?*')
if any((c in chars) for c in name):
name = '"'+name+'"'
@@ -47,7 +47,7 @@ def findArtist(name, limit=1):
if result.artist.name != result.artist.getUniqueName() and limit == 1:
logger.info('Found an artist with a disambiguation: %s - doing an album based search' % name)
logger.debug('Found an artist with a disambiguation: %s - doing an album based search' % name)
artistdict = findArtistbyAlbum(name)
if not artistdict:
headphones/postprocessor.py
+12 -8
View File
@@ -111,11 +111,6 @@ def verify(albumid, albumpath):
tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
downloaded_track_list = []
try:
albumpath = str(albumpath)
except UnicodeEncodeError:
albumpath = unicode(albumpath).encode('unicode_escape')
for r,d,f in os.walk(albumpath):
for files in f:
@@ -203,6 +198,10 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
if headphones.MOVE_FILES and headphones.DESTINATION_DIR:
albumpath = moveFiles(albumpath, release, tracks)
if headphones.MOVE_FILES and not headphones.DESTINATION_DIR:
logger.error('No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to')
pass
myDB = db.DBConnection()
# There's gotta be a better way to update the have tracks - sqlite
@@ -306,10 +305,12 @@ def moveFiles(albumpath, release, tracks):
# Chmod the directories using the folder_format (script courtesy of premiso!)
folder_list = folder.split('/')
temp_f = os.path.join(headphones.DESTINATION_DIR);
for f in folder_list:
temp_f = os.path.join(temp_f, f)
os.chmod(temp_f, 0755)
os.chmod(temp_f, int(headphones.FOLDER_PERMISSIONS, 8))
except Exception, e:
logger.error('Could not create folder for %s. Not moving: %s' % (release['AlbumTitle'], e))
@@ -442,6 +443,7 @@ def renameUnprocessedFolder(albumpath):
def forcePostProcess():
if not headphones.DOWNLOAD_DIR:
logger.error('No DOWNLOAD_DIR has been set. Set "Music Download Directory:" to your SAB download directory on the settings page.')
return
else:
download_dir = headphones.DOWNLOAD_DIR
@@ -459,8 +461,11 @@ def forcePostProcess():
# Parse the folder names to get artist album info
for folder in folders:
folder = unicode(folder)
albumpath = unicode(os.path.join(download_dir, folder))
albumpath = os.path.join(download_dir, folder)
try:
name, album, year = helpers.extract_data(folder)
except:
@@ -482,7 +487,6 @@ def forcePostProcess():
logger.error('Can not get release information for this album')
continue
if rgid:
rgid = unicode(rgid)
verify(rgid, albumpath)
headphones/searcher.py
+45 -27
View File
@@ -4,7 +4,7 @@ from xml.dom import minidom
from xml.parsers.expat import ExpatError
import os, re, time
import headphones
import headphones, exceptions
from headphones import logger, db, helpers, classes, sab
class NewzbinDownloader(urllib.FancyURLopener):
@@ -23,6 +23,10 @@ class NewzbinDownloader(urllib.FancyURLopener):
rtext = str(headers.getheader('X-DNZB-RText'))
result = re.search("wait (\d+) seconds", rtext)
logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds")
time.sleep(int(result.group(1)))
raise exceptions.NewzbinAPIThrottled()
elif newzbinErrCode == 401:
logger.info("Newzbin error 401")
#raise exceptions.AuthException("Newzbin username or password incorrect")
@@ -31,12 +35,6 @@ class NewzbinDownloader(urllib.FancyURLopener):
#raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs")
logger.info("Newzbin error 402")
logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + "seconds")
time.sleep(int(result.group(1)))
#raise exceptions.NewzbinAPIThrottled()
#this should be in a class somewhere
def getNewzbinURL(url):
@@ -72,7 +70,7 @@ def searchNZB(albumid=None, new=False):
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':''}
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':''}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
@@ -296,7 +294,11 @@ def searchNZB(albumid=None, new=False):
"q": term
}
searchURL = providerurl + "search/?%s" % urllib.urlencode(params)
data = getNewzbinURL(searchURL)
try:
data = getNewzbinURL(searchURL)
except exceptions.NewzbinAPIThrottled:
#try again if we were throttled
data = getNewzbinURL(searchURL)
if data:
logger.info(u'Parsing results from <a href="%s">%s</a>' % (searchURL, providerurl))
@@ -306,7 +308,7 @@ def searchNZB(albumid=None, new=False):
items = d.getElementsByTagName("item")
except ExpatError:
logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are')
items = None
items = []
if len(items):
@@ -343,7 +345,7 @@ def searchNZB(albumid=None, new=False):
#when looking for "Foo - Foo" we don't want "Foobar"
#this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
if len(resultlist):
resultlist[:] = [result for result in resultlist if verifyresult(result[0], artistterm)]
resultlist[:] = [result for result in resultlist if verifyresult(result[0], artistterm, term)]
if len(resultlist):
@@ -431,26 +433,36 @@ def searchNZB(albumid=None, new=False):
myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]])
myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name])
def verifyresult(title, term):
def verifyresult(title, artistterm, term):
title = re.sub('[\.\-\/\_]', ' ', title)
if term == 'Various Artists':
return True
if not re.search('^' + re.escape(term), title, re.IGNORECASE):
logger.info("Removed from results: " + title + " (artist not at string start).")
return False
elif re.search(re.escape(term) + '\w', title, re.IGNORECASE | re.UNICODE):
logger.info("Removed from results: " + title + " (post substring result).")
return False
elif re.search('\w' + re.escape(term), title, re.IGNORECASE | re.UNICODE):
logger.info("Removed from results: " + title + " (pre substring result).")
return False
else:
return True
if artistterm != 'Various Artists':
if not re.search('^' + re.escape(artistterm), title, re.IGNORECASE):
logger.info("Removed from results: " + title + " (artist not at string start).")
return False
elif re.search(re.escape(artistterm) + '\w', title, re.IGNORECASE | re.UNICODE):
logger.info("Removed from results: " + title + " (post substring result).")
return False
elif re.search('\w' + re.escape(artistterm), title, re.IGNORECASE | re.UNICODE):
logger.info("Removed from results: " + title + " (pre substring result).")
return False
#another attempt to weed out substrings. We don't want "Vol III" when we were looking for "Vol II"
tokens = re.split('\W', term, re.IGNORECASE | re.UNICODE)
for token in tokens:
if token == 'Various' or token == 'Artists' or token == 'VA':
continue
if not re.search('(?:\W|^)+' + token + '(?:\W|$)+', title, re.IGNORECASE | re.UNICODE):
logger.info("Removed from results: " + title + " (missing token: " + token + ")")
return False
return True
def getresultNZB(result):
nzb = None
if result[3] == 'newzbin':
params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]})
url = "https://www.newzbin.com" + "/api/dnzb/"
@@ -459,6 +471,12 @@ def getresultNZB(result):
nzb = urllib.urlopen(url, data=params).read()
except urllib2.URLError, e:
logger.warn('Error fetching nzb from url: %s. Error: %s' % (url, e))
except exceptions.NewzbinAPIThrottled:
#TODO: This has created a potentially infinite loop? As long as they keep throttling we keep trying.
logger.info("Done waiting for Newzbin API throttle limit, starting downloads again")
getresultNZB(result)
except AttributeError:
logger.warn("AttributeError in getresultNZB.")
else:
try:
nzb = urllib2.urlopen(result[2], timeout=30).read()
@@ -469,7 +487,7 @@ def getresultNZB(result):
def preprocess(resultlist):
if not headphones.USENET_RETENTION:
usenet_retention = 1000
usenet_retention = 2000
else:
usenet_retention = int(headphones.USENET_RETENTION)
headphones/webserve.py
+28
View File
@@ -183,6 +183,34 @@ class WebInterface(object):
return serve_template(templatename="manage.html", title="Manage")
manage.exposed = True
def manageArtists(self):
myDB = db.DBConnection()
artists = myDB.select('SELECT * from artists order by ArtistSortName COLLATE NOCASE')
return serve_template(templatename="manageartists.html", title="Manage Artists", artists=artists)
manageArtists.exposed = True
def markArtists(self, action=None, **args):
myDB = db.DBConnection()
for ArtistID in args:
if action == 'delete':
myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
elif action == 'pause':
controlValueDict = {'ArtistID': ArtistID}
newValueDict = {'Status': 'Paused'}
myDB.upsert("artists", newValueDict, controlValueDict)
elif action == 'resume':
controlValueDict = {'ArtistID': ArtistID}
newValueDict = {'Status': 'Active'}
myDB.upsert("artists", newValueDict, controlValueDict)
else:
# These may and probably will collide - need to make a better way to queue musicbrainz queries
threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start()
time.sleep(30)
raise cherrypy.HTTPRedirect("home")
markArtists.exposed = True
def importLastFM(self, username):
headphones.LASTFM_USERNAME = username
headphones.config_write()