diff --git a/data/css/style.css b/data/css/style.css index 8b7803e7..552c9a5b 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -143,6 +143,7 @@ div#main { margin: 0; padding: 80px 0 0 0; } table#artist_table { background-color: white; width: 100%; padding: 20px; } +table#artist_table th#select { text-align: left; } table#artist_table th#name { text-align: left; min-width: 200px; } table#artist_table th#status { text-align: left; min-width: 50px; } table#artist_table th#album { text-align: left; min-width: 300px; } diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 241df532..4b296eda 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -74,7 +74,7 @@ %> - + ${album['AlbumTitle']} ${album['ReleaseDate']} diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 8b891317..3e6f7452 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -2,8 +2,18 @@ <%! import headphones %> +<%def name="headerIncludes()"> +
+ +
+ <%def name="body()"> +
+

+

Scan Music Library


Where do you keep your music?

diff --git a/data/interfaces/default/manageartists.html b/data/interfaces/default/manageartists.html new file mode 100644 index 00000000..d1762cb2 --- /dev/null +++ b/data/interfaces/default/manageartists.html @@ -0,0 +1,69 @@ +<%inherit file="base.html" /> + +<%def name="body()"> +
+

Manage Artists

+

+
+

+ + selected artists + +

+ + + + + + + + + + %for artist in artists: + <% + if artist['Status'] == 'Paused': + grade = 'X' + elif artist['Status'] == 'Loading': + grade = 'C' + else: + grade = 'Z' + %> + + + + + + %endfor + +
Artist NameStatus
${artist['ArtistName']}${artist['Status']}
+
+ + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + \ No newline at end of file diff --git a/data/interfaces/remix/artist.html b/data/interfaces/remix/artist.html index 241df532..4b296eda 100644 --- a/data/interfaces/remix/artist.html +++ b/data/interfaces/remix/artist.html @@ -74,7 +74,7 @@ %> - + ${album['AlbumTitle']} ${album['ReleaseDate']} diff --git a/data/interfaces/remix/manage.html b/data/interfaces/remix/manage.html index 8b891317..3e6f7452 100644 --- a/data/interfaces/remix/manage.html +++ b/data/interfaces/remix/manage.html @@ -2,8 +2,18 @@ <%! import headphones %> +<%def name="headerIncludes()"> +
+ +
+ <%def name="body()"> +
+

+

Scan Music Library


Where do you keep your music?

diff --git a/data/interfaces/remix/manageartists.html b/data/interfaces/remix/manageartists.html new file mode 100644 index 00000000..d1762cb2 --- /dev/null +++ b/data/interfaces/remix/manageartists.html @@ -0,0 +1,69 @@ +<%inherit file="base.html" /> + +<%def name="body()"> +
+

Manage Artists

+

+
+

+ + selected artists + +

+ + + + + + + + + + %for artist in artists: + <% + if artist['Status'] == 'Paused': + grade = 'X' + elif artist['Status'] == 'Loading': + grade = 'C' + else: + grade = 'Z' + %> + + + + + + %endfor + +
Artist NameStatus
${artist['ArtistName']}${artist['Status']}
+
+ + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + \ No newline at end of file diff --git a/headphones/__init__.py b/headphones/__init__.py index e916057a..7f4d7e6b 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -105,6 +105,7 @@ LASTFM_USERNAME = None MEDIA_FORMATS = ["mp3", "flac", "aac", "ogg", "ape", "m4a"] INTERFACE = None +FOLDER_PERMISSIONS = None def CheckSection(sec): """ Check if INI section exists, if not create it """ @@ -163,7 +164,7 @@ def initialize(): ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE + NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS if __INITIALIZED__: return False @@ -241,6 +242,7 @@ def initialize(): LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '') INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default') + FOLDER_PERMISSIONS = check_setting_str(CFG, 'General', 'folder_permissions', '0755') if not LOG_DIR: LOG_DIR = os.path.join(DATA_DIR, 'logs') @@ -408,6 +410,7 @@ def config_write(): new_config['General']['lastfm_username'] = LASTFM_USERNAME new_config['General']['interface'] = INTERFACE + new_config['General']['folder_permissions'] = FOLDER_PERMISSIONS new_config.write() diff --git a/headphones/exceptions.py b/headphones/exceptions.py new file mode 100644 index 00000000..d591edff --- /dev/null +++ b/headphones/exceptions.py @@ -0,0 +1,26 @@ +def ex(e): + """ + Returns a string from the exception text if it exists. + """ + + # sanity check + if not e.args or not e.args[0]: + return "" + + e_message = e.args[0] + + # if fixStupidEncodings doesn't fix it then maybe it's not a string, in which case we'll try printing it anyway + if not e_message: + try: + e_message = str(e.args[0]) + except: + e_message = "" + + return e_message + + +class HeadphonesException(Exception): + "Generic Headphones Exception - should never be thrown, only subclassed" + +class NewzbinAPIThrottled(HeadphonesException): + "Newzbin has throttled us, deal with it" diff --git a/headphones/mb.py b/headphones/mb.py index 1d6b20cd..1bf35308 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -24,7 +24,7 @@ def findArtist(name, limit=1): attempt = 0 artistResults = None - chars = set('!?') + chars = set('!?*') if any((c in chars) for c in name): name = '"'+name+'"' @@ -47,7 +47,7 @@ def findArtist(name, limit=1): if result.artist.name != result.artist.getUniqueName() and limit == 1: - logger.info('Found an artist with a disambiguation: %s - doing an album based search' % name) + logger.debug('Found an artist with a disambiguation: %s - doing an album based search' % name) artistdict = findArtistbyAlbum(name) if not artistdict: diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index d8b8cd14..2b1f87c9 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -111,11 +111,6 @@ def verify(albumid, albumpath): tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] - - try: - albumpath = str(albumpath) - except UnicodeEncodeError: - albumpath = unicode(albumpath).encode('unicode_escape') for r,d,f in os.walk(albumpath): for files in f: @@ -203,6 +198,10 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) if headphones.MOVE_FILES and headphones.DESTINATION_DIR: albumpath = moveFiles(albumpath, release, tracks) + if headphones.MOVE_FILES and not headphones.DESTINATION_DIR: + logger.error('No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to') + pass + myDB = db.DBConnection() # There's gotta be a better way to update the have tracks - sqlite @@ -306,10 +305,12 @@ def moveFiles(albumpath, release, tracks): # Chmod the directories using the folder_format (script courtesy of premiso!) folder_list = folder.split('/') + + temp_f = os.path.join(headphones.DESTINATION_DIR); for f in folder_list: temp_f = os.path.join(temp_f, f) - os.chmod(temp_f, 0755) + os.chmod(temp_f, int(headphones.FOLDER_PERMISSIONS, 8)) except Exception, e: logger.error('Could not create folder for %s. Not moving: %s' % (release['AlbumTitle'], e)) @@ -442,6 +443,7 @@ def renameUnprocessedFolder(albumpath): def forcePostProcess(): if not headphones.DOWNLOAD_DIR: + logger.error('No DOWNLOAD_DIR has been set. Set "Music Download Directory:" to your SAB download directory on the settings page.') return else: download_dir = headphones.DOWNLOAD_DIR @@ -459,8 +461,11 @@ def forcePostProcess(): # Parse the folder names to get artist album info for folder in folders: + + folder = unicode(folder) - albumpath = unicode(os.path.join(download_dir, folder)) + albumpath = os.path.join(download_dir, folder) + try: name, album, year = helpers.extract_data(folder) except: @@ -482,7 +487,6 @@ def forcePostProcess(): logger.error('Can not get release information for this album') continue if rgid: - rgid = unicode(rgid) verify(rgid, albumpath) \ No newline at end of file diff --git a/headphones/searcher.py b/headphones/searcher.py index 00cbfeb0..c83f975d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -4,7 +4,7 @@ from xml.dom import minidom from xml.parsers.expat import ExpatError import os, re, time -import headphones +import headphones, exceptions from headphones import logger, db, helpers, classes, sab class NewzbinDownloader(urllib.FancyURLopener): @@ -23,6 +23,10 @@ class NewzbinDownloader(urllib.FancyURLopener): rtext = str(headers.getheader('X-DNZB-RText')) result = re.search("wait (\d+) seconds", rtext) + logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds") + time.sleep(int(result.group(1))) + raise exceptions.NewzbinAPIThrottled() + elif newzbinErrCode == 401: logger.info("Newzbin error 401") #raise exceptions.AuthException("Newzbin username or password incorrect") @@ -31,12 +35,6 @@ class NewzbinDownloader(urllib.FancyURLopener): #raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs") logger.info("Newzbin error 402") - logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + "seconds") - - time.sleep(int(result.group(1))) - - #raise exceptions.NewzbinAPIThrottled() - #this should be in a class somewhere def getNewzbinURL(url): @@ -72,7 +70,7 @@ def searchNZB(albumid=None, new=False): except TypeError: year = '' - dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':''} + dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':''} cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic)) cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic)) @@ -296,7 +294,11 @@ def searchNZB(albumid=None, new=False): "q": term } searchURL = providerurl + "search/?%s" % urllib.urlencode(params) - data = getNewzbinURL(searchURL) + try: + data = getNewzbinURL(searchURL) + except exceptions.NewzbinAPIThrottled: + #try again if we were throttled + data = getNewzbinURL(searchURL) if data: logger.info(u'Parsing results from %s' % (searchURL, providerurl)) @@ -306,7 +308,7 @@ def searchNZB(albumid=None, new=False): items = d.getElementsByTagName("item") except ExpatError: logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are') - items = None + items = [] if len(items): @@ -343,7 +345,7 @@ def searchNZB(albumid=None, new=False): #when looking for "Foo - Foo" we don't want "Foobar" #this should be less of an issue when it isn't a self-titled album so we'll only check vs artist if len(resultlist): - resultlist[:] = [result for result in resultlist if verifyresult(result[0], artistterm)] + resultlist[:] = [result for result in resultlist if verifyresult(result[0], artistterm, term)] if len(resultlist): @@ -431,26 +433,36 @@ def searchNZB(albumid=None, new=False): myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) -def verifyresult(title, term): +def verifyresult(title, artistterm, term): title = re.sub('[\.\-\/\_]', ' ', title) - if term == 'Various Artists': - return True - - if not re.search('^' + re.escape(term), title, re.IGNORECASE): - logger.info("Removed from results: " + title + " (artist not at string start).") - return False - elif re.search(re.escape(term) + '\w', title, re.IGNORECASE | re.UNICODE): - logger.info("Removed from results: " + title + " (post substring result).") - return False - elif re.search('\w' + re.escape(term), title, re.IGNORECASE | re.UNICODE): - logger.info("Removed from results: " + title + " (pre substring result).") - return False - else: - return True + if artistterm != 'Various Artists': + + if not re.search('^' + re.escape(artistterm), title, re.IGNORECASE): + logger.info("Removed from results: " + title + " (artist not at string start).") + return False + elif re.search(re.escape(artistterm) + '\w', title, re.IGNORECASE | re.UNICODE): + logger.info("Removed from results: " + title + " (post substring result).") + return False + elif re.search('\w' + re.escape(artistterm), title, re.IGNORECASE | re.UNICODE): + logger.info("Removed from results: " + title + " (pre substring result).") + return False + + #another attempt to weed out substrings. We don't want "Vol III" when we were looking for "Vol II" + tokens = re.split('\W', term, re.IGNORECASE | re.UNICODE) + for token in tokens: + if token == 'Various' or token == 'Artists' or token == 'VA': + continue + if not re.search('(?:\W|^)+' + token + '(?:\W|$)+', title, re.IGNORECASE | re.UNICODE): + logger.info("Removed from results: " + title + " (missing token: " + token + ")") + return False + return True def getresultNZB(result): + + nzb = None + if result[3] == 'newzbin': params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}) url = "https://www.newzbin.com" + "/api/dnzb/" @@ -459,6 +471,12 @@ def getresultNZB(result): nzb = urllib.urlopen(url, data=params).read() except urllib2.URLError, e: logger.warn('Error fetching nzb from url: %s. Error: %s' % (url, e)) + except exceptions.NewzbinAPIThrottled: + #TODO: This has created a potentially infinite loop? As long as they keep throttling we keep trying. + logger.info("Done waiting for Newzbin API throttle limit, starting downloads again") + getresultNZB(result) + except AttributeError: + logger.warn("AttributeError in getresultNZB.") else: try: nzb = urllib2.urlopen(result[2], timeout=30).read() @@ -469,7 +487,7 @@ def getresultNZB(result): def preprocess(resultlist): if not headphones.USENET_RETENTION: - usenet_retention = 1000 + usenet_retention = 2000 else: usenet_retention = int(headphones.USENET_RETENTION) diff --git a/headphones/webserve.py b/headphones/webserve.py index 7be70060..c03c2790 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -183,6 +183,34 @@ class WebInterface(object): return serve_template(templatename="manage.html", title="Manage") manage.exposed = True + def manageArtists(self): + myDB = db.DBConnection() + artists = myDB.select('SELECT * from artists order by ArtistSortName COLLATE NOCASE') + return serve_template(templatename="manageartists.html", title="Manage Artists", artists=artists) + manageArtists.exposed = True + + def markArtists(self, action=None, **args): + myDB = db.DBConnection() + for ArtistID in args: + if action == 'delete': + myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + elif action == 'pause': + controlValueDict = {'ArtistID': ArtistID} + newValueDict = {'Status': 'Paused'} + myDB.upsert("artists", newValueDict, controlValueDict) + elif action == 'resume': + controlValueDict = {'ArtistID': ArtistID} + newValueDict = {'Status': 'Active'} + myDB.upsert("artists", newValueDict, controlValueDict) + else: + # These may and probably will collide - need to make a better way to queue musicbrainz queries + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() + time.sleep(30) + raise cherrypy.HTTPRedirect("home") + markArtists.exposed = True + def importLastFM(self, username): headphones.LASTFM_USERNAME = username headphones.config_write()