diff --git a/data/css/style.css b/data/css/style.css index 552c9a5b..15b4af99 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -157,35 +157,45 @@ div#paddingheader { padding-top: 48px; font-size: 24px; font-weight: bold; text- div#nopaddingheader { font-size: 24px; font-weight: bold; text-align: center; } table#album_table { background-color: white; } -table#album_table th#select { vertical-align: middle; text-align: left; min-width: 25px; } +table#album_table th#select { vertical-align: middle; text-align: left; min-width: 10px; } table#album_table th#albumart { text-align: left; min-width: 50px; } table#album_table th#albumname { text-align: center; min-width: 150px; } -table#album_table th#reldate { width: 175px; text-align: center; min-width: 100px; } -table#album_table th#status { width: 175px; text-align: center; min-width: 100px; } +table#album_table th#reldate { width: 175px; text-align: center; min-width: 70px; } +table#album_table th#status { width: 175px; text-align: center; min-width: 80px; } table#album_table th#type { width: 175px; text-align: center; min-width: 100px; } +table#album_table th#bitrate { text-align: center; min-width: 60px; } table#album_table td#select { vertical-align: middle; text-align: left; } table#album_table td#albumart { vertical-align: middle; text-align: left; } table#album_table td#albumname { vertical-align: middle; text-align: center; } table#album_table td#reldate { vertical-align: middle; text-align: center; } -table#album_table td#status { vertical-align: middle; text-align: center; } +table#album_table td#status { vertical-align: middle; text-align: center; font-size: 13px; } table#album_table td#type { vertical-align: middle; text-align: center; } table#album_table td#have { vertical-align: middle; } +table#album_table td#bitrate { vertical-align: middle; text-align: center; font-size: 13px; } img.albumArt { float: left; padding-right: 5px; } div#albumheader { padding-top: 48px; height: 200px; } -div#track_wrapper { padding-top: 20px; text-align: center; font-size: 16px; } +div#track_wrapper { margin-left: -50px; padding-top: 20px; font-size: 16px; width: 100%; } -table#track_table th#number { text-align: right; min-width: 20px; } +table#track_table th#number { text-align: right; min-width: 10px; } table#track_table th#name { text-align: center; min-width: 350px; } table#track_table th#duration { width: 175px; text-align: center; min-width: 100px; } -table#track_table th#have { width: 175px; text-align: center; min-width: 100px; } +table#track_table th#location { text-align: center; width: 250px; } +table#track_table th#bitrate { text-align: center; min-width: 75px; } table#track_table td#number { vertical-align: middle; text-align: right; } -table#track_table td#name { vertical-align: middle; text-align: center; } +table#track_table td#name { vertical-align: middle; text-align: center; font-size: 15px; } table#track_table td#duration { vertical-align: middle; text-align: center; } -table#track_table td#have { vertical-align: middle; text-align: center; } +table#track_table td#location { vertical-align: middle; text-align: center; font-size: 11px; } +table#track_table td#bitrate { vertical-align: middle; text-align: center; font-size: 12px; } -table#history_table { background-color: white; width: 100%; } +table#history_table { background-color: white; width: 100%; font-size: 13px; } + +table#history_table td#dateadded { vertical-align: middle; text-align: center; min-width: 150px; font-size: 14px; } +table#history_table td#filename { vertical-align: middle; text-align: center; min-width: 100px; font-size: 15px; } +table#history_table td#size { vertical-align: middle; text-align: center; min-width: 75px; font-size: 14px; } +table#history_table td#status { vertical-align: middle; text-align: center; font-size: 14px; } +table#history_table td#action { vertical-align: middle; text-align: center; font-size: 14px; } table#log_table { background-color: white; } diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index da287746..ca2e9ce3 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -51,35 +51,55 @@ # Track Title Duration - + Local File + Bit Rate - <% - i = 0 - %> %for track in tracks: <% - i += 1 - have = myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ? AND TrackTitle like ?', [track['ArtistName'], track['AlbumTitle'], track['TrackTitle']]) - if len(have): + if track['Location']: grade = 'A' - check = 'checkmark' + location = track['Location'] else: - grade = 'Z' - check = '' + grade = 'X' + location = '' + + if track['BitRate']: + bitrate = str(track['BitRate']/1000) + ' kbps' + else: + bitrate = '' + try: trackduration = helpers.convert_milliseconds(track['TrackDuration']) except: trackduration = 'n/a' %> - ${i} + ${track['TrackNumber']} ${track['TrackTitle']} ${trackduration} - ${check} + ${location} + ${bitrate} %endfor + <% + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + %> + %if unmatched: + %for track in unmatched: + <% + duration = helpers.convert_seconds(float(track['TrackLength'])) + %> + + ${track['TrackNumber']} + ${track['TrackTitle']} + ${duration} + ${track['Location']} + ${int(track['BitRate'])/1000} kbps + + %endfor + %endif @@ -97,6 +117,7 @@ { $('#track_table').dataTable( { + "aaSorting": [], "bFilter": false, "bInfo": false, "bPaginate": false diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 4b296eda..7c82f4fc 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -41,11 +41,12 @@ - Album Name - Release Date - Release Type + Name + Date + Type Status Have + Bitrate @@ -62,7 +63,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks @@ -71,6 +72,12 @@ except (ZeroDivisionError, TypeError): percent = 0 totaltracks = '?' + + avgbitrate = myDB.action("SELECT AVG(BitRate) FROM tracks WHERE AlbumID=?", [album['AlbumID']]).fetchone()[0] + if avgbitrate: + bitrate = str(int(avgbitrate)/1000) + ' kbps' + else: + bitrate = '' %> @@ -89,6 +96,7 @@ %endif
${havetracks}/${totaltracks}
+ ${bitrate} %endfor @@ -114,7 +122,8 @@ null, null, null, - { "sType": "title-numeric"} + { "sType": "title-numeric"}, + null ], "oLanguage": { "sLengthMenu":"Show _MENU_ albums per page", diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 5f4aa2da..b392e559 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -194,7 +194,7 @@ Highest Quality including Lossless
Lossless Only
Preferred Bitrate: - kbps
+ kbps
Auto-Detect Preferred Bitrate diff --git a/data/interfaces/default/history.html b/data/interfaces/default/history.html index b79a43af..f8f60112 100644 --- a/data/interfaces/default/history.html +++ b/data/interfaces/default/history.html @@ -42,7 +42,7 @@ %> ${item['DateAdded']} - ${item['Title']} + ${item['Title']} [nzb][album page] ${helpers.bytes_to_mb(item['Size'])} ${item['Status']} [retry][new] diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 3e6f7452..cf5a476a 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -1,11 +1,15 @@ <%inherit file="base.html" /> <%! import headphones + from headphones.helpers import checked %> <%def name="headerIncludes()">
@@ -30,6 +34,9 @@ %endif +
+

Automatically add new artists

+

diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html new file mode 100644 index 00000000..bea493b2 --- /dev/null +++ b/data/interfaces/default/managenew.html @@ -0,0 +1,52 @@ +<%inherit file="base.html" /> +<%! + import headphones +%> +<%def name="body()"> +
+

Manage New Artists

+

Scan Music Library

+
+
+

+ Add selected artists + +

+ + + + + + + + + %for artist in headphones.NEW_ARTISTS: + + + + + %endfor + +
Artist Name
${artist}
+
+ + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + \ No newline at end of file diff --git a/data/interfaces/remix/album.html b/data/interfaces/remix/album.html index da287746..355dd9b3 100644 --- a/data/interfaces/remix/album.html +++ b/data/interfaces/remix/album.html @@ -51,33 +51,36 @@ # Track Title Duration - + Local File + Bit Rate - <% - i = 0 - %> %for track in tracks: <% - i += 1 - have = myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ? AND TrackTitle like ?', [track['ArtistName'], track['AlbumTitle'], track['TrackTitle']]) - if len(have): + if track['Location']: grade = 'A' - check = 'checkmark' + location = track['Location'] else: grade = 'Z' - check = '' + location = '' + + if track['BitRate']: + bitrate = str(track['BitRate']/1000) + ' kbps' + else: + bitrate = '' + try: trackduration = helpers.convert_milliseconds(track['TrackDuration']) except: trackduration = 'n/a' %> - ${i} + ${track['TrackNumber']} ${track['TrackTitle']} ${trackduration} - ${check} + ${location} + ${bitrate} %endfor diff --git a/headphones/__init__.py b/headphones/__init__.py index 7b1fc2fb..dadff537 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -11,7 +11,7 @@ from lib.configobj import ConfigObj import cherrypy -from headphones import updater, searcher, importer, versioncheck, logger, postprocessor, version, sab +from headphones import updater, searcher, importer, versioncheck, logger, postprocessor, version, sab, librarysync from headphones.common import * FULL_PATH = None @@ -62,6 +62,8 @@ PATH_TO_XML = None PREFERRED_QUALITY = None PREFERRED_BITRATE = None DETECT_BITRATE = False +ADD_ARTISTS = False +NEW_ARTISTS = [] CORRECT_METADATA = False MOVE_FILES = False RENAME_FILES = False @@ -160,7 +162,7 @@ def initialize(): global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \ HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \ - CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ + ADD_ARTISTS, CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ @@ -199,6 +201,7 @@ def initialize(): PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '') DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) + ADD_ARTISTS = bool(check_setting_int(CFG, 'General', 'auto_add_artists', 1)) CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) @@ -363,6 +366,7 @@ def config_write(): new_config['General']['preferred_quality'] = PREFERRED_QUALITY new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) + new_config['General']['auto_add_artists'] = int(ADD_ARTISTS) new_config['General']['correct_metadata'] = int(CORRECT_METADATA) new_config['General']['move_files'] = int(MOVE_FILES) new_config['General']['rename_files'] = int(RENAME_FILES) @@ -425,7 +429,7 @@ def start(): SCHED.add_cron_job(updater.dbUpdate, hour=4, minute=0, second=0) SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) - SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL) + SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL) SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) @@ -439,9 +443,9 @@ def dbcheck(): c=conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS descriptions (ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT)') c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') @@ -489,7 +493,32 @@ def dbcheck(): try: c.execute('SELECT FolderName from snatched') except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + + try: + c.execute('SELECT Location from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + + try: + c.execute('SELECT Location from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN Location TEXT') + + try: + c.execute('SELECT BitRate from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') + + try: + c.execute('SELECT CleanName from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') + + try: + c.execute('SELECT CleanName from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') conn.commit() c.close() diff --git a/headphones/helpers.py b/headphones/helpers.py index 4ecc6941..9428f319 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -84,6 +84,16 @@ def convert_milliseconds(ms): return minutes +def convert_seconds(s): + + gmtime = time.gmtime(s) + if s > 3600: + minutes = time.strftime("%H:%M:%S", gmtime) + else: + minutes = time.strftime("%M:%S", gmtime) + + return minutes + def today(): today = datetime.date.today() yyyymmdd = datetime.date.isoformat(today) @@ -104,6 +114,13 @@ def replace_all(text, dic): text = text.replace(i, j) return text +def cleanName(string): + + pass1 = latinToAscii(string).lower() + out_string = re.sub('[\.\-\/\!\@\#\$\%\^\&\*\(\)\+\-\"\'\,\;\:\[\]\{\}\<\>\=\_]', '', pass1).encode('utf-8') + + return out_string + def extract_data(s): from headphones import logger @@ -143,4 +160,35 @@ def extract_logline(s): message = match.group("message") return (timestamp, level, thread, message) else: - return None \ No newline at end of file + return None + +def extract_song_data(s): + + #headphones default format + music_dir = headphones.MUSIC_DIR + folder_format = headphones.FOLDER_FORMAT + file_format = headphones.FILE_FORMAT + + full_format = os.path.join(headphones.MUSIC_DIR) + pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\[(?P.*?)\]', re.VERBOSE) + match = pattern.match(s) + + if match: + name = match.group("name") + album = match.group("album") + year = match.group("year") + return (name, album, year) + else: + logger.info("Couldn't parse " + s + " into a valid default format") + + #newzbin default format + pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\((?P\d+?\))', re.VERBOSE) + match = pattern.match(s) + if match: + name = match.group("name") + album = match.group("album") + year = match.group("year") + return (name, album, year) + else: + logger.info("Couldn't parse " + s + " into a valid Newbin format") + return (name, album, year) \ No newline at end of file diff --git a/headphones/importer.py b/headphones/importer.py index 11dc15d2..893115c5 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -7,89 +7,6 @@ import headphones from headphones import logger, helpers, db, mb, albumart, lastfm various_artists_mbid = '89ad4ac3-39f7-470e-963a-56509c546377' - -def scanMusic(dir=None): - - if not dir: - dir = headphones.MUSIC_DIR - - try: - dir = str(dir) - except UnicodeEncodeError: - dir = unicode(dir).encode('unicode_escape') - - logger.info('Starting Music Scan with directory: %s' % dir) - - results = [] - - for r,d,f in os.walk(dir): - for files in f: - if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): - results.append(os.path.join(r, files)) - - logger.info(u'%i music files found. Reading metadata....' % len(results)) - - if results: - - myDB = db.DBConnection() - myDB.action('''DELETE from have''') - - for song in results: - try: - f = MediaFile(song) - except: - logger.warn('Could not read file: %s' % song) - continue - else: - if f.albumartist: - artist = f.albumartist - elif f.artist: - artist = f.artist - else: - continue - - if not f.album: - album = None - else: - album = f.album - - myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?)', [artist, album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid]) - - # Get the average bitrate if the option is selected - if headphones.DETECT_BITRATE: - try: - avgbitrate = myDB.action("SELECT AVG(BitRate) FROM have").fetchone()[0] - headphones.PREFERRED_BITRATE = int(avgbitrate)/1000 - - except Exception, e: - logger.error('Error detecting preferred bitrate:' + str(e)) - - artistlist = myDB.action('SELECT DISTINCT ArtistName FROM have').fetchall() - logger.info(u"Preparing to import %i artists" % len(artistlist)) - - artistlist_to_mbids(artistlist) - -def itunesImport(pathtoxml): - - if os.path.splitext(pathtoxml)[1] == '.xml': - logger.info(u"Loading xml file from"+ pathtoxml) - pl = XMLLibraryParser(pathtoxml) - l = Library(pl.dictionary) - lst = [] - for song in l.songs: - lst.append(song.artist) - rawlist = {}.fromkeys(lst).keys() - artistlist = [f for f in rawlist if f != None] - - else: - rawlist = os.listdir(pathtoxml) - logger.info(u"Loading artists from directory:" +pathtoxml) - exclude = ['.ds_store', 'various artists', 'untitled folder', 'va'] - artistlist = [f for f in rawlist if f.lower() not in exclude] - - logger.info('Starting directory/xml import...') - artistlist_to_mbids(artistlist) - def is_exists(artistid): @@ -99,19 +16,23 @@ def is_exists(artistid): artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): - logger.debug(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") + logger.info(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") return True else: return False -def artistlist_to_mbids(artistlist): +def artistlist_to_mbids(artistlist, forced=False): for artist in artistlist: - - results = mb.findArtist(artist['ArtistName'], limit=1) + + if forced: + artist = unicode(artist, 'utf-8') + + results = mb.findArtist(artist, limit=1) if not results: + logger.info('No results found for: %' % artist) continue try: @@ -124,16 +45,12 @@ def artistlist_to_mbids(artistlist): # Add to database if it doesn't exist if artistid != various_artists_mbid and not is_exists(artistid): addArtisttoDB(artistid) - - # Update track count regardless of whether it already exists - if artistid != various_artists_mbid: - + + # Just update the tracks if it does + else: myDB = db.DBConnection() - havetracks = len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) - - controlValueDict = {"ArtistID": artistid} - newValueDict = {"HaveTracks": havetracks} - myDB.upsert("artists", newValueDict, controlValueDict) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) # Update the similar artist tag cloud: logger.info('Updating artist information from Last.fm') @@ -228,9 +145,10 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.action('DELETE from albums WHERE AlbumID=?', [release['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [release['releaseid']]) - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) for track in release_dict['tracks']: + cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) + controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, @@ -239,14 +157,27 @@ def addArtisttoDB(artistid, extrasonly=False): "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], - "TrackNumber": track['number'] + "TrackNumber": track['number'], + "CleanName": cleanname } - + + match = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + myDB.upsert("tracks", newValueDict, controlValueDict) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + controlValueDict = {"ArtistID": artistid} if latestalbum: @@ -254,9 +185,12 @@ def addArtisttoDB(artistid, extrasonly=False): "LatestAlbum": latestalbum['AlbumTitle'], "ReleaseDate": latestalbum['ReleaseDate'], "AlbumID": latestalbum['AlbumID'], - "TotalTracks": totaltracks} + "TotalTracks": totaltracks, + "HaveTracks": havetracks} else: - newValueDict = {"Status": "Active"} + newValueDict = {"Status": "Active", + "TotalTracks": totaltracks, + "HaveTracks": havetracks} myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Updating complete for: " + artist['artist_name']) @@ -339,6 +273,8 @@ def addReleaseById(rid): for track in release_dict['tracks']: + cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) + controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], @@ -347,8 +283,22 @@ def addReleaseById(rid): "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], - "TrackNumber": track['number'] + "TrackNumber": track['number'], + "CleanName": cleanname } + + match = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() + + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 814a35e8..f46ea2f7 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -2,6 +2,7 @@ import urllib from xml.dom import minidom from collections import defaultdict import random +import time import headphones from headphones import db, logger @@ -19,7 +20,16 @@ def getSimilar(): for result in results[:12]: url = 'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&mbid=%s&api_key=%s' % (result['ArtistID'], api_key) - data = urllib.urlopen(url).read() + + try: + data = urllib.urlopen(url).read() + except: + time.sleep(1) + continue + + len(data) < 200: + continue + d = minidom.parseString(data) node = d.documentElement artists = d.getElementsByTagName("artist") diff --git a/headphones/librarysync.py b/headphones/librarysync.py new file mode 100644 index 00000000..37cd10cb --- /dev/null +++ b/headphones/librarysync.py @@ -0,0 +1,185 @@ +import os +import glob + +from lib.beets.mediafile import MediaFile + +import headphones +from headphones import db, logger, helpers, importer + +def libraryScan(dir=None): + + if not dir: + dir = headphones.MUSIC_DIR + + try: + dir = str(dir) + except UnicodeEncodeError: + dir = unicode(dir).encode('unicode_escape') + + logger.info('Scanning music directory: %s' % dir) + + new_artists = [] + bitrates = [] + + myDB = db.DBConnection() + myDB.action('''DELETE from have''') + + for r,d,f in os.walk(dir): + for files in f: + # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): + + file = unicode(os.path.join(r, files), "utf-8") + + # Try to read the metadata + try: + f = MediaFile(file) + except: + logger.error('Cannot read file: ' + file) + continue + + # Grab the bitrates for the auto detect bit rate option + if f.bitrate: + bitrates.append(f.bitrate) + + # Try to match on metadata first, starting with the track id + if f.mb_trackid: + + # Wondering if theres a better way to do this -> do one thing if the row exists, + # do something else if it doesn't + track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() + + if track: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + continue + + # Try to find a match based on artist/album/tracktitle + if f.albumartist: + f_artist = f.albumartist + elif f.artist: + f_artist = f.artist + else: + continue + + if f_artist and f.album and f.title: + + track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() + + if not track: + track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() + + if track: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + continue + + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release + new_artists.append(f_artist) + + # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database + myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title)]) + + # Now check empty file paths to see if we can find a match based on their folder format + tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') + for track in tracks: + + release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() + + try: + year = release['ReleaseDate'][:4] + except TypeError: + year = '' + + artist = release['ArtistName'].replace('/', '_') + album = release['AlbumTitle'].replace('/', '_') + + if release['ArtistName'].startswith('The '): + sortname = release['ArtistName'][4:] + else: + sortname = release['ArtistName'] + + if sortname.isdigit(): + firstchar = '0-9' + else: + firstchar = sortname[0] + + + albumvalues = { 'artist': artist, + 'album': album, + 'year': year, + 'first': firstchar, + } + + + folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) + folder = folder.replace('./', '_/').replace(':','_').replace('?','_') + + if folder.endswith('.'): + folder = folder.replace(folder[len(folder)-1], '_') + + if not track['TrackNumber']: + tracknumber = '' + else: + tracknumber = '%02d' % track['TrackNumber'] + + trackvalues = { 'tracknumber': tracknumber, + 'title': track['TrackTitle'], + 'artist': release['ArtistName'], + 'album': release['AlbumTitle'], + 'year': year + } + + new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*' + + new_file_name = new_file_name.replace('?','_').replace(':', '_') + + full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)) + + match = glob.glob(full_path_to_file) + + if match: + + myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [match[0], track['TrackID']]) + myDB.action('DELETE from have WHERE Location=?', [match[0]]) + + # Try to insert the appropriate track id so we don't have to keep doing this + try: + f = MediaFile(match[0]) + f.mb_trackid = track['TrackID'] + f.save() + myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) + logger.debug('Wrote mbid to track: %s' % match[0]) + except: + logger.error('Error embedding track id into: %s' % match[0]) + continue + + # Clean up bad filepaths + tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') + + for track in tracks: + if not os.path.isfile(track['Location']): + myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [None, track['TrackID']]) + + logger.info('Completed scanning of directory: %s. Updating track counts' % dir) + + # Clean up the new artist list + unique_artists = {}.fromkeys(new_artists).keys() + current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') + + artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] + + # Update track counts + for artist in current_artists: + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) + + logger.info('Found %i new artists' % len(artist_list)) + + if headphones.ADD_ARTISTS: + logger.info('Importing %i new artists' % len(artist_list)) + importer.artistlist_to_mbids(artist_list) + else: + logger.info('To add these artists, go to Manage->Manage New Artists') + headphones.NEW_ARTISTS = artist_list + + if headphones.DETECT_BITRATE: + headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 \ No newline at end of file diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 8bf96692..2efb6942 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -450,7 +450,7 @@ def updateHave(albumpath): else: continue - myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?)', [artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid]) + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song, f.bitrate, artist, f.album, f.title]) def renameUnprocessedFolder(albumpath): diff --git a/headphones/webserve.py b/headphones/webserve.py index 399d22ee..d5fe8006 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -10,7 +10,7 @@ import threading import headphones -from headphones import logger, searcher, db, importer, mb, lastfm +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync from headphones.helpers import checked, radio @@ -148,7 +148,12 @@ class WebInterface(object): else: raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - + + def addArtists(self, **args): + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + time.sleep(5) + raise cherrypy.HTTPRedirect("home") + addArtists.exposed = True def queueAlbum(self, AlbumID, ArtistID=None, new=False, redirect=None): logger.info(u"Marking album: " + AlbumID + "as wanted...") @@ -189,6 +194,10 @@ class WebInterface(object): return serve_template(templatename="manageartists.html", title="Manage Artists", artists=artists) manageArtists.exposed = True + def manageNew(self): + return serve_template(templatename="managenew.html", title="Manage New Artists") + manageNew.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() for ArtistID in args: @@ -227,15 +236,19 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") importItunes.exposed = True - def musicScan(self, path): + def musicScan(self, path, redirect=None, autoadd=0): + headphones.ADD_ARTISTS = autoadd headphones.MUSIC_DIR = path headphones.config_write() try: - threading.Thread(target=importer.scanMusic, args=[path]).start() + threading.Thread(target=librarysync.libraryScan).start() except Exception, e: logger.error('Unable to complete the scan: %s' % e) time.sleep(10) - raise cherrypy.HTTPRedirect("home") + if redirect: + raise cherrypy.HTTPRedirect(redirect) + else: + raise cherrypy.HTTPRedirect("home") musicScan.exposed = True def forceUpdate(self):