From 98427c23ff6e07d089ae00a36f994e70cafe5185 Mon Sep 17 00:00:00 2001 From: Remy Date: Sun, 14 Aug 2011 18:16:48 -0700 Subject: [PATCH 1/9] Added Location column to track database and album template --- data/interfaces/default/album.html | 10 ++++------ headphones/__init__.py | 9 +++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index da287746..da7fc4f8 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -51,7 +51,7 @@ # Track Title Duration - + @@ -61,13 +61,11 @@ %for track in tracks: <% i += 1 - have = myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ? AND TrackTitle like ?', [track['ArtistName'], track['AlbumTitle'], track['TrackTitle']]) - if len(have): + if track['Location']: grade = 'A' - check = 'checkmark' else: grade = 'Z' - check = '' + try: trackduration = helpers.convert_milliseconds(track['TrackDuration']) except: @@ -77,7 +75,7 @@ ${i} ${track['TrackTitle']} ${trackduration} - ${check} + ${track['Location']} %endfor diff --git a/headphones/__init__.py b/headphones/__init__.py index 7b1fc2fb..ca776d30 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -439,7 +439,7 @@ def dbcheck(): c=conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') @@ -489,7 +489,12 @@ def dbcheck(): try: c.execute('SELECT FolderName from snatched') except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + + try: + c.execute('SELECT Location from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') conn.commit() c.close() From 22aef717d153c7277270d6e6273675794171f8fe Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 15 Aug 2011 02:32:32 -0700 Subject: [PATCH 2/9] Added librarysync, added bitrate and location to db and album template --- data/css/style.css | 8 +- data/interfaces/default/album.html | 19 ++-- data/interfaces/remix/album.html | 25 +++-- headphones/__init__.py | 11 +- headphones/helpers.py | 33 +++++- headphones/librarysync.py | 160 +++++++++++++++++++++++++++++ 6 files changed, 231 insertions(+), 25 deletions(-) create mode 100644 headphones/librarysync.py diff --git a/data/css/style.css b/data/css/style.css index 552c9a5b..b7023a7f 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -173,17 +173,19 @@ table#album_table td#have { vertical-align: middle; } img.albumArt { float: left; padding-right: 5px; } div#albumheader { padding-top: 48px; height: 200px; } -div#track_wrapper { padding-top: 20px; text-align: center; font-size: 16px; } +div#track_wrapper { margin-left: -50px; padding-top: 20px; font-size: 16px; width: 100%; } table#track_table th#number { text-align: right; min-width: 20px; } table#track_table th#name { text-align: center; min-width: 350px; } table#track_table th#duration { width: 175px; text-align: center; min-width: 100px; } -table#track_table th#have { width: 175px; text-align: center; min-width: 100px; } +table#track_table th#location { text-align: center; min-width: 200px; } +table#track_table th#bitrate { text-align: center; min-width: 75px; } table#track_table td#number { vertical-align: middle; text-align: right; } table#track_table td#name { vertical-align: middle; text-align: center; } table#track_table td#duration { vertical-align: middle; text-align: center; } -table#track_table td#have { vertical-align: middle; text-align: center; } +table#track_table td#location { vertical-align: middle; text-align: center; font-size: 11px; } +table#track_table td#location { vertical-align: middle; text-align: center; } table#history_table { background-color: white; width: 100%; } diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index da7fc4f8..355dd9b3 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -51,20 +51,24 @@ # Track Title Duration - + Local File + Bit Rate - <% - i = 0 - %> %for track in tracks: <% - i += 1 if track['Location']: grade = 'A' + location = track['Location'] else: grade = 'Z' + location = '' + + if track['BitRate']: + bitrate = str(track['BitRate']/1000) + ' kbps' + else: + bitrate = '' try: trackduration = helpers.convert_milliseconds(track['TrackDuration']) @@ -72,10 +76,11 @@ trackduration = 'n/a' %> - ${i} + ${track['TrackNumber']} ${track['TrackTitle']} ${trackduration} - ${track['Location']} + ${location} + ${bitrate} %endfor diff --git a/data/interfaces/remix/album.html b/data/interfaces/remix/album.html index da287746..355dd9b3 100644 --- a/data/interfaces/remix/album.html +++ b/data/interfaces/remix/album.html @@ -51,33 +51,36 @@ # Track Title Duration - + Local File + Bit Rate - <% - i = 0 - %> %for track in tracks: <% - i += 1 - have = myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ? AND TrackTitle like ?', [track['ArtistName'], track['AlbumTitle'], track['TrackTitle']]) - if len(have): + if track['Location']: grade = 'A' - check = 'checkmark' + location = track['Location'] else: grade = 'Z' - check = '' + location = '' + + if track['BitRate']: + bitrate = str(track['BitRate']/1000) + ' kbps' + else: + bitrate = '' + try: trackduration = helpers.convert_milliseconds(track['TrackDuration']) except: trackduration = 'n/a' %> - ${i} + ${track['TrackNumber']} ${track['TrackTitle']} ${trackduration} - ${check} + ${location} + ${bitrate} %endfor diff --git a/headphones/__init__.py b/headphones/__init__.py index ca776d30..1454fb79 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -11,7 +11,7 @@ from lib.configobj import ConfigObj import cherrypy -from headphones import updater, searcher, importer, versioncheck, logger, postprocessor, version, sab +from headphones import updater, searcher, importer, versioncheck, logger, postprocessor, version, sab, librarysync from headphones.common import * FULL_PATH = None @@ -439,7 +439,7 @@ def dbcheck(): c=conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') @@ -494,7 +494,12 @@ def dbcheck(): try: c.execute('SELECT Location from tracks') except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + + try: + c.execute('SELECT BitRate from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') conn.commit() c.close() diff --git a/headphones/helpers.py b/headphones/helpers.py index 4ecc6941..c3246542 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -143,4 +143,35 @@ def extract_logline(s): message = match.group("message") return (timestamp, level, thread, message) else: - return None \ No newline at end of file + return None + +def extract_song_data(s): + + #headphones default format + music_dir = headphones.MUSIC_DIR + folder_format = headphones.FOLDER_FORMAT + file_format = headphones.FILE_FORMAT + + full_format = os.path.join(headphones.MUSIC_DIR) + pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\[(?P.*?)\]', re.VERBOSE) + match = pattern.match(s) + + if match: + name = match.group("name") + album = match.group("album") + year = match.group("year") + return (name, album, year) + else: + logger.info("Couldn't parse " + s + " into a valid default format") + + #newzbin default format + pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\((?P\d+?\))', re.VERBOSE) + match = pattern.match(s) + if match: + name = match.group("name") + album = match.group("album") + year = match.group("year") + return (name, album, year) + else: + logger.info("Couldn't parse " + s + " into a valid Newbin format") + return (name, album, year) \ No newline at end of file diff --git a/headphones/librarysync.py b/headphones/librarysync.py new file mode 100644 index 00000000..8a093b04 --- /dev/null +++ b/headphones/librarysync.py @@ -0,0 +1,160 @@ +import os +import glob + +from lib.beets.mediafile import MediaFile + +import headphones +from headphones import db, logger + +def LibraryScan(): + + if not headphones.MUSIC_DIR: + return + + unmatched_files = [] + new_artists = [] + + myDB = db.DBConnection() + + for r,d,f in os.walk(headphones.MUSIC_DIR): + for files in f: + # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): + + file = unicode(os.path.join(r, files), "utf-8") + print repr(file) + # Try to read the metadata + try: + f = MediaFile(file) + except: + logger.error('Cannot read file: ' + file) + continue + + # Try to match on metadata first, + if f.mb_trackid: + print 'has track id:' + repr(f.mb_trackid) + # Wondering if theres a better way to do this -> do one thing if the row exists, + # do something else if it doesn't + track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() + + if not track: + if f.albumartist: + new_artists.append(f.albumartist) + elif f.artist: + new_artists.append(f.artist) + else: + unmatched_files.append(file) + + else: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + continue + + elif f.albumartist and f.album and f.title: + + track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f.albumartist, f.album, f.title]).fetchone() + + if not track: + new_artists.append(f.albumartist) + + else: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + + elif f.artist and f.album and f.title: + + track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f.artist, f.album, f.title]).fetchone() + + if not track: + new_artists.append(file) + else: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + + else: + continue + + # Try to parse the unmatched files based on the folder & file formats in the config + for file in unmatched_files: + # Will do later + pass + + # Now check empty file paths to see if we can find a match based on their folder format + tracks = myDB.select('SELECT * from tracks WHERE Location=NULL') + for track in tracks: + release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() + + try: + year = release['ReleaseDate'][:4] + except TypeError: + year = '' + + artist = release['ArtistName'].replace('/', '_') + album = release['AlbumTitle'].replace('/', '_') + + if release['ArtistName'].startswith('The '): + sortname = release['ArtistName'][4:] + else: + sortname = release['ArtistName'] + + if sortname.isdigit(): + firstchar = '0-9' + else: + firstchar = sortname[0] + + + albumvalues = { 'artist': artist, + 'album': album, + 'year': year, + 'first': firstchar, + } + + + folder = helpers.replace_all(headphones.FOLDER_FORMAT, albumvalues) + folder = folder.replace('./', '_/').replace(':','_').replace('?','_') + + if folder.endswith('.'): + folder = folder.replace(folder[len(folder)-1], '_') + + if not track['TrackNumber']: + tracknumber = '' + else: + tracknumber = '%02d' % track['TrackNumber'] + + trackvalues = { 'tracknumber': tracknumber, + 'title': track['TrackTitle'], + 'artist': release['ArtistName'], + 'album': release['AlbumTitle'], + 'year': year + } + + new_file_name = helpers.replace_all(headphones.FILE_FORMAT, trackvalues).replace('/','_') + '.*' + + new_file_name = new_file_name.replace('?','_').replace(':', '_') + + full_path_to_file = os.path.join(headphones.MUSIC_DIR, folder, new_file_name) + + print 'Full path to file is: ' + repr(full_path_to_file) + + if glob.glob(full_path_to_file): + myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [full_path_to_file, track['TrackID']]) + + # Try to insert the appropriate track id so we don't have to keep doing this + try: + f = MediaFile(full_path_to_file) + f.mb_trackid = track['TrackID'] + myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) + f.save() + except: + logger.error('Error embedding track id into: %s' % full_path_to_file) + + # Lastly, clean up any old paths that don't exist + tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') + for track in tracks: + if not os.path.isfile(track['Location']): + myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [None, track['TrackID']]) + + # Clean up the new artist list + unique_artists = {}.fromkeys(new_artists).keys() + current_artists = myDB.action('SELECT ArtistName from artists').fetchall() + + artist_list = [f for f in unique_artists if f not in current_artists] + + logger.info('Found %i artists to import: ' % len(artist_list)) \ No newline at end of file From 4983852b076ebcf6f8e54c205f8a289a2cbbb293 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 15 Aug 2011 19:54:11 -0700 Subject: [PATCH 3/9] Moved everything over to the new library scan - needs cleanup --- data/css/style.css | 12 ++- data/interfaces/default/album.html | 27 +++++ data/interfaces/default/artist.html | 19 +++- data/interfaces/default/history.html | 2 +- headphones/__init__.py | 29 +++++- headphones/helpers.py | 7 ++ headphones/importer.py | 142 +++++++++------------------ headphones/librarysync.py | 116 ++++++++++++---------- 8 files changed, 188 insertions(+), 166 deletions(-) diff --git a/data/css/style.css b/data/css/style.css index b7023a7f..2d3fb458 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -157,19 +157,21 @@ div#paddingheader { padding-top: 48px; font-size: 24px; font-weight: bold; text- div#nopaddingheader { font-size: 24px; font-weight: bold; text-align: center; } table#album_table { background-color: white; } -table#album_table th#select { vertical-align: middle; text-align: left; min-width: 25px; } +table#album_table th#select { vertical-align: middle; text-align: left; min-width: 10px; } table#album_table th#albumart { text-align: left; min-width: 50px; } table#album_table th#albumname { text-align: center; min-width: 150px; } -table#album_table th#reldate { width: 175px; text-align: center; min-width: 100px; } -table#album_table th#status { width: 175px; text-align: center; min-width: 100px; } +table#album_table th#reldate { width: 175px; text-align: center; min-width: 70px; } +table#album_table th#status { width: 175px; text-align: center; min-width: 80px; } table#album_table th#type { width: 175px; text-align: center; min-width: 100px; } +table#album_table th#bitrate { text-align: center; min-width: 60px; } table#album_table td#select { vertical-align: middle; text-align: left; } table#album_table td#albumart { vertical-align: middle; text-align: left; } table#album_table td#albumname { vertical-align: middle; text-align: center; } table#album_table td#reldate { vertical-align: middle; text-align: center; } -table#album_table td#status { vertical-align: middle; text-align: center; } +table#album_table td#status { vertical-align: middle; text-align: center; font-size: 13px; } table#album_table td#type { vertical-align: middle; text-align: center; } table#album_table td#have { vertical-align: middle; } +table#album_table td#bitrate { vertical-align: middle; text-align: center; font-size: 13px; } img.albumArt { float: left; padding-right: 5px; } div#albumheader { padding-top: 48px; height: 200px; } @@ -185,7 +187,7 @@ table#track_table td#number { vertical-align: middle; text-align: right; } table#track_table td#name { vertical-align: middle; text-align: center; } table#track_table td#duration { vertical-align: middle; text-align: center; } table#track_table td#location { vertical-align: middle; text-align: center; font-size: 11px; } -table#track_table td#location { vertical-align: middle; text-align: center; } +table#track_table td#bitrate { vertical-align: middle; text-align: center; font-size: 12px; } table#history_table { background-color: white; width: 100%; } diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 355dd9b3..073a77e7 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -85,6 +85,33 @@ %endfor + <% + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + %> + %if unmatched: + + + + + + + + + + + + %for track in unmatched: + + + + + + + + %endfor + +
#Track TitleDurationLocal FileBit Rate
${track['TrackNumber']}${track['TrackTitle']}${track['TrackLength']}${track['Location']}${track['BitRate']}
+ %endif diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 4b296eda..0438ac4c 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -41,11 +41,12 @@ - Album Name - Release Date - Release Type + Name + Date + Type Status Have + Bitrate @@ -62,7 +63,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) try: percent = (havetracks*100.0)/totaltracks @@ -71,6 +72,12 @@ except (ZeroDivisionError, TypeError): percent = 0 totaltracks = '?' + + avgbitrate = myDB.action("SELECT AVG(BitRate) FROM tracks WHERE AlbumID=?", [album['AlbumID']]).fetchone()[0] + if avgbitrate: + bitrate = str(int(avgbitrate)/1000) + ' kbps' + else: + bitrate = '' %> @@ -89,6 +96,7 @@ %endif
${havetracks}/${totaltracks}
+ ${bitrate} %endfor @@ -114,7 +122,8 @@ null, null, null, - { "sType": "title-numeric"} + { "sType": "title-numeric"}, + null ], "oLanguage": { "sLengthMenu":"Show _MENU_ albums per page", diff --git a/data/interfaces/default/history.html b/data/interfaces/default/history.html index b79a43af..8c98db9d 100644 --- a/data/interfaces/default/history.html +++ b/data/interfaces/default/history.html @@ -42,7 +42,7 @@ %> ${item['DateAdded']} - ${item['Title']} + ${item['Title']} [album page] ${helpers.bytes_to_mb(item['Size'])} ${item['Status']} [retry][new] diff --git a/headphones/__init__.py b/headphones/__init__.py index 1454fb79..335768d1 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -62,6 +62,8 @@ PATH_TO_XML = None PREFERRED_QUALITY = None PREFERRED_BITRATE = None DETECT_BITRATE = False +ADD_ARTISTS = False +NEW_ARTISTS = None CORRECT_METADATA = False MOVE_FILES = False RENAME_FILES = False @@ -160,7 +162,7 @@ def initialize(): global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \ HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \ - CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ + ADD_ARTISTS, CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ @@ -199,6 +201,7 @@ def initialize(): PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '') DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) + ADD_ARTISTS = bool(check_setting_int(CFG, 'General', 'add_artists', 1)) CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) @@ -363,6 +366,7 @@ def config_write(): new_config['General']['preferred_quality'] = PREFERRED_QUALITY new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) + new_config['General']['add_artists'] = int(ADD_ARTISTS) new_config['General']['correct_metadata'] = int(CORRECT_METADATA) new_config['General']['move_files'] = int(MOVE_FILES) new_config['General']['rename_files'] = int(RENAME_FILES) @@ -425,10 +429,10 @@ def start(): SCHED.add_cron_job(updater.dbUpdate, hour=4, minute=0, second=0) SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) - SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL) + SCHED.add_interval_job(librarysync.libraryScan(), minutes=LIBRARYSCAN_INTERVAL) SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) - + librarysync.libraryScan() SCHED.start() started = True @@ -439,9 +443,9 @@ def dbcheck(): c=conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS descriptions (ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT)') c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') @@ -495,11 +499,26 @@ def dbcheck(): c.execute('SELECT Location from tracks') except sqlite3.OperationalError: c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + + try: + c.execute('SELECT Location from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN Location TEXT') try: c.execute('SELECT BitRate from tracks') except sqlite3.OperationalError: c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') + + try: + c.execute('SELECT CleanName from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') + + try: + c.execute('SELECT CleanName from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') conn.commit() c.close() diff --git a/headphones/helpers.py b/headphones/helpers.py index c3246542..568bb0f7 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -104,6 +104,13 @@ def replace_all(text, dic): text = text.replace(i, j) return text +def cleanName(string): + + pass1 = latinToAscii(string).lower() + out_string = re.sub('[\.\-\/\!\@\#\$\%\^\&\*\(\)\+\-\"\'\,\;\:\[\]\{\}\<\>\=\_]', '', pass1).encode('utf-8') + + return out_string + def extract_data(s): from headphones import logger diff --git a/headphones/importer.py b/headphones/importer.py index 11dc15d2..70f5701c 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -7,89 +7,6 @@ import headphones from headphones import logger, helpers, db, mb, albumart, lastfm various_artists_mbid = '89ad4ac3-39f7-470e-963a-56509c546377' - -def scanMusic(dir=None): - - if not dir: - dir = headphones.MUSIC_DIR - - try: - dir = str(dir) - except UnicodeEncodeError: - dir = unicode(dir).encode('unicode_escape') - - logger.info('Starting Music Scan with directory: %s' % dir) - - results = [] - - for r,d,f in os.walk(dir): - for files in f: - if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): - results.append(os.path.join(r, files)) - - logger.info(u'%i music files found. Reading metadata....' % len(results)) - - if results: - - myDB = db.DBConnection() - myDB.action('''DELETE from have''') - - for song in results: - try: - f = MediaFile(song) - except: - logger.warn('Could not read file: %s' % song) - continue - else: - if f.albumartist: - artist = f.albumartist - elif f.artist: - artist = f.artist - else: - continue - - if not f.album: - album = None - else: - album = f.album - - myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?)', [artist, album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid]) - - # Get the average bitrate if the option is selected - if headphones.DETECT_BITRATE: - try: - avgbitrate = myDB.action("SELECT AVG(BitRate) FROM have").fetchone()[0] - headphones.PREFERRED_BITRATE = int(avgbitrate)/1000 - - except Exception, e: - logger.error('Error detecting preferred bitrate:' + str(e)) - - artistlist = myDB.action('SELECT DISTINCT ArtistName FROM have').fetchall() - logger.info(u"Preparing to import %i artists" % len(artistlist)) - - artistlist_to_mbids(artistlist) - -def itunesImport(pathtoxml): - - if os.path.splitext(pathtoxml)[1] == '.xml': - logger.info(u"Loading xml file from"+ pathtoxml) - pl = XMLLibraryParser(pathtoxml) - l = Library(pl.dictionary) - lst = [] - for song in l.songs: - lst.append(song.artist) - rawlist = {}.fromkeys(lst).keys() - artistlist = [f for f in rawlist if f != None] - - else: - rawlist = os.listdir(pathtoxml) - logger.info(u"Loading artists from directory:" +pathtoxml) - exclude = ['.ds_store', 'various artists', 'untitled folder', 'va'] - artistlist = [f for f in rawlist if f.lower() not in exclude] - - logger.info('Starting directory/xml import...') - artistlist_to_mbids(artistlist) - def is_exists(artistid): @@ -109,7 +26,7 @@ def artistlist_to_mbids(artistlist): for artist in artistlist: - results = mb.findArtist(artist['ArtistName'], limit=1) + results = mb.findArtist(artist, limit=1) if not results: continue @@ -124,16 +41,12 @@ def artistlist_to_mbids(artistlist): # Add to database if it doesn't exist if artistid != various_artists_mbid and not is_exists(artistid): addArtisttoDB(artistid) - - # Update track count regardless of whether it already exists - if artistid != various_artists_mbid: - + + # Just update the tracks if it does + else: myDB = db.DBConnection() - havetracks = len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) - - controlValueDict = {"ArtistID": artistid} - newValueDict = {"HaveTracks": havetracks} - myDB.upsert("artists", newValueDict, controlValueDict) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) # Update the similar artist tag cloud: logger.info('Updating artist information from Last.fm') @@ -231,6 +144,8 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) for track in release_dict['tracks']: + cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) + controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, @@ -239,14 +154,29 @@ def addArtisttoDB(artistid, extrasonly=False): "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], - "TrackNumber": track['number'] + "TrackNumber": track['number'], + "CleanName": cleanname } - + + location = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() + + if not location: + location = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + + if not location: + location = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() + + if location: + newValueDict['Location'] = location['Location'] + newValueDict['BitRate'] = location['BitRate'] + myDB.action('DELETE from have WHERE Location=?', [location['location']]) + myDB.upsert("tracks", newValueDict, controlValueDict) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + controlValueDict = {"ArtistID": artistid} if latestalbum: @@ -254,7 +184,8 @@ def addArtisttoDB(artistid, extrasonly=False): "LatestAlbum": latestalbum['AlbumTitle'], "ReleaseDate": latestalbum['ReleaseDate'], "AlbumID": latestalbum['AlbumID'], - "TotalTracks": totaltracks} + "TotalTracks": totaltracks, + "HaveTracks": havetracks} else: newValueDict = {"Status": "Active"} @@ -339,6 +270,8 @@ def addReleaseById(rid): for track in release_dict['tracks']: + cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) + controlValueDict = {"TrackID": track['id'], "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], @@ -347,8 +280,21 @@ def addReleaseById(rid): "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], - "TrackNumber": track['number'] + "TrackNumber": track['number'], + "CleanName": cleanname } + + location = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() + + if not location: + location = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + + if not location: + location = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() + + if location: + newValueDict['Location'] = location['Location'] + newValueDict['BitRate'] = location['BitRate'] myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 8a093b04..d5d7f742 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -4,17 +4,18 @@ import glob from lib.beets.mediafile import MediaFile import headphones -from headphones import db, logger +from headphones import db, logger, helpers, importer -def LibraryScan(): +def libraryScan(): if not headphones.MUSIC_DIR: return - unmatched_files = [] new_artists = [] + bitrates = [] myDB = db.DBConnection() + myDB.action('''DELETE from have''') for r,d,f in os.walk(headphones.MUSIC_DIR): for files in f: @@ -22,63 +23,58 @@ def LibraryScan(): if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): file = unicode(os.path.join(r, files), "utf-8") - print repr(file) + # Try to read the metadata try: f = MediaFile(file) except: logger.error('Cannot read file: ' + file) - continue + continue + + # Grab the bitrates for the auto detect bit rate option + if f.bitrate: + bitrates.append(f.bitrate) - # Try to match on metadata first, + # Try to match on metadata first, starting with the track id if f.mb_trackid: - print 'has track id:' + repr(f.mb_trackid) + # Wondering if theres a better way to do this -> do one thing if the row exists, # do something else if it doesn't track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() - if not track: - if f.albumartist: - new_artists.append(f.albumartist) - elif f.artist: - new_artists.append(f.artist) - else: - unmatched_files.append(file) - - else: + if track: myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) continue - elif f.albumartist and f.album and f.title: - - track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f.albumartist, f.album, f.title]).fetchone() - - if not track: - new_artists.append(f.albumartist) - - else: - myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) - - elif f.artist and f.album and f.title: - - track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f.artist, f.album, f.title]).fetchone() - - if not track: - new_artists.append(file) - else: - myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) - + # Try to find a match based on artist/album/tracktitle + if f.albumartist: + f_artist = f.albumartist + elif f.artist: + f_artist = f.artist else: continue + + if f_artist and f.album and f.title: + + track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() + + if not track: + track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() - # Try to parse the unmatched files based on the folder & file formats in the config - for file in unmatched_files: - # Will do later - pass + if track: + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE TrackID=?', [file, f.bitrate, track['TrackID']]) + continue + + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release + new_artists.append(f_artist) + + # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database + myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, file, helpers.cleanName(f_artist+' '+f.album+' '+f.title)]) # Now check empty file paths to see if we can find a match based on their folder format - tracks = myDB.select('SELECT * from tracks WHERE Location=NULL') + tracks = myDB.select('SELECT * from tracks WHERE Location IS NULL') for track in tracks: + release = myDB.action('SELECT * from albums WHERE AlbumID=?', [track['AlbumID']]).fetchone() try: @@ -129,32 +125,48 @@ def LibraryScan(): new_file_name = new_file_name.replace('?','_').replace(':', '_') - full_path_to_file = os.path.join(headphones.MUSIC_DIR, folder, new_file_name) + full_path_to_file = os.path.normpath(os.path.join(headphones.MUSIC_DIR, folder, new_file_name)) + + match = glob.glob(full_path_to_file) - print 'Full path to file is: ' + repr(full_path_to_file) - - if glob.glob(full_path_to_file): - myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [full_path_to_file, track['TrackID']]) + if match: + + myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [match[0], track['TrackID']]) # Try to insert the appropriate track id so we don't have to keep doing this try: - f = MediaFile(full_path_to_file) + f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) f.save() + logger.info('Wrote mbid to track: %s' % match[0]) except: - logger.error('Error embedding track id into: %s' % full_path_to_file) + logger.error('Error embedding track id into: %s' % match[0]) - # Lastly, clean up any old paths that don't exist + # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') + for track in tracks: if not os.path.isfile(track['Location']): myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [None, track['TrackID']]) - + # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() - current_artists = myDB.action('SELECT ArtistName from artists').fetchall() + current_artists = myDB.select('SELECT ArtistName from artists') - artist_list = [f for f in unique_artists if f not in current_artists] + artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] - logger.info('Found %i artists to import: ' % len(artist_list)) \ No newline at end of file + logger.info('Found %i new artists to import.' % len(artist_list)) + + # Update track counts + for artist in current_artists: + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ?', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) + + if headphones.ADD_ARTISTS: + importer.artistlist_to_mbids(artist_list) + else: + headphones.NEW_ARTISTS = artist_list + + if headphones.DETECT_BITRATE: + headphones.PREFERRED_BITRATE = float(sum(bitrates))/len(bitrates)/1000 \ No newline at end of file From f3c4d684ebdf8e6fc879113361d1e4d35e663c77 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 15 Aug 2011 22:43:31 -0700 Subject: [PATCH 4/9] Added unmatched tracks to album page, added bit rate --- data/interfaces/default/album.html | 39 +++++++++++------------------ data/interfaces/default/artist.html | 2 +- headphones/__init__.py | 2 +- headphones/helpers.py | 10 ++++++++ headphones/importer.py | 2 +- headphones/librarysync.py | 7 +++--- headphones/postprocessor.py | 31 ----------------------- 7 files changed, 31 insertions(+), 62 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 073a77e7..ca2e9ce3 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -62,7 +62,7 @@ grade = 'A' location = track['Location'] else: - grade = 'Z' + grade = 'X' location = '' if track['BitRate']: @@ -83,35 +83,25 @@ ${bitrate} %endfor - - - <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) - %> - %if unmatched: - - - - - - - - - - - - %for track in unmatched: - + <% + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + %> + %if unmatched: + %for track in unmatched: + <% + duration = helpers.convert_seconds(float(track['TrackLength'])) + %> + - + - + - %endfor + %endfor + %endif
#Track TitleDurationLocal FileBit Rate
${track['TrackNumber']} ${track['TrackTitle']}${track['TrackLength']}${duration} ${track['Location']}${track['BitRate']}${int(track['BitRate'])/1000} kbps
- %endif @@ -127,6 +117,7 @@ { $('#track_table').dataTable( { + "aaSorting": [], "bFilter": false, "bInfo": false, "bPaginate": false diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 0438ac4c..7c82f4fc 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -63,7 +63,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/headphones/__init__.py b/headphones/__init__.py index 335768d1..6c1a1e02 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -429,7 +429,7 @@ def start(): SCHED.add_cron_job(updater.dbUpdate, hour=4, minute=0, second=0) SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) - SCHED.add_interval_job(librarysync.libraryScan(), minutes=LIBRARYSCAN_INTERVAL) + SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL) SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) librarysync.libraryScan() diff --git a/headphones/helpers.py b/headphones/helpers.py index 568bb0f7..9428f319 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -84,6 +84,16 @@ def convert_milliseconds(ms): return minutes +def convert_seconds(s): + + gmtime = time.gmtime(s) + if s > 3600: + minutes = time.strftime("%H:%M:%S", gmtime) + else: + minutes = time.strftime("%M:%S", gmtime) + + return minutes + def today(): today = datetime.date.today() yyyymmdd = datetime.date.isoformat(today) diff --git a/headphones/importer.py b/headphones/importer.py index 70f5701c..914fe759 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -45,7 +45,7 @@ def artistlist_to_mbids(artistlist): # Just update the tracks if it does else: myDB = db.DBConnection() - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) # Update the similar artist tag cloud: diff --git a/headphones/librarysync.py b/headphones/librarysync.py index d5d7f742..eca74417 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -152,18 +152,17 @@ def libraryScan(): # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() - current_artists = myDB.select('SELECT ArtistName from artists') + current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] - logger.info('Found %i new artists to import.' % len(artist_list)) - # Update track counts for artist in current_artists: - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ?', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) if headphones.ADD_ARTISTS: + logger.info('Found %i new artists to import.' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: headphones.NEW_ARTISTS = artist_list diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 8bf96692..70ae4ac3 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -240,7 +240,6 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [new_track_count, release['ArtistID']]) myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid]) myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid]) - updateHave(albumpath) logger.info('Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle'])) @@ -421,36 +420,6 @@ def renameFiles(albumpath, downloaded_track_list, release): except Exception, e: logger.error('Error renaming file: %s. Error: %s' % (downloaded_track, e)) continue - -def updateHave(albumpath): - - results = [] - - for r,d,f in os.walk(albumpath): - for files in f: - if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): - results.append(os.path.join(r, files)) - - if results: - - myDB = db.DBConnection() - - for song in results: - try: - f = MediaFile(song) - #logger.debug('Reading: %s' % song.decode('UTF-8')) - except: - logger.warn('Could not read file: %s' % song) - continue - else: - if f.albumartist: - artist = f.albumartist - elif f.artist: - artist = f.artist - else: - continue - - myDB.action('INSERT INTO have VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?)', [artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid]) def renameUnprocessedFolder(albumpath): From b4663b39df6c0be9c4561167cd0328d568761071 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 15 Aug 2011 23:19:43 -0700 Subject: [PATCH 5/9] Updated post processor to update have tracks on download --- headphones/librarysync.py | 2 +- headphones/postprocessor.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index eca74417..37be6708 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -168,4 +168,4 @@ def libraryScan(): headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: - headphones.PREFERRED_BITRATE = float(sum(bitrates))/len(bitrates)/1000 \ No newline at end of file + headphones.PREFERRED_BITRATE = round(sum(bitrates))/len(bitrates)/1000 \ No newline at end of file diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 70ae4ac3..2efb6942 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -240,6 +240,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [new_track_count, release['ArtistID']]) myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid]) myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid]) + updateHave(albumpath) logger.info('Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle'])) @@ -420,6 +421,36 @@ def renameFiles(albumpath, downloaded_track_list, release): except Exception, e: logger.error('Error renaming file: %s. Error: %s' % (downloaded_track, e)) continue + +def updateHave(albumpath): + + results = [] + + for r,d,f in os.walk(albumpath): + for files in f: + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): + results.append(os.path.join(r, files)) + + if results: + + myDB = db.DBConnection() + + for song in results: + try: + f = MediaFile(song) + #logger.debug('Reading: %s' % song.decode('UTF-8')) + except: + logger.warn('Could not read file: %s' % song) + continue + else: + if f.albumartist: + artist = f.albumartist + elif f.artist: + artist = f.artist + else: + continue + + myDB.action('UPDATE tracks SET Location=?, BitRate=? WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song, f.bitrate, artist, f.album, f.title]) def renameUnprocessedFolder(albumpath): From 10b334f29e4629b6910b0add85a4ece7d8fe2509 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 16 Aug 2011 13:07:17 -0700 Subject: [PATCH 6/9] Added manage new artists page, fixed some bugs in the importer --- data/css/style.css | 14 ++++-- data/interfaces/default/history.html | 2 +- data/interfaces/default/manage.html | 5 ++- data/interfaces/default/manageartists.html | 31 ++++--------- data/interfaces/default/managenew.html | 51 ++++++++++++++++++++++ headphones/__init__.py | 4 +- headphones/importer.py | 41 +++++++++-------- headphones/librarysync.py | 25 +++++++---- headphones/webserve.py | 15 +++++-- 9 files changed, 125 insertions(+), 63 deletions(-) create mode 100644 data/interfaces/default/managenew.html diff --git a/data/css/style.css b/data/css/style.css index 2d3fb458..24bc2943 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -177,19 +177,25 @@ img.albumArt { float: left; padding-right: 5px; } div#albumheader { padding-top: 48px; height: 200px; } div#track_wrapper { margin-left: -50px; padding-top: 20px; font-size: 16px; width: 100%; } -table#track_table th#number { text-align: right; min-width: 20px; } -table#track_table th#name { text-align: center; min-width: 350px; } +table#track_table th#number { text-align: right; min-width: 10px; } +table#track_table th#name { text-align: center; min-width: 320px; } table#track_table th#duration { width: 175px; text-align: center; min-width: 100px; } table#track_table th#location { text-align: center; min-width: 200px; } table#track_table th#bitrate { text-align: center; min-width: 75px; } table#track_table td#number { vertical-align: middle; text-align: right; } -table#track_table td#name { vertical-align: middle; text-align: center; } +table#track_table td#name { vertical-align: middle; text-align: center; font-size: 15px; } table#track_table td#duration { vertical-align: middle; text-align: center; } table#track_table td#location { vertical-align: middle; text-align: center; font-size: 11px; } table#track_table td#bitrate { vertical-align: middle; text-align: center; font-size: 12px; } -table#history_table { background-color: white; width: 100%; } +table#history_table { background-color: white; width: 100%; font-size: 13px; } + +table#history_table td#dateadded { vertical-align: middle; text-align: center; min-width: 150px; font-size: 14px; } +table#history_table td#filename { vertical-align: middle; text-align: center; min-width: 100px; font-size: 15px; } +table#history_table td#size { vertical-align: middle; text-align: center; min-width: 75px; font-size: 14px; } +table#history_table td#status { vertical-align: middle; text-align: center; font-size: 14px; } +table#history_table td#action { vertical-align: middle; text-align: center; font-size: 14px; } table#log_table { background-color: white; } diff --git a/data/interfaces/default/history.html b/data/interfaces/default/history.html index 8c98db9d..f8f60112 100644 --- a/data/interfaces/default/history.html +++ b/data/interfaces/default/history.html @@ -42,7 +42,7 @@ %> ${item['DateAdded']} - ${item['Title']} [album page] + ${item['Title']} [nzb][album page] ${helpers.bytes_to_mb(item['Size'])} ${item['Status']} [retry][new] diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 3e6f7452..8d283b97 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -5,7 +5,10 @@ <%def name="headerIncludes()">
diff --git a/data/interfaces/default/manageartists.html b/data/interfaces/default/manageartists.html index d1762cb2..d0d6d30d 100644 --- a/data/interfaces/default/manageartists.html +++ b/data/interfaces/default/manageartists.html @@ -1,18 +1,14 @@ <%inherit file="base.html" /> - +<%! + import headphones +%> <%def name="body()">
-

Manage Artists

+

Manage New Artists

-
+

- - selected artists + Add selected artists

@@ -20,23 +16,13 @@ - - %for artist in artists: - <% - if artist['Status'] == 'Paused': - grade = 'X' - elif artist['Status'] == 'Loading': - grade = 'C' - else: - grade = 'Z' - %> - + %for artist in headphones.NEW_ARTISTS: + - %endfor @@ -58,7 +44,6 @@ "aoColumns": [ null, { "sType": "title-string"}, - null ], "bStateSave": true, "bPaginate": false diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html new file mode 100644 index 00000000..779e709e --- /dev/null +++ b/data/interfaces/default/managenew.html @@ -0,0 +1,51 @@ +<%inherit file="base.html" /> +<%! + import headphones +%> +<%def name="body()"> +
+

Manage New Artists

+

+ +

+ Add selected artists + +

+
Artist NameStatus
${artist['ArtistName']}${artist['Status']}
+ + + + + + + + %for artist in headphones.NEW_ARTISTS: + + + + + %endfor + +
Artist Name
${artist}
+
+ + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + \ No newline at end of file diff --git a/headphones/__init__.py b/headphones/__init__.py index 6c1a1e02..25631191 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -63,7 +63,7 @@ PREFERRED_QUALITY = None PREFERRED_BITRATE = None DETECT_BITRATE = False ADD_ARTISTS = False -NEW_ARTISTS = None +NEW_ARTISTS = [] CORRECT_METADATA = False MOVE_FILES = False RENAME_FILES = False @@ -432,7 +432,7 @@ def start(): SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL) SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) - librarysync.libraryScan() + SCHED.start() started = True diff --git a/headphones/importer.py b/headphones/importer.py index 914fe759..08334af0 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -141,7 +141,6 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.action('DELETE from albums WHERE AlbumID=?', [release['releaseid']]) myDB.action('DELETE from tracks WHERE AlbumID=?', [release['releaseid']]) - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) for track in release_dict['tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) @@ -158,18 +157,16 @@ def addArtisttoDB(artistid, extrasonly=False): "CleanName": cleanname } - location = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() - - if not location: - location = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + match = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() - if not location: - location = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - - if location: - newValueDict['Location'] = location['Location'] - newValueDict['BitRate'] = location['BitRate'] - myDB.action('DELETE from have WHERE Location=?', [location['location']]) + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) @@ -187,7 +184,9 @@ def addArtisttoDB(artistid, extrasonly=False): "TotalTracks": totaltracks, "HaveTracks": havetracks} else: - newValueDict = {"Status": "Active"} + newValueDict = {"Status": "Active", + "TotalTracks": totaltracks, + "HaveTracks": havetracks} myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Updating complete for: " + artist['artist_name']) @@ -284,17 +283,17 @@ def addReleaseById(rid): "CleanName": cleanname } - location = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() + match = myDB.action('SELECT Location, BitRate from have WHERE TrackID=?', [track['id']]).fetchone() - if not location: - location = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE CleanName=?', [cleanname]).fetchone() - if not location: - location = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() - if location: - newValueDict['Location'] = location['Location'] - newValueDict['BitRate'] = location['BitRate'] + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 37be6708..62b3affa 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -6,10 +6,17 @@ from lib.beets.mediafile import MediaFile import headphones from headphones import db, logger, helpers, importer -def libraryScan(): +def libraryScan(dir=None): - if not headphones.MUSIC_DIR: - return + if not dir: + dir = headphones.MUSIC_DIR + + try: + dir = str(dir) + except UnicodeEncodeError: + dir = unicode(dir).encode('unicode_escape') + + logger.info('Scanning music directory: %s' % dir) new_artists = [] bitrates = [] @@ -17,7 +24,7 @@ def libraryScan(): myDB = db.DBConnection() myDB.action('''DELETE from have''') - for r,d,f in os.walk(headphones.MUSIC_DIR): + for r,d,f in os.walk(dir): for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): @@ -132,17 +139,19 @@ def libraryScan(): if match: myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [match[0], track['TrackID']]) + myDB.action('DELETE from have WHERE Location=?', [match[0]]) # Try to insert the appropriate track id so we don't have to keep doing this try: f = MediaFile(match[0]) f.mb_trackid = track['TrackID'] - myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) f.save() - logger.info('Wrote mbid to track: %s' % match[0]) + myDB.action('UPDATE tracks SET BitRate=? WHERE TrackID=?', [f.bitrate, track['TrackID']]) + logger.debug('Wrote mbid to track: %s' % match[0]) except: logger.error('Error embedding track id into: %s' % match[0]) - + continue + # Clean up bad filepaths tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') @@ -168,4 +177,4 @@ def libraryScan(): headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: - headphones.PREFERRED_BITRATE = round(sum(bitrates))/len(bitrates)/1000 \ No newline at end of file + headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 \ No newline at end of file diff --git a/headphones/webserve.py b/headphones/webserve.py index 399d22ee..8c2ee10d 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -10,7 +10,7 @@ import threading import headphones -from headphones import logger, searcher, db, importer, mb, lastfm +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync from headphones.helpers import checked, radio @@ -148,7 +148,12 @@ class WebInterface(object): else: raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - + + def addArtists(self, **args): + threading.Thread(target=importer.artistlist_to_mbids, args=[args]).start() + time.sleep(5) + raise cherrypy.HTTPRedirect("home") + addArtists.exposed = True def queueAlbum(self, AlbumID, ArtistID=None, new=False, redirect=None): logger.info(u"Marking album: " + AlbumID + "as wanted...") @@ -189,6 +194,10 @@ class WebInterface(object): return serve_template(templatename="manageartists.html", title="Manage Artists", artists=artists) manageArtists.exposed = True + def manageNew(self): + return serve_template(templatename="managenew.html", title="Manage New Artists") + manageNew.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() for ArtistID in args: @@ -231,7 +240,7 @@ class WebInterface(object): headphones.MUSIC_DIR = path headphones.config_write() try: - threading.Thread(target=importer.scanMusic, args=[path]).start() + threading.Thread(target=librarysync.libraryScan).start() except Exception, e: logger.error('Unable to complete the scan: %s' % e) time.sleep(10) From c866316c090d30ccc05f8a62a809eba3dd34e887 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 16 Aug 2011 13:42:14 -0700 Subject: [PATCH 7/9] Updated artist to mbid function to accept non utf8 chars --- data/interfaces/default/manageartists.html | 31 ++++++++++++++++------ headphones/__init__.py | 4 +-- headphones/importer.py | 7 +++-- headphones/webserve.py | 2 +- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/data/interfaces/default/manageartists.html b/data/interfaces/default/manageartists.html index d0d6d30d..d1762cb2 100644 --- a/data/interfaces/default/manageartists.html +++ b/data/interfaces/default/manageartists.html @@ -1,14 +1,18 @@ <%inherit file="base.html" /> -<%! - import headphones -%> + <%def name="body()">
-

Manage New Artists

+

Manage Artists

-
+

- Add selected artists + + selected artists

@@ -16,13 +20,23 @@ + - %for artist in headphones.NEW_ARTISTS: - + %for artist in artists: + <% + if artist['Status'] == 'Paused': + grade = 'X' + elif artist['Status'] == 'Loading': + grade = 'C' + else: + grade = 'Z' + %> + + %endfor @@ -44,6 +58,7 @@ "aoColumns": [ null, { "sType": "title-string"}, + null ], "bStateSave": true, "bPaginate": false diff --git a/headphones/__init__.py b/headphones/__init__.py index 25631191..dadff537 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -201,7 +201,7 @@ def initialize(): PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '') DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) - ADD_ARTISTS = bool(check_setting_int(CFG, 'General', 'add_artists', 1)) + ADD_ARTISTS = bool(check_setting_int(CFG, 'General', 'auto_add_artists', 1)) CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) @@ -366,7 +366,7 @@ def config_write(): new_config['General']['preferred_quality'] = PREFERRED_QUALITY new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) - new_config['General']['add_artists'] = int(ADD_ARTISTS) + new_config['General']['auto_add_artists'] = int(ADD_ARTISTS) new_config['General']['correct_metadata'] = int(CORRECT_METADATA) new_config['General']['move_files'] = int(MOVE_FILES) new_config['General']['rename_files'] = int(RENAME_FILES) diff --git a/headphones/importer.py b/headphones/importer.py index 08334af0..c0cb0e60 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -22,10 +22,13 @@ def is_exists(artistid): return False -def artistlist_to_mbids(artistlist): +def artistlist_to_mbids(artistlist, forced=False): for artist in artistlist: - + + if forced: + artist = unicode(artist, 'utf-8') + results = mb.findArtist(artist, limit=1) if not results: diff --git a/headphones/webserve.py b/headphones/webserve.py index 8c2ee10d..295ff6b6 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -150,7 +150,7 @@ class WebInterface(object): markAlbums.exposed = True def addArtists(self, **args): - threading.Thread(target=importer.artistlist_to_mbids, args=[args]).start() + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() time.sleep(5) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True From 91576e3f63e992b6c3aa518495a356135fd9c6cb Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 16 Aug 2011 15:02:06 -0700 Subject: [PATCH 8/9] Fixed some stuff --- data/css/style.css | 4 ++-- data/interfaces/default/managenew.html | 1 + headphones/webserve.py | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/data/css/style.css b/data/css/style.css index 24bc2943..a621f1f2 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -178,9 +178,9 @@ div#albumheader { padding-top: 48px; height: 200px; } div#track_wrapper { margin-left: -50px; padding-top: 20px; font-size: 16px; width: 100%; } table#track_table th#number { text-align: right; min-width: 10px; } -table#track_table th#name { text-align: center; min-width: 320px; } +table#track_table th#name { text-align: center; min-width: 300px; } table#track_table th#duration { width: 175px; text-align: center; min-width: 100px; } -table#track_table th#location { text-align: center; min-width: 200px; } +table#track_table th#location { text-align: center; width: 200px; } table#track_table th#bitrate { text-align: center; min-width: 75px; } table#track_table td#number { vertical-align: middle; text-align: right; } diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html index 779e709e..bea493b2 100644 --- a/data/interfaces/default/managenew.html +++ b/data/interfaces/default/managenew.html @@ -5,6 +5,7 @@ <%def name="body()">

Manage New Artists

+

Scan Music Library

diff --git a/headphones/webserve.py b/headphones/webserve.py index 295ff6b6..61f7f1ed 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -236,7 +236,7 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") importItunes.exposed = True - def musicScan(self, path): + def musicScan(self, path, redirect=None): headphones.MUSIC_DIR = path headphones.config_write() try: @@ -244,7 +244,10 @@ class WebInterface(object): except Exception, e: logger.error('Unable to complete the scan: %s' % e) time.sleep(10) - raise cherrypy.HTTPRedirect("home") + if redirect: + raise cherrypy.HTTPRedirect(redirect) + else: + raise cherrypy.HTTPRedirect("home") musicScan.exposed = True def forceUpdate(self): From fec2fbd926aecb7a8bba51ac0362ac82d1751358 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 16 Aug 2011 19:09:42 -0700 Subject: [PATCH 9/9] Template changes, bug fixed, lastfm exception fix --- data/css/style.css | 4 ++-- data/interfaces/default/config.html | 2 +- data/interfaces/default/manage.html | 4 ++++ headphones/importer.py | 4 +++- headphones/lastfm.py | 12 +++++++++++- headphones/librarysync.py | 9 +++++++-- headphones/webserve.py | 3 ++- 7 files changed, 30 insertions(+), 8 deletions(-) diff --git a/data/css/style.css b/data/css/style.css index a621f1f2..15b4af99 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -178,9 +178,9 @@ div#albumheader { padding-top: 48px; height: 200px; } div#track_wrapper { margin-left: -50px; padding-top: 20px; font-size: 16px; width: 100%; } table#track_table th#number { text-align: right; min-width: 10px; } -table#track_table th#name { text-align: center; min-width: 300px; } +table#track_table th#name { text-align: center; min-width: 350px; } table#track_table th#duration { width: 175px; text-align: center; min-width: 100px; } -table#track_table th#location { text-align: center; width: 200px; } +table#track_table th#location { text-align: center; width: 250px; } table#track_table th#bitrate { text-align: center; min-width: 75px; } table#track_table td#number { vertical-align: middle; text-align: right; } diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 5f4aa2da..b392e559 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -194,7 +194,7 @@ Highest Quality including Lossless
Lossless Only
Preferred Bitrate: - kbps
+ kbps
Auto-Detect Preferred Bitrate

Artist NameStatus
${artist['ArtistName']}${artist['Status']}
diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 8d283b97..cf5a476a 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -1,6 +1,7 @@ <%inherit file="base.html" /> <%! import headphones + from headphones.helpers import checked %> <%def name="headerIncludes()">
@@ -33,6 +34,9 @@ %endif +
+

Automatically add new artists

+

diff --git a/headphones/importer.py b/headphones/importer.py index c0cb0e60..893115c5 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -16,7 +16,7 @@ def is_exists(artistid): artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): - logger.debug(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") + logger.info(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") return True else: return False @@ -32,6 +32,7 @@ def artistlist_to_mbids(artistlist, forced=False): results = mb.findArtist(artist, limit=1) if not results: + logger.info('No results found for: %' % artist) continue try: @@ -297,6 +298,7 @@ def addReleaseById(rid): if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] + myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 814a35e8..f46ea2f7 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -2,6 +2,7 @@ import urllib from xml.dom import minidom from collections import defaultdict import random +import time import headphones from headphones import db, logger @@ -19,7 +20,16 @@ def getSimilar(): for result in results[:12]: url = 'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&mbid=%s&api_key=%s' % (result['ArtistID'], api_key) - data = urllib.urlopen(url).read() + + try: + data = urllib.urlopen(url).read() + except: + time.sleep(1) + continue + + len(data) < 200: + continue + d = minidom.parseString(data) node = d.documentElement artists = d.getElementsByTagName("artist") diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 62b3affa..37cd10cb 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -158,7 +158,9 @@ def libraryScan(dir=None): for track in tracks: if not os.path.isfile(track['Location']): myDB.action('UPDATE tracks SET Location=? WHERE TrackID=?', [None, track['TrackID']]) - + + logger.info('Completed scanning of directory: %s. Updating track counts' % dir) + # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') @@ -170,10 +172,13 @@ def libraryScan(dir=None): havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) + logger.info('Found %i new artists' % len(artist_list)) + if headphones.ADD_ARTISTS: - logger.info('Found %i new artists to import.' % len(artist_list)) + logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: + logger.info('To add these artists, go to Manage->Manage New Artists') headphones.NEW_ARTISTS = artist_list if headphones.DETECT_BITRATE: diff --git a/headphones/webserve.py b/headphones/webserve.py index 61f7f1ed..d5fe8006 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -236,7 +236,8 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") importItunes.exposed = True - def musicScan(self, path, redirect=None): + def musicScan(self, path, redirect=None, autoadd=0): + headphones.ADD_ARTISTS = autoadd headphones.MUSIC_DIR = path headphones.config_write() try: