diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index bec42cfb..687e4540 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -138,7 +138,7 @@ %endfor <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) %> %if unmatched: %for track in unmatched: diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index e891b1fa..845f6719 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -93,7 +93,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index b4246cfe..65e5370a 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -97,6 +97,14 @@ mins +
+ + hours +
+
+ + days +
diff --git a/data/interfaces/default/js/script.js b/data/interfaces/default/js/script.js index e021074b..a0bb860e 100644 --- a/data/interfaces/default/js/script.js +++ b/data/interfaces/default/js/script.js @@ -315,6 +315,7 @@ function doAjaxCall(url,elem,reload,form) { console.log('refresh'); refreshTable(); } if ( reload == "tabs") refreshTab(); + if ( reload == "page") location.reload(); if ( form ) { // Change the option to 'choose...' $(formID + " select").children('option[disabled=disabled]').attr('selected','selected'); diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 66405aab..6d98bb53 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -22,6 +22,7 @@ %if not headphones.ADD_ARTISTS: Manage New Artists %endif + Manage Unmatched @@ -35,6 +36,7 @@
  • Scan Music Library
  • Imports
  • Force Actions
    • +
  • Force Legacy
    • @@ -119,7 +121,7 @@ Force Search
    Force Check for Wanted Albums - Force Update Active Artists + Force Update Active Artists [Fast] Force Post-Process Albums in Download Folder Check for Headphones Updates Delete empty Artists @@ -139,6 +141,26 @@
    + + +
    +
    + Force Legacy +

    Please note that these functions will take a significant amount of time to complete.

    +
    + Force Update Active Artists [Comprehensive] +
    + Force Re-scan Library [Comprehensive] +
    + *Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched". + +
    +
    + +
    + + + <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/managealbums.html b/data/interfaces/default/managealbums.html index a30dde6a..487028d6 100644 --- a/data/interfaces/default/managealbums.html +++ b/data/interfaces/default/managealbums.html @@ -57,7 +57,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/managemanual.html b/data/interfaces/default/managemanual.html new file mode 100644 index 00000000..e20882ca --- /dev/null +++ b/data/interfaces/default/managemanual.html @@ -0,0 +1,121 @@ +<%inherit file="base.html" /> +<%! + import headphones + from headphones import db, helpers + myDB = db.DBConnection() +%> + +<%def name="headerIncludes()"> +
    +
    +
    +
    + « Back to Unmatched Albums + + + +<%def name="body()"> +
    +
    +

    manageManage Manually Changed Albums

    +
    + + + + + + + + + + + <% count_albums=0 %> + %for album in manualalbums: + + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace('+', '%2B').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace('+', '%2B').replace("'","%27") + %> + + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal AlbumPrevious Action
    ${album['ArtistName']}
    + + +     		${album['AlbumTitle']}
    + + + +     		${album['AlbumStatus']} + +
    +
    + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html index 17d567f7..41f9263d 100644 --- a/data/interfaces/default/managenew.html +++ b/data/interfaces/default/managenew.html @@ -20,7 +20,10 @@
    - Add selected artists +
    diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html new file mode 100644 index 00000000..3fdc7eaa --- /dev/null +++ b/data/interfaces/default/manageunmatched.html @@ -0,0 +1,259 @@ +<%inherit file="base.html" /> +<%! + import headphones + import json + from headphones import db, helpers + myDB = db.DBConnection() + artist_json = {} + counter = 0 + artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE") + for artist in artist_list: + artist_json[counter] = artist['ArtistName'] + counter+=1 + json_artists = json.dumps(artist_json) +%> + +<%def name="headerIncludes()"> +
    + +
    + « Back to manage overview + + + +<%def name="body()"> +
    +
    +

    manageManage Unmatched Albums

    +
    + +
    + + + + + + + + <% count_albums=0 %> + %for album in unmatchedalbums: + + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace("'","%27") + old_artist_js = album['ArtistName'].replace("'","\\'").replace('"','\\"') + old_album_js = album['AlbumTitle'].replace("'","\\'").replace('"','\\"') + %> + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal Album
    ${album['ArtistName']}
    + + + + + + +     		${album['AlbumTitle']}
    + + + + + + +
    + + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/headphones/__init__.py b/headphones/__init__.py index 234d2cea..6059325d 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -130,6 +130,8 @@ SEARCH_INTERVAL = 360 LIBRARYSCAN = False LIBRARYSCAN_INTERVAL = 300 DOWNLOAD_SCAN_INTERVAL = 5 +UPDATE_DB_INTERVAL = 24 +MB_IGNORE_AGE = 365 SAB_HOST = None SAB_USERNAME = None @@ -302,7 +304,7 @@ def initialize(): ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \ TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, PIRATEBAY_PROXY_URL, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \ RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \ UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \ NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \ @@ -405,6 +407,8 @@ def initialize(): LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300) DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) + UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24) + MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365) TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '') NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10') @@ -792,6 +796,8 @@ def config_write(): new_config['General']['libraryscan'] = int(LIBRARYSCAN) new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL + new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL + new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE new_config['SABnzbd'] = {} new_config['SABnzbd']['sab_host'] = SAB_HOST @@ -917,7 +923,7 @@ def start(): # Start our scheduled background tasks from headphones import updater, searcher, librarysync, postprocessor - SCHED.add_interval_job(updater.dbUpdate, hours=24) + SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL) SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL) SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) @@ -1118,6 +1124,7 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') + conn.commit() c.close() diff --git a/headphones/importer.py b/headphones/importer.py index 0365a872..45204464 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -15,6 +15,7 @@ from lib.pyItunes import * import time +import threading import os from lib.beets.mediafile import MediaFile @@ -108,7 +109,7 @@ def addArtistIDListToDB(artistidlist): for artistid in artistidlist: addArtisttoDB(artistid) -def addArtisttoDB(artistid, extrasonly=False): +def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache @@ -189,65 +190,155 @@ def addArtisttoDB(artistid, extrasonly=False): db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: - includeExtras = False + includeExtras = False + + #Clean all references to release group in dB that are no longer referenced in musicbrainz + group_list = [] + force_repackage = 0 + #Don't nuke the database if there's a MusicBrainz error + if len(artist['releasegroups']) != 0 and not extrasonly: + for groups in artist['releasegroups']: + group_list.append(groups['id']) + remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + for items in remove_missing_groups_from_albums: + if items['AlbumID'] not in group_list: + # Remove all from albums/tracks that aren't in release groups + myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) + logger.info("Removing all references to group %s to reflect MusicBrainz" % items['AlbumID']) + force_repackage = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks + for rg in artist['releasegroups']: - logger.info("Now adding/updating: " + rg['title']) - + al_title = rg['title'] + today = helpers.today() rgid = rg['id'] - - # check if the album already exists - rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - - releases = mb.get_all_releases(rgid,includeExtras) - if releases == []: - logger.info('No official releases in release group %s' % rg['title']) - continue - if not releases: - errors = True - logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) - continue - - # This will be used later to build a hybrid release - fullreleaselist = [] + skip_log = 0 + #Make a user configurable variable to skip update of albums with release dates older than this date (in days) + pause_delta = headphones.MB_IGNORE_AGE - for release in releases: - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + if not forcefull: + check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() + if check_release_date: + if check_release_date[0] is None: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + elif len(check_release_date[0])!=10: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < pause_delta: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + logger.info('%s is over %s days old; not updating' % (al_title, pause_delta)) + skip_log = 1 + new_releases = 0 + else: + logger.info("Now adding/updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + if force_repackage == 1: + new_releases = -1 + logger.info('Forcing repackage of %s, since release groups have been removed' % al_title) + else: + new_releases = new_releases + else: + logger.info("Now adding/updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras,forcefull) + + #What this does is adds new releases per artist to the allalbums + alltracks databases + #new_releases = mb.get_new_releases(rgid,includeExtras) + #print al_title + #print new_releases + + if new_releases != 0: + #Dump existing hybrid release since we're repackaging/replacing it + myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) + # This will be used later to build a hybrid release + fullreleaselist = [] + #Search for releases within a release group + find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) + # Build the dictionary for the fullreleaselist + for items in find_hybrid_releases: + if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing + hybrid_release_id = items['ReleaseID'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] + } + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) + totalTracks = 1 + hybrid_track_array = [] + for hybrid_tracks in find_hybrid_tracks: + hybrid_track_array.append({ + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], + #'url': hybrid_tracks['TrackURL'], + 'duration': hybrid_tracks['TrackDuration'] + }) + totalTracks += 1 + newValueDict['ReleaseID'] = hybrid_release_id + newValueDict['Tracks'] = hybrid_track_array + fullreleaselist.append(newValueDict) + + #print fullreleaselist + + # Basically just do the same thing again for the hybrid release + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + logger.info('Packaging %s releases into hybrid title' % rg['title']) + except Exception, e: + errors = True + logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) + continue + + # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it + # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it + # The hybrid won't have a country or a format + controlValueDict = {"ReleaseID": rg['id']} + + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumID": rg['id'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "ReleaseDate": hybridrelease['ReleaseDate'], + "Type": rg['type'] } - + myDB.upsert("allalbums", newValueDict, controlValueDict) - # Build the dictionary for the fullreleaselist - newValueDict['ReleaseID'] = release['ReleaseID'] - newValueDict['Tracks'] = release['Tracks'] - fullreleaselist.append(newValueDict) - - for track in release['Tracks']: + for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} + "ReleaseID": rg['id']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], @@ -264,178 +355,111 @@ def addArtisttoDB(artistid, extrasonly=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Basically just do the same thing again for the hybrid release - # This may end up being called with an empty fullreleaselist - try: - hybridrelease = getHybridRelease(fullreleaselist) - except Exception, e: - errors = True - logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) - continue - - # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it - # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it - # The hybrid won't have a country or a format - controlValueDict = {"ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "ReleaseDate": hybridrelease['ReleaseDate'], - "Type": rg['type'] - } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in hybridrelease['Tracks']: - - cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) - - controlValueDict = {"TrackID": track['id'], - "ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "AlbumID": rg['id'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } - - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') - - # If there's no release in the main albums tables, add the default (hybrid) - # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) - if not rg_exists: - releaseid = rg['id'] - elif rg_exists and not rg_exists['ReleaseID']: - # Need to do some importing here - to transition the old format of using the release group - # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID - # so we'll need to move over the locations, bitrates & formats from the tracks table to the new - # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups - logger.info("Copying current track information to alternate releases") - tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() - for track in tracks: - if track['Location']: - controlValueDict = {"TrackID": track['TrackID']} - newValueDict = {"Location": track['Location'], - "BitRate": track['BitRate'], - "Format": track['Format'], - } - myDB.upsert("alltracks", newValueDict, controlValueDict) - releaseid = rg['id'] - else: - releaseid = rg_exists['ReleaseID'] - - album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - - controlValueDict = {"AlbumID": rg['id']} - - newValueDict = {"ArtistID": album['ArtistID'], - "ArtistName": album['ArtistName'], - "AlbumTitle": album['AlbumTitle'], - "ReleaseID": album['ReleaseID'], - "AlbumASIN": album['AlbumASIN'], - "ReleaseDate": album['ReleaseDate'], - "Type": album['Type'], - "ReleaseCountry": album['ReleaseCountry'], - "ReleaseFormat": album['ReleaseFormat'] - } - if not rg_exists: + # Delete matched tracks from the have table + #myDB.action('DELETE from have WHERE Matched="True"') - today = helpers.today() - - newValueDict['DateAdded']= today - - if headphones.AUTOWANT_ALL: - newValueDict['Status'] = "Wanted" - elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" - # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these - # The first test just makes sure we have year-month-day - elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" + # If there's no release in the main albums tables, add the default (hybrid) + # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) + # check if the album already exists + rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() + if not rg_exists: + releaseid = rg['id'] else: - newValueDict['Status'] = "Skipped" - - myDB.upsert("albums", newValueDict, controlValueDict) + releaseid = rg_exists['ReleaseID'] + + album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) - tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + controlValueDict = {"AlbumID": rg['id']} - # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT - total_track_count = len(tracks) - - for track in tracks: - - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": rg['id']} - - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] + newValueDict = {"ArtistID": album['ArtistID'], + "ArtistName": album['ArtistName'], + "AlbumTitle": album['AlbumTitle'], + "ReleaseID": album['ReleaseID'], + "AlbumASIN": album['AlbumASIN'], + "ReleaseDate": album['ReleaseDate'], + "Type": album['Type'], + "ReleaseCountry": album['ReleaseCountry'], + "ReleaseFormat": album['ReleaseFormat'] } - - myDB.upsert("tracks", newValueDict, controlValueDict) + + if not rg_exists: + + today = helpers.today() + + newValueDict['DateAdded']= today + + if headphones.AUTOWANT_ALL: + newValueDict['Status'] = "Wanted" + elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these + # The first test just makes sure we have year-month-day + elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + else: + newValueDict['Status'] = "Skipped" + + myDB.upsert("albums", newValueDict, controlValueDict) - # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album - have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) - marked_as_downloaded = False - - if rg_exists: - if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True + tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + + # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT + total_track_count = len(tracks) + + for track in tracks: + + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": rg['id']} + + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] + } + + myDB.upsert("tracks", newValueDict, controlValueDict) + + # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album + have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) + marked_as_downloaded = False + + if rg_exists: + if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + else: + if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + + logger.info(u"Seeing if we need album art for " + rg['title']) + cache.getThumb(AlbumID=rg['id']) + + #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: + if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: + from headphones import searcher + searcher.searchforalbum(albumid=rg['id']) else: - if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True - - logger.info(u"Seeing if we need album art for " + rg['title']) - cache.getThumb(AlbumID=rg['id']) - - #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: - if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: - from headphones import searcher - searcher.searchforalbum(albumid=rg['id']) + if skip_log == 0: + logger.info(u"No new releases, so no changes made to " + rg['title']) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['artist_name']])) controlValueDict = {"ArtistID": artistid} @@ -463,6 +487,7 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info("Finished updating artist: " + artist['artist_name'] + " but with errors, so not marking it as updated in the database") else: logger.info(u"Updating complete for: " + artist['artist_name']) + def addReleaseById(rid): @@ -569,7 +594,7 @@ def addReleaseById(rid): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 5ee57d8b..a9f76d11 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): + if cron and not headphones.LIBRARYSCAN: return @@ -46,13 +47,23 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not append: # Clean up bad filepaths - tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') - - for track in tracks: - if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) + tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') - myDB.action('DELETE from have') + for track in tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + + del_have_tracks = myDB.select('SELECT Location, Matched from have') + + for track in del_have_tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']]) + logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) + ###############myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) @@ -60,6 +71,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal bitrates = [] song_list = [] + new_song_count = 0 for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one @@ -99,8 +111,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) - song_dict = { 'TrackID' : f.mb_trackid, - 'ReleaseID' : f.mb_albumid, + if f_artist and f.album and f.title: + CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) + else: + CleanName = None + + controlValueDict = {'Location' : unicode_song_path} + + newValueDict = { 'TrackID' : f.mb_trackid, + #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, @@ -110,191 +129,116 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, - 'Location' : unicode_song_path } + 'CleanName' : CleanName + } - song_list.append(song_dict) + #song_list.append(song_dict) + check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() + #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. + if not check_exist_song: + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + elif check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: + newValueDict['Matched'] = None + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + # Now we start track matching - total_number_of_songs = len(song_list) - logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") + logger.info("%s new/modified songs found and added to the database" % new_song_count) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND CleanName IS NOT NULL AND LOCATION LIKE ?", [dir+"%"]) + total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] + logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches - song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 + latest_artist = [] for song in song_list: + + latest_artist.append(song['ArtistName']) + if song_count == 0: + logger.info("Now matching songs by %s" % song['ArtistName']) + elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: + logger.info("Now matching songs by %s" % song['ArtistName']) + #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") - # If the track has a trackid & releaseid (beets: albumid) that the most surefire way - # of identifying a track to a specific release so we'll use that first - if song['TrackID'] and song['ReleaseID']: + #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT + #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, + #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM + #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. - # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - if track: - # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Matched. Move on to the next one: - continue - - # If we can't find it with TrackID & ReleaseID, next most specific will be - # releaseid + tracktitle, although perhaps less reliable due to a higher - # likelihood of variations in the song title (e.g. feat. artists) - if song['ReleaseID'] and song['TrackTitle']: - - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if track: - # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) - # So use both TrackID & ReleaseID as the control values - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both tables here as well - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Done - continue - - # Next most specific will be the opposite: a TrackID and an AlbumTitle - # TrackIDs span multiple releases so if something is on an official album - # and a compilation, for example, this will match it to the right one - # However - there may be multiple matches here - if song['TrackID'] and song['AlbumTitle']: - - # Even though there might be multiple matches, we just need to grab one to confirm a match - track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if track: - # Don't need the hybridControlValueDict here since ReleaseID is not unique - controlValueDict = { 'TrackID' : track['TrackID'], - 'AlbumTitle' : track['AlbumTitle'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue - - # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably - # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], - 'TrackTitle' : track['TrackTitle'] } - + 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) - # This is more reliable than the former but requires some string manipulation so we'll do it only - # if we can't find a match with the original data - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - - track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if not track: - track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if track: - controlValueDict = { 'CleanName' : track['CleanName'] } - + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if track: + controlValueDict = { 'CleanName' : track['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("tracks", newValueDict, controlValueDict) + + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + + + alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + if alltrack: + controlValueDict = { 'ArtistName' : alltrack['ArtistName'], + 'AlbumTitle' : alltrack['AlbumTitle'], + 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable - # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track - # with more specific information exists in the library, it'll overwrite these values - if song['TrackID']: - - track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if track: - controlValueDict = { 'TrackID' : track['TrackID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if alltrack: + controlValueDict = { 'CleanName' : alltrack['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: @@ -302,23 +246,18 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal else: continue - # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - else: - continue - - myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - + if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') - - artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] + + #There was a bug where artists with special characters (-,') would show up in new artists. + artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') @@ -326,7 +265,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) @@ -348,6 +287,42 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) - + + update_album_status() + logger.info('Library scan complete') + + #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED +def update_album_status(AlbumID=None): + myDB = db.DBConnection() + logger.info('Counting matched tracks to mark albums as skipped/downloaded') + if AlbumID: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + else: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') + for album in album_status_updater: + track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']]) + total_tracks = 0 + have_tracks = 0 + for track in track_counter: + total_tracks+=1 + if track['Location']: + have_tracks+=1 + if total_tracks != 0: + album_completion = float(float(have_tracks) / float(total_tracks)) * 100 + else: + album_completion = 0 + logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) + + if album_completion >= headphones.ALBUM_COMPLETION_PCT: + new_album_status = "Downloaded" + else: + if album['Status'] == "Skipped" or album['Status'] == "Downloaded": + new_album_status = "Skipped" + else: + new_album_status = album['Status'] + myDB.upsert("albums", {'Status' : new_album_status}, {'AlbumID' : album['AlbumID']}) + if new_album_status != album['Status']: + logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + logger.info('Album status update complete') \ No newline at end of file diff --git a/headphones/mb.py b/headphones/mb.py index ce35d641..89578e18 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -19,7 +19,7 @@ import time import threading import headphones -from headphones import logger, db +from headphones import logger, db, helpers from headphones.helpers import multikeysort, replace_all import lib.musicbrainzngs as musicbrainzngs @@ -332,7 +332,9 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_all_releases(rgid,includeExtras=False): +def get_new_releases(rgid,includeExtras=False,forcefull=False): + + myDB = db.DBConnection() results = [] try: limit = 100 @@ -352,8 +354,29 @@ def get_all_releases(rgid,includeExtras=False): if not results or len(results) == 0: return False - - releases = [] + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage1 = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + if remove_missing_releases: + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage1 = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + num_new_releases = 0 + for releasedata in results: #releasedata.get will return None if it doesn't have a status #all official releases should have the Official status included @@ -361,36 +384,102 @@ def get_all_releases(rgid,includeExtras=False): continue release = {} - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker or forcefull: + #DELETE all references to this release since we're updating it anyway. + myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) + myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent - try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - releases.append(release) + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) - return releases + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) + + for track in release['Tracks']: + + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname + } + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) + if force_repackage1 == 1: + num_new_releases = -1 + logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) + else: + num_new_releases = num_new_releases + + return num_new_releases def getTracksFromRelease(release): totalTracks = 1 diff --git a/headphones/searcher.py b/headphones/searcher.py index a12c8cdd..478de86d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -141,6 +141,8 @@ def searchforalbum(albumid=None, new=False, lossless=False): if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none": searchTorrent(albumid, new, lossless) + logger.info('Search for Wanted albums complete') + def searchNZB(albumid=None, new=False, losslessOnly=False): myDB = db.DBConnection() diff --git a/headphones/updater.py b/headphones/updater.py index c80eec0b..6bd0450f 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -17,17 +17,16 @@ import headphones from headphones import logger, db, importer -def dbUpdate(): +def dbUpdate(forcefull=False): myDB = db.DBConnection() activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC') - logger.info('Starting update for %i active artists' % len(activeartists)) for artist in activeartists: artistid = artist[0] - importer.addArtisttoDB(artistid) + importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) - logger.info('Update complete') + logger.info('Active artist update complete') diff --git a/headphones/webserve.py b/headphones/webserve.py index 3477c293..3cd9242b 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -24,11 +24,14 @@ from mako import exceptions import time import threading +import string +import json +from operator import itemgetter import headphones -from headphones import logger, searcher, db, importer, mb, lastfm, librarysync -from headphones.helpers import checked, radio,today +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers +from headphones.helpers import checked, radio,today, cleanName import lib.simplejson as simplejson @@ -150,7 +153,7 @@ class WebInterface(object): newValueDict = {'IncludeExtras': 1, 'Extras': extras} myDB.upsert("artists", newValueDict, controlValueDict) - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) getExtras.exposed = True @@ -163,6 +166,8 @@ class WebInterface(object): for album in extraalbums: myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) removeExtras.exposed = True @@ -187,11 +192,15 @@ class WebInterface(object): def deleteArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() + namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) + for name in namecheck: + artistname=name['ArtistName'] myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True @@ -211,7 +220,7 @@ class WebInterface(object): deleteEmptyArtists.exposed = True def refreshArtist(self, ArtistID): - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) refreshArtist.exposed=True @@ -238,8 +247,15 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - def addArtists(self, **args): - threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + def addArtists(self, action=None, **args): + if action == "add": + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + if action == "ignore": + myDB = db.DBConnection() + for artist in args: + myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist]) + logger.info("Artist %s removed from new artist list and set to ignored" % artist) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True @@ -274,6 +290,8 @@ class WebInterface(object): myDB = db.DBConnection() myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -334,6 +352,193 @@ class WebInterface(object): return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists) manageNew.exposed = True + def manageUnmatched(self): + myDB = db.DBConnection() + have_album_dictionary = [] + headphones_album_dictionary = [] + unmatched_albums = [] + have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched IS NULL GROUP BY AlbumTitle ORDER BY ArtistName') + for albums in have_albums: + #Have to skip over manually matched tracks + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + # else: + # original_clean = None + if original_clean == albums['CleanName']: + have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_album_dictionary.append(have_dict) + headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') + for albums in headphones_albums: + headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_album_dictionary.append(headphones_dict) + #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] + + check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) + unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] + + + return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) + manageUnmatched.exposed = True + + def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None): + myDB = db.DBConnection() + + if action == "ignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched IS NULL', [artist]) + + elif action == "ignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched IS NULL', (artist, album)) + + elif action == "matchArtist": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + if new_artist_clean != existing_artist_clean: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist]) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_artist_clean): + new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) + logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist) + + elif action == "matchAlbum": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + existing_album_clean = helpers.cleanName(existing_album).lower() + new_album_clean = helpers.cleanName(new_album).lower() + existing_clean_string = existing_artist_clean+" "+existing_album_clean + new_clean_string = new_artist_clean+" "+new_album_clean + if existing_clean_string != new_clean_string: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album)) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_clean_string): + new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + album_id = match_tracks['AlbumID'] + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string)) + logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (update_count, new_artist, new_album)) + if update_count > 0: + librarysync.update_album_status(album_id) + else: + logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album)) + + markUnmatched.exposed = True + + def manageManual(self): + myDB = db.DBConnection() + manual_albums = [] + manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have') + for albums in manualalbums: + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + if albums['Matched'] == "Ignored": + album_status = "Ignored" + elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + album_status = "Matched" + manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status } + if manual_dict not in manual_albums: + manual_albums.append(manual_dict) + manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle')) + + return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted) + manageManual.exposed = True + + def markManual(self, action=None, existing_artist=None, existing_album=None): + myDB = db.DBConnection() + if action == "unignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND Matched="Ignored"', [artist]) + logger.info("Artist: %s successfully restored to unmatched list" % artist) + + elif action == "unignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album)) + logger.info("Album: %s successfully restored to unmatched list" % album) + + elif action == "unmatchArtist": + artist = existing_artist + update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', [artist]) + update_count = 0 + for tracks in update_clean: + original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + album = tracks['AlbumTitle'] + track_title = tracks['TrackTitle'] + if tracks['CleanName'] != original_clean: + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE have SET CleanName=?, Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + update_count+=1 + if update_count > 0: + librarysync.update_album_status() + logger.info("Artist: %s successfully restored to unmatched list" % artist) + + elif action == "unmatchAlbum": + artist = existing_artist + album = existing_album + update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) + update_count = 0 + for tracks in update_clean: + original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + track_title = tracks['TrackTitle'] + if tracks['CleanName'] != original_clean: + album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', [tracks['CleanName']]).fetchone() + if album_id_check: + album_id = album_id_check[0] + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE have SET CleanName=?, Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + update_count+=1 + if update_count > 0: + librarysync.update_album_status(album_id) + logger.info("Album: %s successfully restored to unmatched list" % album) + + markManual.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() artistsToAdd = [] @@ -342,6 +547,8 @@ class WebInterface(object): myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) elif action == 'pause': controlValueDict = {'ArtistID': ArtistID} @@ -397,10 +604,16 @@ class WebInterface(object): def forceUpdate(self): from headphones import updater - threading.Thread(target=updater.dbUpdate).start() + threading.Thread(target=updater.dbUpdate, args=[False]).start() raise cherrypy.HTTPRedirect("home") forceUpdate.exposed = True + def forceFullUpdate(self): + from headphones import updater + threading.Thread(target=updater.dbUpdate, args=[True]).start() + raise cherrypy.HTTPRedirect("home") + forceFullUpdate.exposed = True + def forceSearch(self): from headphones import searcher threading.Thread(target=searcher.searchforalbum).start() @@ -533,6 +746,20 @@ class WebInterface(object): return s getArtists_json.exposed=True + def getAlbumsByArtist_json(self, artist=None): + myDB = db.DBConnection() + album_json = {} + counter = 0 + album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) + for album in album_list: + album_json[counter] = album['AlbumTitle'] + counter+=1 + json_albums = json.dumps(album_json) + + cherrypy.response.headers['Content-type'] = 'application/json' + return json_albums + getAlbumsByArtist_json.exposed=True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': @@ -554,6 +781,26 @@ class WebInterface(object): generateAPI.exposed = True + def forceScan(self, keepmatched=None): + myDB = db.DBConnection() + ######################################### + #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE + myDB.select('DELETE from Have') + logger.info('Removed all entries in local library database') + myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL') + myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL') + logger.info('All tracks in library unmatched') + myDB.action('UPDATE artists SET HaveTracks=NULL') + logger.info('Reset track counts for all artists') + myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"') + logger.info('Marking all unwanted albums as Skipped') + try: + threading.Thread(target=librarysync.libraryScan).start() + except Exception, e: + logger.error('Unable to complete the scan: %s' % e) + raise cherrypy.HTTPRedirect("home") + forceScan.exposed = True + def config(self): interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/') @@ -571,6 +818,8 @@ class WebInterface(object): "api_enabled" : checked(headphones.API_ENABLED), "api_key" : headphones.API_KEY, "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL, + "update_db_interval" : headphones.UPDATE_DB_INTERVAL, + "mb_ignore_age" : headphones.MB_IGNORE_AGE, "nzb_search_interval" : headphones.SEARCH_INTERVAL, "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL, "sab_host" : headphones.SAB_HOST, @@ -720,7 +969,7 @@ class WebInterface(object): config.exposed = True def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, - download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, + download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, @@ -747,6 +996,8 @@ class WebInterface(object): headphones.API_ENABLED = api_enabled headphones.API_KEY = api_key headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval + headphones.UPDATE_DB_INTERVAL = update_db_interval + headphones.MB_IGNORE_AGE = mb_ignore_age headphones.SEARCH_INTERVAL = nzb_search_interval headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval headphones.SAB_HOST = sab_host