diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index bec42cfb..c3de67cb 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -21,7 +21,7 @@ +
+ omgwtfnzbs +
+ +
+
+
+ + +
+
+ + +
+
+
@@ -1259,6 +1283,7 @@ initConfigCheckbox("#usenewznab"); initConfigCheckbox("#usenzbsrus"); initConfigCheckbox("#usenzbsorg"); + initConfigCheckbox("#useomgwtfnzbs"); initConfigCheckbox("#usepiratebay"); initConfigCheckbox("#usewaffles"); initConfigCheckbox("#userutracker"); diff --git a/data/interfaces/default/js/script.js b/data/interfaces/default/js/script.js index e021074b..a0bb860e 100644 --- a/data/interfaces/default/js/script.js +++ b/data/interfaces/default/js/script.js @@ -315,6 +315,7 @@ function doAjaxCall(url,elem,reload,form) { console.log('refresh'); refreshTable(); } if ( reload == "tabs") refreshTab(); + if ( reload == "page") location.reload(); if ( form ) { // Change the option to 'choose...' $(formID + " select").children('option[disabled=disabled]').attr('selected','selected'); diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 66405aab..6d98bb53 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -22,6 +22,7 @@ %if not headphones.ADD_ARTISTS: Manage New Artists %endif + Manage Unmatched @@ -35,6 +36,7 @@
  • Scan Music Library
  • Imports
  • Force Actions
    • +
  • Force Legacy
    • @@ -119,7 +121,7 @@ Force Search
    Force Check for Wanted Albums - Force Update Active Artists + Force Update Active Artists [Fast] Force Post-Process Albums in Download Folder Check for Headphones Updates Delete empty Artists @@ -139,6 +141,26 @@
    + + +
    +
    + Force Legacy +

    Please note that these functions will take a significant amount of time to complete.

    +
    + Force Update Active Artists [Comprehensive] +
    + Force Re-scan Library [Comprehensive] +
    + *Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched". + +
    +
    + +
    + + + <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/managealbums.html b/data/interfaces/default/managealbums.html index a30dde6a..02201131 100644 --- a/data/interfaces/default/managealbums.html +++ b/data/interfaces/default/managealbums.html @@ -57,7 +57,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/managemanual.html b/data/interfaces/default/managemanual.html new file mode 100644 index 00000000..e20882ca --- /dev/null +++ b/data/interfaces/default/managemanual.html @@ -0,0 +1,121 @@ +<%inherit file="base.html" /> +<%! + import headphones + from headphones import db, helpers + myDB = db.DBConnection() +%> + +<%def name="headerIncludes()"> +
    +
    +
    +
    + « Back to Unmatched Albums + + + +<%def name="body()"> +
    +
    +

    manageManage Manually Changed Albums

    +
    + + + + + + + + + + + <% count_albums=0 %> + %for album in manualalbums: + + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace('+', '%2B').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace('+', '%2B').replace("'","%27") + %> + + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal AlbumPrevious Action
    ${album['ArtistName']}
    + + +     		${album['AlbumTitle']}
    + + + +     		${album['AlbumStatus']} + +
    +
    + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html index 17d567f7..41f9263d 100644 --- a/data/interfaces/default/managenew.html +++ b/data/interfaces/default/managenew.html @@ -20,7 +20,10 @@
    - Add selected artists +
    diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html new file mode 100644 index 00000000..3fdc7eaa --- /dev/null +++ b/data/interfaces/default/manageunmatched.html @@ -0,0 +1,259 @@ +<%inherit file="base.html" /> +<%! + import headphones + import json + from headphones import db, helpers + myDB = db.DBConnection() + artist_json = {} + counter = 0 + artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE") + for artist in artist_list: + artist_json[counter] = artist['ArtistName'] + counter+=1 + json_artists = json.dumps(artist_json) +%> + +<%def name="headerIncludes()"> + + « Back to manage overview + + + +<%def name="body()"> +
    +
    +

    manageManage Unmatched Albums

    +
    + +
    + + + + + + + + <% count_albums=0 %> + %for album in unmatchedalbums: + + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace("'","%27") + old_artist_js = album['ArtistName'].replace("'","\\'").replace('"','\\"') + old_album_js = album['AlbumTitle'].replace("'","\\'").replace('"','\\"') + %> + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal Album
    ${album['ArtistName']}
    + + + + + + +     		${album['AlbumTitle']}
    + + + + + + +
    + + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/headphones/__init__.py b/headphones/__init__.py index 234d2cea..7f567afc 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -130,6 +130,8 @@ SEARCH_INTERVAL = 360 LIBRARYSCAN = False LIBRARYSCAN_INTERVAL = 300 DOWNLOAD_SCAN_INTERVAL = 5 +UPDATE_DB_INTERVAL = 24 +MB_IGNORE_AGE = 365 SAB_HOST = None SAB_USERNAME = None @@ -166,6 +168,10 @@ NZBSRUS = False NZBSRUS_UID = None NZBSRUS_APIKEY = None +OMGWTFNZBS = False +OMGWTFNZBS_UID = None +OMGWTFNZBS_APIKEY = None + PREFERRED_WORDS = None IGNORED_WORDS = None REQUIRED_WORDS = None @@ -302,11 +308,12 @@ def initialize(): ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \ TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, PIRATEBAY_PROXY_URL, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \ RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \ UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \ - LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \ + NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \ + NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \ + INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \ MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, \ PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \ MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \ @@ -326,6 +333,7 @@ def initialize(): CheckSection('Newznab') CheckSection('NZBsorg') CheckSection('NZBsRus') + CheckSection('omgwtfnzbs') CheckSection('Waffles') CheckSection('Rutracker') CheckSection('What.cd') @@ -405,6 +413,8 @@ def initialize(): LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300) DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) + UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24) + MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365) TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '') NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10') @@ -465,6 +475,10 @@ def initialize(): NZBSRUS_UID = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_uid', '') NZBSRUS_APIKEY = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_apikey', '') + OMGWTFNZBS = bool(check_setting_int(CFG, 'omgwtfnzbs', 'omgwtfnzbs', 0)) + OMGWTFNZBS_UID = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_uid', '') + OMGWTFNZBS_APIKEY = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_apikey', '') + PREFERRED_WORDS = check_setting_str(CFG, 'General', 'preferred_words', '') IGNORED_WORDS = check_setting_str(CFG, 'General', 'ignored_words', '') REQUIRED_WORDS = check_setting_str(CFG, 'General', 'required_words', '') @@ -792,6 +806,8 @@ def config_write(): new_config['General']['libraryscan'] = int(LIBRARYSCAN) new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL + new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL + new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE new_config['SABnzbd'] = {} new_config['SABnzbd']['sab_host'] = SAB_HOST @@ -842,6 +858,11 @@ def config_write(): new_config['NZBsRus']['nzbsrus_uid'] = NZBSRUS_UID new_config['NZBsRus']['nzbsrus_apikey'] = NZBSRUS_APIKEY + new_config['omgwtfnzbs'] = {} + new_config['omgwtfnzbs']['omgwtfnzbs'] = int(OMGWTFNZBS) + new_config['omgwtfnzbs']['omgwtfnzbs_uid'] = OMGWTFNZBS_UID + new_config['omgwtfnzbs']['omgwtfnzbs_apikey'] = OMGWTFNZBS_APIKEY + new_config['General']['preferred_words'] = PREFERRED_WORDS new_config['General']['ignored_words'] = IGNORED_WORDS new_config['General']['required_words'] = REQUIRED_WORDS @@ -917,9 +938,9 @@ def start(): # Start our scheduled background tasks from headphones import updater, searcher, librarysync, postprocessor - SCHED.add_interval_job(updater.dbUpdate, hours=24) + SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL) SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL) - SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) + SCHED.add_interval_job(librarysync.libraryScan, hours=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) if CHECK_GITHUB: SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL) @@ -953,6 +974,19 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + #Below creates indices to speed up Active Artist updating + c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') + c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') + c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)') + #Below creates indices to speed up library scanning & matching + c.execute('CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)') try: c.execute('SELECT IncludeExtras from artists') @@ -1118,6 +1152,7 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') + conn.commit() c.close() diff --git a/headphones/api.py b/headphones/api.py index 16a22229..7754f3f6 100644 --- a/headphones/api.py +++ b/headphones/api.py @@ -123,8 +123,9 @@ class Api(object): artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"') albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') + description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"') - self.data = { 'artist': artist, 'albums': albums } + self.data = { 'artist': artist, 'albums': albums, 'description' : description } return def _getAlbum(self, **kwargs): diff --git a/headphones/importer.py b/headphones/importer.py index 0365a872..b78c69ab 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -15,6 +15,7 @@ from lib.pyItunes import * import time +import threading import os from lib.beets.mediafile import MediaFile @@ -108,7 +109,7 @@ def addArtistIDListToDB(artistidlist): for artistid in artistidlist: addArtisttoDB(artistid) -def addArtisttoDB(artistid, extrasonly=False): +def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache @@ -189,65 +190,160 @@ def addArtisttoDB(artistid, extrasonly=False): db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: - includeExtras = False + includeExtras = False + + #Clean all references to release group in dB that are no longer referenced in musicbrainz + group_list = [] + force_repackage = 0 + #Don't nuke the database if there's a MusicBrainz error + if len(artist['releasegroups']) != 0 and not extrasonly: + for groups in artist['releasegroups']: + group_list.append(groups['id']) + remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + for items in remove_missing_groups_from_albums: + if items['AlbumID'] not in group_list: + # Remove all from albums/tracks that aren't in release groups + myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) + logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID'])) + force_repackage = 1 + else: + logger.info("[%s] Error pulling data from MusicBrainz: Maintaining dB" % artist['artist_name']) + + # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks + for rg in artist['releasegroups']: - logger.info("Now adding/updating: " + rg['title']) - + al_title = rg['title'] + today = helpers.today() rgid = rg['id'] - - # check if the album already exists - rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - - releases = mb.get_all_releases(rgid,includeExtras) - if releases == []: - logger.info('No official releases in release group %s' % rg['title']) - continue - if not releases: - errors = True - logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) - continue - - # This will be used later to build a hybrid release - fullreleaselist = [] + skip_log = 0 + #Make a user configurable variable to skip update of albums with release dates older than this date (in days) + pause_delta = headphones.MB_IGNORE_AGE - for release in releases: - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + if not forcefull: + check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() + if check_release_date: + if check_release_date[0] is None: + logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) + new_releases = mb.get_new_releases(rgid,includeExtras,True) + else: + if len(check_release_date[0]) == 10: + release_date = check_release_date[0] + elif len(check_release_date[0]) == 7: + release_date = check_release_date[0]+"-31" + elif len(check_release_date[0]) == 4: + release_date = check_release_date[0]+"-12-31" + else: + release_date = today + if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: + logger.info("[%s] Now updating: %s (Release Date <%s Days) " % (artist['artist_name'], rg['title'], pause_delta)) + new_releases = mb.get_new_releases(rgid,includeExtras,True) + else: + logger.info("[%s] Skipping: %s (Release Date >%s Days)" % (artist['artist_name'], rg['title'], pause_delta)) + skip_log = 1 + new_releases = 0 + else: + logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) + new_releases = mb.get_new_releases(rgid,includeExtras) - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + if force_repackage == 1: + new_releases = -1 + logger.info('[%s] Forcing repackage of %s (Release Group Removed)' % (artist['artist_name'], al_title)) + else: + new_releases = new_releases + else: + logger.info("[%s] Now adding/updating: %s (Comprehensive Force)" % (artist['artist_name'], rg['title'])) + new_releases = mb.get_new_releases(rgid,includeExtras,forcefull) + + #What this does is adds new releases per artist to the allalbums + alltracks databases + #new_releases = mb.get_new_releases(rgid,includeExtras) + #print al_title + #print new_releases + + if new_releases != 0: + #Dump existing hybrid release since we're repackaging/replacing it + myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) + # This will be used later to build a hybrid release + fullreleaselist = [] + #Search for releases within a release group + find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) + # Build the dictionary for the fullreleaselist + for items in find_hybrid_releases: + if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing + hybrid_release_id = items['ReleaseID'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] + } + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) + totalTracks = 1 + hybrid_track_array = [] + for hybrid_tracks in find_hybrid_tracks: + hybrid_track_array.append({ + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], + #'url': hybrid_tracks['TrackURL'], + 'duration': hybrid_tracks['TrackDuration'] + }) + totalTracks += 1 + newValueDict['ReleaseID'] = hybrid_release_id + newValueDict['Tracks'] = hybrid_track_array + fullreleaselist.append(newValueDict) + + #print fullreleaselist + + # Basically just do the same thing again for the hybrid release + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) + except Exception, e: + errors = True + logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e)) + continue + + # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it + # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it + # The hybrid won't have a country or a format + controlValueDict = {"ReleaseID": rg['id']} + + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumID": rg['id'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "ReleaseDate": hybridrelease['ReleaseDate'], + "Type": rg['type'] } - + myDB.upsert("allalbums", newValueDict, controlValueDict) - # Build the dictionary for the fullreleaselist - newValueDict['ReleaseID'] = release['ReleaseID'] - newValueDict['Tracks'] = release['Tracks'] - fullreleaselist.append(newValueDict) - - for track in release['Tracks']: + for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} + "ReleaseID": rg['id']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], @@ -258,184 +354,117 @@ def addArtisttoDB(artistid, extrasonly=False): if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Basically just do the same thing again for the hybrid release - # This may end up being called with an empty fullreleaselist - try: - hybridrelease = getHybridRelease(fullreleaselist) - except Exception, e: - errors = True - logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) - continue - - # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it - # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it - # The hybrid won't have a country or a format - controlValueDict = {"ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "ReleaseDate": hybridrelease['ReleaseDate'], - "Type": rg['type'] - } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in hybridrelease['Tracks']: - - cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) - - controlValueDict = {"TrackID": track['id'], - "ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "AlbumID": rg['id'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } - - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') - - # If there's no release in the main albums tables, add the default (hybrid) - # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) - if not rg_exists: - releaseid = rg['id'] - elif rg_exists and not rg_exists['ReleaseID']: - # Need to do some importing here - to transition the old format of using the release group - # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID - # so we'll need to move over the locations, bitrates & formats from the tracks table to the new - # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups - logger.info("Copying current track information to alternate releases") - tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() - for track in tracks: - if track['Location']: - controlValueDict = {"TrackID": track['TrackID']} - newValueDict = {"Location": track['Location'], - "BitRate": track['BitRate'], - "Format": track['Format'], - } - myDB.upsert("alltracks", newValueDict, controlValueDict) - releaseid = rg['id'] - else: - releaseid = rg_exists['ReleaseID'] - - album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - - controlValueDict = {"AlbumID": rg['id']} - - newValueDict = {"ArtistID": album['ArtistID'], - "ArtistName": album['ArtistName'], - "AlbumTitle": album['AlbumTitle'], - "ReleaseID": album['ReleaseID'], - "AlbumASIN": album['AlbumASIN'], - "ReleaseDate": album['ReleaseDate'], - "Type": album['Type'], - "ReleaseCountry": album['ReleaseCountry'], - "ReleaseFormat": album['ReleaseFormat'] - } - if not rg_exists: + # Delete matched tracks from the have table + #myDB.action('DELETE from have WHERE Matched="True"') - today = helpers.today() - - newValueDict['DateAdded']= today - - if headphones.AUTOWANT_ALL: - newValueDict['Status'] = "Wanted" - elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" - # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these - # The first test just makes sure we have year-month-day - elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" + # If there's no release in the main albums tables, add the default (hybrid) + # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) + # check if the album already exists + rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() + if not rg_exists: + releaseid = rg['id'] else: - newValueDict['Status'] = "Skipped" - - myDB.upsert("albums", newValueDict, controlValueDict) + releaseid = rg_exists['ReleaseID'] + + album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) - tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + controlValueDict = {"AlbumID": rg['id']} - # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT - total_track_count = len(tracks) - - for track in tracks: - - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": rg['id']} - - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] + newValueDict = {"ArtistID": album['ArtistID'], + "ArtistName": album['ArtistName'], + "AlbumTitle": album['AlbumTitle'], + "ReleaseID": album['ReleaseID'], + "AlbumASIN": album['AlbumASIN'], + "ReleaseDate": album['ReleaseDate'], + "Type": album['Type'], + "ReleaseCountry": album['ReleaseCountry'], + "ReleaseFormat": album['ReleaseFormat'] } - - myDB.upsert("tracks", newValueDict, controlValueDict) + + if not rg_exists: + + today = helpers.today() + + newValueDict['DateAdded']= today + + if headphones.AUTOWANT_ALL: + newValueDict['Status'] = "Wanted" + elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these + # The first test just makes sure we have year-month-day + elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + else: + newValueDict['Status'] = "Skipped" + + myDB.upsert("albums", newValueDict, controlValueDict) - # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album - have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) - marked_as_downloaded = False - - if rg_exists: - if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True + tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + + # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT + total_track_count = len(tracks) + + for track in tracks: + + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": rg['id']} + + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] + } + + myDB.upsert("tracks", newValueDict, controlValueDict) + + # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album + have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) + marked_as_downloaded = False + + if rg_exists: + if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + else: + if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + + logger.info(u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) + cache.getThumb(AlbumID=rg['id']) + + #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: + if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: + from headphones import searcher + searcher.searchforalbum(albumid=rg['id']) else: - if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True - - logger.info(u"Seeing if we need album art for " + rg['title']) - cache.getThumb(AlbumID=rg['id']) - - #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: - if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: - from headphones import searcher - searcher.searchforalbum(albumid=rg['id']) + if skip_log == 0: + logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title'])) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist['artist_name']])) controlValueDict = {"ArtistID": artistid} @@ -456,13 +485,15 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("artists", newValueDict, controlValueDict) - logger.info(u"Seeing if we need album art for: " + artist['artist_name']) + logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) cache.getThumb(ArtistID=artistid) if errors: - logger.info("Finished updating artist: " + artist['artist_name'] + " but with errors, so not marking it as updated in the database") + logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name'])) else: - logger.info(u"Updating complete for: " + artist['artist_name']) + myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) + logger.info(u"Updating complete for: %s" % artist['artist_name']) + def addReleaseById(rid): @@ -562,14 +593,14 @@ def addReleaseById(rid): if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 5ee57d8b..843eaf11 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): + if cron and not headphones.LIBRARYSCAN: return @@ -43,23 +44,39 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal return myDB = db.DBConnection() + new_artists = [] + + logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths - tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') - + tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') + for track in tracks: - if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + + del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') - myDB.action('DELETE from have') + for track in del_have_tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + if track['ArtistName']: + #Make sure deleted files get accounted for when updating artist track counts + new_artists.append(track['ArtistName']) + myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) + ###############myDB.action('DELETE from have') - logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - - new_artists = [] bitrates = [] song_list = [] + new_song_count = 0 + file_count = 0 + + latest_subdirectory = [] for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one @@ -68,9 +85,17 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if directory.startswith("."): d.remove(directory) for files in f: + # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): + subdirectory = r.replace(dir,'') + latest_subdirectory.append(subdirectory) + if file_count == 0 and r.replace(dir,'') !='': + logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) + elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0: + logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) + song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database @@ -99,8 +124,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) - song_dict = { 'TrackID' : f.mb_trackid, - 'ReleaseID' : f.mb_albumid, + if f_artist and f.album and f.title: + CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) + else: + CleanName = None + + controlValueDict = {'Location' : unicode_song_path} + + newValueDict = { 'TrackID' : f.mb_trackid, + #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, @@ -110,224 +142,169 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, - 'Location' : unicode_song_path } + 'CleanName' : CleanName + } - song_list.append(song_dict) + #song_list.append(song_dict) + check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() + #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. + if not check_exist_song: + #This is a new track + if f_artist: + new_artists.append(f_artist) + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + else: + if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: + #Important track metadata has been modified, need to run matcher again + if f_artist and f_artist != check_exist_song['ArtistName']: + new_artists.append(f_artist) + elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored": + new_artists.append(f_artist) + else: + continue + + newValueDict['Matched'] = None + myDB.upsert("have", newValueDict, controlValueDict) + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) + new_song_count+=1 + else: + #This track information hasn't changed + if f_artist and check_exist_song['Matched'] != "Ignored": + new_artists.append(f_artist) + + file_count+=1 + # Now we start track matching - total_number_of_songs = len(song_list) - logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") + logger.info("%s new/modified songs found and added to the database" % new_song_count) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) + total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] + logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches - song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 + latest_artist = [] for song in song_list: + + latest_artist.append(song['ArtistName']) + if song_count == 0: + logger.info("Now matching songs by %s" % song['ArtistName']) + elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: + logger.info("Now matching songs by %s" % song['ArtistName']) + #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") - # If the track has a trackid & releaseid (beets: albumid) that the most surefire way - # of identifying a track to a specific release so we'll use that first - if song['TrackID'] and song['ReleaseID']: + #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT + #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, + #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM + #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. - # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - if track: - # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Matched. Move on to the next one: - continue - - # If we can't find it with TrackID & ReleaseID, next most specific will be - # releaseid + tracktitle, although perhaps less reliable due to a higher - # likelihood of variations in the song title (e.g. feat. artists) - if song['ReleaseID'] and song['TrackTitle']: - - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if track: - # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) - # So use both TrackID & ReleaseID as the control values - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both tables here as well - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Done - continue - - # Next most specific will be the opposite: a TrackID and an AlbumTitle - # TrackIDs span multiple releases so if something is on an official album - # and a compilation, for example, this will match it to the right one - # However - there may be multiple matches here - if song['TrackID'] and song['AlbumTitle']: - - # Even though there might be multiple matches, we just need to grab one to confirm a match - track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if track: - # Don't need the hybridControlValueDict here since ReleaseID is not unique - controlValueDict = { 'TrackID' : track['TrackID'], - 'AlbumTitle' : track['AlbumTitle'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue - - # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably - # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], - 'TrackTitle' : track['TrackTitle'] } - + 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) - # This is more reliable than the former but requires some string manipulation so we'll do it only - # if we can't find a match with the original data - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - - track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if not track: - track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if track: - controlValueDict = { 'CleanName' : track['CleanName'] } - + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if track: + controlValueDict = { 'CleanName' : track['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("tracks", newValueDict, controlValueDict) + + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) + + + alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + if alltrack: + controlValueDict = { 'ArtistName' : alltrack['ArtistName'], + 'AlbumTitle' : alltrack['AlbumTitle'], + 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable - # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track - # with more specific information exists in the library, it'll overwrite these values - if song['TrackID']: - - track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if track: - controlValueDict = { 'TrackID' : track['TrackID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if alltrack: + controlValueDict = { 'CleanName' : alltrack['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue - - # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release - if song['ArtistName']: - new_artists.append(song['ArtistName']) + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) else: - continue + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) - # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - else: - continue - - myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - + if not append: + logger.info('Updating scanned artist track counts') + # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') - - artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] - + + #There was a bug where artists with special characters (-,') would show up in new artists. + artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] + artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]] + # Update track counts - logger.info('Updating current artist track counts') - for artist in current_artists: + for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) + #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases + # (can fix by getting rid of second len statement) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) @@ -337,9 +314,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') - myDB.action('DELETE from newartists') + #myDB.action('DELETE from newartists') for artist in artist_list: - myDB.action('INSERT into newartists VALUES (?)', [artist]) + myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 @@ -348,6 +325,42 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) - + + update_album_status() + logger.info('Library scan complete') + + #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED +def update_album_status(AlbumID=None): + myDB = db.DBConnection() + logger.info('Counting matched tracks to mark albums as skipped/downloaded') + if AlbumID: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + else: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') + for album in album_status_updater: + track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']]) + total_tracks = 0 + have_tracks = 0 + for track in track_counter: + total_tracks+=1 + if track['Location']: + have_tracks+=1 + if total_tracks != 0: + album_completion = float(float(have_tracks) / float(total_tracks)) * 100 + else: + album_completion = 0 + logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) + + if album_completion >= headphones.ALBUM_COMPLETION_PCT: + new_album_status = "Downloaded" + else: + if album['Status'] == "Skipped" or album['Status'] == "Downloaded": + new_album_status = "Skipped" + else: + new_album_status = album['Status'] + myDB.upsert("albums", {'Status' : new_album_status}, {'AlbumID' : album['AlbumID']}) + if new_album_status != album['Status']: + logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + logger.info('Album status update complete') \ No newline at end of file diff --git a/headphones/mb.py b/headphones/mb.py index ce35d641..e7629bba 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -19,7 +19,7 @@ import time import threading import headphones -from headphones import logger, db +from headphones import logger, db, helpers from headphones.helpers import multikeysort, replace_all import lib.musicbrainzngs as musicbrainzngs @@ -332,7 +332,9 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_all_releases(rgid,includeExtras=False): +def get_new_releases(rgid,includeExtras=False,forcefull=False): + + myDB = db.DBConnection() results = [] try: limit = 100 @@ -352,8 +354,29 @@ def get_all_releases(rgid,includeExtras=False): if not results or len(results) == 0: return False - - releases = [] + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage1 = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + if remove_missing_releases: + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage1 = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + num_new_releases = 0 + for releasedata in results: #releasedata.get will return None if it doesn't have a status #all official releases should have the Official status included @@ -361,45 +384,130 @@ def get_all_releases(rgid,includeExtras=False): continue release = {} - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker or forcefull: + #DELETE all references to this release since we're updating it anyway. + myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) + myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent - try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - releases.append(release) + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + additional_medium='' + for position in releasedata['medium-list']: + if position['format'] == releasedata['medium-list'][0]['format']: + medium_count = int(position['position']) + else: + additional_medium = additional_medium+' + '+position['format'] + if medium_count == 1: + disc_number = '' + else: + disc_number = str(medium_count)+'x' + packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium + release['ReleaseFormat'] = unicode(packaged_medium) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) - return releases + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) + + for track in release['Tracks']: + + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname + } + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location'])) + + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + if album_checker: + logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) + else: + logger.info('[%s] New release %s (%s) added' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) + if force_repackage1 == 1: + num_new_releases = -1 + logger.info('[%s] Forcing repackage of %s, since dB releases have been removed' % (release['ArtistName'], release_title)) + else: + num_new_releases = num_new_releases + + return num_new_releases def getTracksFromRelease(release): totalTracks = 1 tracks = [] for medium in release['medium-list']: for track in medium['track-list']: + try: + track_title = unicode(track['title']) + except: + track_title = unicode(track['recording']['title']) tracks.append({ 'number': totalTracks, - 'title': unicode(track['recording']['title']), + 'title': track_title, 'id': unicode(track['recording']['id']), 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], 'duration': int(track['length']) if 'length' in track else 0 diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 2c54974c..e37f2d43 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -244,7 +244,16 @@ def command(encoder,musicSource,musicDest,albumPath): logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'))) logger.debug(subprocess.list2cmdline(cmd)) - p = subprocess.Popen(cmd, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # stop windows opening the cmd + startupinfo = None + if headphones.SYS_PLATFORM == "win32": + startupinfo = subprocess.STARTUPINFO() + try: + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + except AttributeError: + startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW + + p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate(headphones.ENCODER) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 368bd770..48e6c3ec 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -533,7 +533,7 @@ def moveFiles(albumpath, release, tracks): } folder = helpers.replace_all(headphones.FOLDER_FORMAT.strip(), values) - folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_') + folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_').replace('|','_') if folder.endswith('.'): folder = folder.replace(folder[len(folder)-1], '_') @@ -843,6 +843,7 @@ def renameFiles(albumpath, downloaded_track_list, release): new_file_name = new_file_name.replace('?','_').replace(':', '_').encode(headphones.SYS_ENCODING, 'replace') + new_file_name = new_file_name.replace('*','_') if headphones.FILE_UNDERSCORES: new_file_name = new_file_name.replace(' ', '_') diff --git a/headphones/searcher.py b/headphones/searcher.py index a12c8cdd..108feae4 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -118,7 +118,7 @@ def searchforalbum(albumid=None, new=False, lossless=False): for result in results: foundNZB = "none" - if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): + if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): if result['Status'] == "Wanted Lossless": foundNZB = searchNZB(result['AlbumID'], new, losslessOnly=True) else: @@ -135,12 +135,14 @@ def searchforalbum(albumid=None, new=False, lossless=False): foundNZB = "none" - if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): + if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): foundNZB = searchNZB(albumid, new, lossless) if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none": searchTorrent(albumid, new, lossless) + logger.info('Search for Wanted albums complete') + def searchNZB(albumid=None, new=False, losslessOnly=False): myDB = db.DBConnection() @@ -437,6 +439,65 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): except Exception, e: logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) + + if headphones.OMGWTFNZBS: + + provider = "omgwtfnzbs" + + if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + categories = "22" + elif headphones.PREFERRED_QUALITY: + categories = "22,7" + else: + categories = "7" + + if albums['Type'] == 'Other': + categories = "29" + logger.info("Album type is audiobook/spokenword. Searching all music categories") + + params = { "user": headphones.OMGWTFNZBS_UID, + "api": headphones.OMGWTFNZBS_APIKEY, + "catid": categories, + "retention": headphones.USENET_RETENTION, + "search": term + } + + searchURL = 'http://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params) + + # Add a user-agent + request = urllib2.Request(searchURL) + request.add_header('User-Agent', 'headphones/0.0 +https://github.com/rembo10/headphones') + opener = urllib2.build_opener() + + logger.info(u'Parsing results from omgwtfnzbs' % searchURL) + + try: + data = opener.open(request).read() + except Exception, e: + logger.warn('Error fetching data from omgwtfnzbs: %s' % e) + data = False + + if data: + + d = json.loads(data) + + if 'notice' in data: + logger.info(u"No results returned from omgwtfnzbs: %s" % d['notice']) + pass + + else: + for item in d: + try: + url = item['getnzb'] + title = item['release'] + size = int(item['sizebytes']) + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + + except Exception, e: + logger.error(u"An unknown error occurred trying to parse the results: %s" % e) + # attempt to verify that this isn't a substring result # when looking for "Foo - Foo" we don't want "Foobar" # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist @@ -944,7 +1005,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): try: title = item.title desc_match = re.search(r"Size: (\d+)<", item.description) - size = desc_match.group(1) + size = int(desc_match.group(1)) url = item.link resultlist.append((title, size, url, provider)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) @@ -1009,11 +1070,11 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): bitrate = None bitrate_string = bitrate - if headphones.PREFERRED_QUALITY == 2 or losslessOnly: # Lossless Only mode + if headphones.PREFERRED_QUALITY == 3 or losslessOnly: # Lossless Only mode search_formats = [gazelleformat.FLAC] maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 3: # Preferred quality mode - search_formats=[None] # should return all + elif headphones.PREFERRED_QUALITY == 2: # Preferred quality mode + search_formats = [None] # should return all bitrate = headphones.PREFERRED_BITRATE if bitrate: for encoding_string in gazelleencoding.ALL_ENCODINGS: diff --git a/headphones/transmission.py b/headphones/transmission.py index 9b31bd96..f0b477e3 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -20,6 +20,7 @@ import urllib2 import lib.simplejson as json import base64 import time +import re # This is just a simple script to send torrents to transmission. The # intention is to turn this into a class where we can check the state @@ -83,8 +84,13 @@ def torrentAction(method, arguments): if host.endswith('/'): host = host[:-1] - - host = host + "/transmission/rpc" + + if not host.endswith('/rpc'): + if host.endswith(':\d{2,6}'): + host = host + "/transmission/rpc" + else: + host = host + "/rpc" + request = urllib2.Request(host) if username and password: base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') diff --git a/headphones/updater.py b/headphones/updater.py index c80eec0b..6bd0450f 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -17,17 +17,16 @@ import headphones from headphones import logger, db, importer -def dbUpdate(): +def dbUpdate(forcefull=False): myDB = db.DBConnection() activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC') - logger.info('Starting update for %i active artists' % len(activeartists)) for artist in activeartists: artistid = artist[0] - importer.addArtisttoDB(artistid) + importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) - logger.info('Update complete') + logger.info('Active artist update complete') diff --git a/headphones/webserve.py b/headphones/webserve.py index 3477c293..130a17b6 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -24,11 +24,14 @@ from mako import exceptions import time import threading +import string +import json +from operator import itemgetter import headphones -from headphones import logger, searcher, db, importer, mb, lastfm, librarysync -from headphones.helpers import checked, radio,today +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers +from headphones.helpers import checked, radio,today, cleanName import lib.simplejson as simplejson @@ -105,7 +108,7 @@ class WebInterface(object): def albumPage(self, AlbumID): myDB = db.DBConnection() album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() - tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [AlbumID]) + tracks = myDB.select('SELECT * from tracks WHERE AlbumID=? ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID]) description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=?', [AlbumID]).fetchone() title = album['ArtistName'] + ' - ' + album['AlbumTitle'] return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description) @@ -150,7 +153,7 @@ class WebInterface(object): newValueDict = {'IncludeExtras': 1, 'Extras': extras} myDB.upsert("artists", newValueDict, controlValueDict) - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) getExtras.exposed = True @@ -163,6 +166,8 @@ class WebInterface(object): for album in extraalbums: myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) removeExtras.exposed = True @@ -187,11 +192,15 @@ class WebInterface(object): def deleteArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() + namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) + for name in namecheck: + artistname=name['ArtistName'] myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True @@ -211,7 +220,7 @@ class WebInterface(object): deleteEmptyArtists.exposed = True def refreshArtist(self, ArtistID): - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) refreshArtist.exposed=True @@ -238,8 +247,15 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - def addArtists(self, **args): - threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + def addArtists(self, action=None, **args): + if action == "add": + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + if action == "ignore": + myDB = db.DBConnection() + for artist in args: + myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist]) + logger.info("Artist %s removed from new artist list and set to ignored" % artist) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True @@ -274,6 +290,8 @@ class WebInterface(object): myDB = db.DBConnection() myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -334,6 +352,193 @@ class WebInterface(object): return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists) manageNew.exposed = True + def manageUnmatched(self): + myDB = db.DBConnection() + have_album_dictionary = [] + headphones_album_dictionary = [] + unmatched_albums = [] + have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName') + for albums in have_albums: + #Have to skip over manually matched tracks + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + # else: + # original_clean = None + if original_clean == albums['CleanName']: + have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_album_dictionary.append(have_dict) + headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') + for albums in headphones_albums: + headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_album_dictionary.append(headphones_dict) + #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] + + check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) + unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] + + + return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) + manageUnmatched.exposed = True + + def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None): + myDB = db.DBConnection() + + if action == "ignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched = "Failed"', [artist]) + + elif action == "ignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched = "Failed"', (artist, album)) + + elif action == "matchArtist": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + if new_artist_clean != existing_artist_clean: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist]) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_artist_clean): + new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) + logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist) + + elif action == "matchAlbum": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + existing_album_clean = helpers.cleanName(existing_album).lower() + new_album_clean = helpers.cleanName(new_album).lower() + existing_clean_string = existing_artist_clean+" "+existing_album_clean + new_clean_string = new_artist_clean+" "+new_album_clean + if existing_clean_string != new_clean_string: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album)) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_clean_string): + new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + album_id = match_tracks['AlbumID'] + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string)) + logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (update_count, new_artist, new_album)) + if update_count > 0: + librarysync.update_album_status(album_id) + else: + logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album)) + + markUnmatched.exposed = True + + def manageManual(self): + myDB = db.DBConnection() + manual_albums = [] + manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have') + for albums in manualalbums: + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + if albums['Matched'] == "Ignored": + album_status = "Ignored" + elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + album_status = "Matched" + manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status } + if manual_dict not in manual_albums: + manual_albums.append(manual_dict) + manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle')) + + return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted) + manageManual.exposed = True + + def markManual(self, action=None, existing_artist=None, existing_album=None): + myDB = db.DBConnection() + if action == "unignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND Matched="Ignored"', [artist]) + logger.info("Artist: %s successfully restored to unmatched list" % artist) + + elif action == "unignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album)) + logger.info("Album: %s successfully restored to unmatched list" % album) + + elif action == "unmatchArtist": + artist = existing_artist + update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', [artist]) + update_count = 0 + for tracks in update_clean: + original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + album = tracks['AlbumTitle'] + track_title = tracks['TrackTitle'] + if tracks['CleanName'] != original_clean: + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + update_count+=1 + if update_count > 0: + librarysync.update_album_status() + logger.info("Artist: %s successfully restored to unmatched list" % artist) + + elif action == "unmatchAlbum": + artist = existing_artist + album = existing_album + update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) + update_count = 0 + for tracks in update_clean: + original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + track_title = tracks['TrackTitle'] + if tracks['CleanName'] != original_clean: + album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', [tracks['CleanName']]).fetchone() + if album_id_check: + album_id = album_id_check[0] + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) + myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + update_count+=1 + if update_count > 0: + librarysync.update_album_status(album_id) + logger.info("Album: %s successfully restored to unmatched list" % album) + + markManual.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() artistsToAdd = [] @@ -342,6 +547,8 @@ class WebInterface(object): myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) elif action == 'pause': controlValueDict = {'ArtistID': ArtistID} @@ -397,10 +604,16 @@ class WebInterface(object): def forceUpdate(self): from headphones import updater - threading.Thread(target=updater.dbUpdate).start() + threading.Thread(target=updater.dbUpdate, args=[False]).start() raise cherrypy.HTTPRedirect("home") forceUpdate.exposed = True + def forceFullUpdate(self): + from headphones import updater + threading.Thread(target=updater.dbUpdate, args=[True]).start() + raise cherrypy.HTTPRedirect("home") + forceFullUpdate.exposed = True + def forceSearch(self): from headphones import searcher threading.Thread(target=searcher.searchforalbum).start() @@ -533,6 +746,20 @@ class WebInterface(object): return s getArtists_json.exposed=True + def getAlbumsByArtist_json(self, artist=None): + myDB = db.DBConnection() + album_json = {} + counter = 0 + album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) + for album in album_list: + album_json[counter] = album['AlbumTitle'] + counter+=1 + json_albums = json.dumps(album_json) + + cherrypy.response.headers['Content-type'] = 'application/json' + return json_albums + getAlbumsByArtist_json.exposed=True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': @@ -554,6 +781,26 @@ class WebInterface(object): generateAPI.exposed = True + def forceScan(self, keepmatched=None): + myDB = db.DBConnection() + ######################################### + #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE + myDB.select('DELETE from Have') + logger.info('Removed all entries in local library database') + myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL') + myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL') + logger.info('All tracks in library unmatched') + myDB.action('UPDATE artists SET HaveTracks=NULL') + logger.info('Reset track counts for all artists') + myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"') + logger.info('Marking all unwanted albums as Skipped') + try: + threading.Thread(target=librarysync.libraryScan).start() + except Exception, e: + logger.error('Unable to complete the scan: %s' % e) + raise cherrypy.HTTPRedirect("home") + forceScan.exposed = True + def config(self): interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/') @@ -571,6 +818,8 @@ class WebInterface(object): "api_enabled" : checked(headphones.API_ENABLED), "api_key" : headphones.API_KEY, "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL, + "update_db_interval" : headphones.UPDATE_DB_INTERVAL, + "mb_ignore_age" : headphones.MB_IGNORE_AGE, "nzb_search_interval" : headphones.SEARCH_INTERVAL, "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL, "sab_host" : headphones.SAB_HOST, @@ -610,6 +859,9 @@ class WebInterface(object): "use_nzbsrus" : checked(headphones.NZBSRUS), "nzbsrus_uid" : headphones.NZBSRUS_UID, "nzbsrus_apikey" : headphones.NZBSRUS_APIKEY, + "use_omgwtfnzbs" : checked(headphones.OMGWTFNZBS), + "omgwtfnzbs_uid" : headphones.OMGWTFNZBS_UID, + "omgwtfnzbs_apikey" : headphones.OMGWTFNZBS_APIKEY, "preferred_words" : headphones.PREFERRED_WORDS, "ignored_words" : headphones.IGNORED_WORDS, "required_words" : headphones.REQUIRED_WORDS, @@ -720,10 +972,10 @@ class WebInterface(object): config.exposed = True def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, - download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, + download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, - use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, + use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, omgwtfnzbs=0, omgwtfnzbs_uid=None, omgwtfnzbs_apikey=None, preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None, rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0, @@ -747,6 +999,8 @@ class WebInterface(object): headphones.API_ENABLED = api_enabled headphones.API_KEY = api_key headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval + headphones.UPDATE_DB_INTERVAL = update_db_interval + headphones.MB_IGNORE_AGE = mb_ignore_age headphones.SEARCH_INTERVAL = nzb_search_interval headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval headphones.SAB_HOST = sab_host @@ -780,6 +1034,9 @@ class WebInterface(object): headphones.NZBSRUS = nzbsrus headphones.NZBSRUS_UID = nzbsrus_uid headphones.NZBSRUS_APIKEY = nzbsrus_apikey + headphones.OMGWTFNZBS = omgwtfnzbs + headphones.OMGWTFNZBS_UID = omgwtfnzbs_uid + headphones.OMGWTFNZBS_APIKEY = omgwtfnzbs_apikey headphones.PREFERRED_WORDS = preferred_words headphones.IGNORED_WORDS = ignored_words headphones.REQUIRED_WORDS = required_words diff --git a/lib/beets/mediafile.py b/lib/beets/mediafile.py index 785ce2a9..905d70e0 100644 --- a/lib/beets/mediafile.py +++ b/lib/beets/mediafile.py @@ -572,9 +572,23 @@ class ImageField(object): # No cover found. return None + elif obj.type == 'flac': + if 'metadata_block_picture' not in obj.mgfile: + return None + + for data in obj.mgfile['metadata_block_picture']: + try: + pic = lib.mutagen.flac.Picture(data) + break + except TypeError: + pass + else: + return None + + return pic.data else: - # Here we're assuming everything but MP3 and MPEG-4 uses + # Here we're assuming everything but MP3, FLAC and MPEG-4 use # the Xiph/Vorbis Comments standard. This may not be valid. # http://wiki.xiph.org/VorbisComment#Cover_art @@ -623,6 +637,13 @@ class ImageField(object): else: cover = lib.mutagen.mp4.MP4Cover(val, self._mp4kind(val)) obj.mgfile['covr'] = [cover] + + elif obj.type == 'flac': + if val is None: + pic = lib.mutagen.flac.Picture() + pic.data = val + pic.mime = self._mime(val) + obj.mgfile['metadata_block_picture'] = [pic.write()] else: # Again, assuming Vorbis Comments standard. diff --git a/lib/pygazelle/api.py b/lib/pygazelle/api.py index 75272cd5..4e93b47d 100644 --- a/lib/pygazelle/api.py +++ b/lib/pygazelle/api.py @@ -51,6 +51,7 @@ class GazelleAPI(object): self.passkey = None self.userid = None self.logged_in_user = None + self.default_timeout = 30 self.cached_users = {} self.cached_artists = {} self.cached_tags = {} @@ -97,7 +98,8 @@ class GazelleAPI(object): loginpage = 'https://what.cd/login.php' data = {'username': self.username, 'password': self.password} - r = self.session.post(loginpage, data=data) + r = self.session.post(loginpage, data=data, timeout=self.default_timeout) + self.past_request_timestamps.append(time.time()) if r.status_code != 200: raise LoginException("Login returned status code %s" % r.status_code) @@ -112,7 +114,6 @@ class GazelleAPI(object): self.passkey = accountinfo['passkey'] self.logged_in_user = User(self.userid, self) self.logged_in_user.set_index_data(accountinfo) - self.past_request_timestamps.append(time.time()) def request(self, action, autologin=True, **kwargs): """ @@ -153,7 +154,7 @@ class GazelleAPI(object): if self.authkey: params['auth'] = self.authkey params.update(kwargs) - r = self.session.get(url, params=params, allow_redirects=False) + r = self.session.get(url, params=params, allow_redirects=False, timeout=self.default_timeout) self.past_request_timestamps.append(time.time()) return r.content