From 1bf8c35154a179040e1a07c4b4f4145df882a81e Mon Sep 17 00:00:00 2001 From: Steve McAfee Date: Wed, 3 Jul 2024 21:22:55 -0400 Subject: [PATCH 1/5] decode paths before using them to fix manual post process --- headphones/postprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index b0b67cc2..26cc91db 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -1230,9 +1230,9 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig subfolders = helpers.expand_subfolders(path_to_folder) if expand_subfolders and subfolders is not None: - folders.extend(subfolders) + folders.extend(subfolders.decode(headphones.SYS_ENCODING, 'replace')) else: - folders.append(path_to_folder) + folders.append(path_to_folder.decode(headphones.SYS_ENCODING, 'replace')) # Log number of folders if folders: From ebe8a60ca5c01b7044906e7fe38826c1c203fd6c Mon Sep 17 00:00:00 2001 From: AdeHub Date: Sun, 16 Jun 2024 20:01:35 +1200 Subject: [PATCH 2/5] Soulseek tweaks -include soulseek results in overall search results -fixed windows paths not quite working on macOS -allow user search term -tighten search for self titled, Various Artists -postprocess by user/folder -bit more logging --- headphones/postprocessor.py | 19 ++++--- headphones/searcher.py | 81 +++++++++++++++++++++++---- headphones/soulseek.py | 109 +++++++++++++++++++++++++++++------- headphones/webserve.py | 2 +- 4 files changed, 169 insertions(+), 42 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 26cc91db..4ba268fa 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -41,21 +41,22 @@ def checkFolder(): with postprocessor_lock: myDB = db.DBConnection() snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"') - - # If soulseek is used, this part will get the status from the soulseek api and return completed and errored albums - completed_albums, errored_albums = set(), set() - if any(album['Kind'] == 'soulseek' for album in snatched): - completed_albums, errored_albums = soulseek.download_completed() - for album in snatched: if album['FolderName']: folder_name = album['FolderName'] single = False + + # Soulseek, check download complete or errored if album['Kind'] == 'soulseek': - if folder_name in errored_albums: + match = re.search(r'\{(.*?)\}(.*?)$', folder_name) # get soulseek user from folder_name + user_name = match.group(1) + folder_name = match.group(2) + completed, errored = soulseek.download_completed_album(user_name, folder_name) + if errored: # If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted. - logger.info(f"Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.") + logger.info(f"Soulseek: Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.") myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],)) + myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE AlbumID=?', (album['AlbumID'],)) # Folder will be removed from configured complete and Incomplete directory complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name) @@ -66,7 +67,7 @@ def checkFolder(): except Exception as e: pass continue - elif folder_name in completed_albums: + elif completed: download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR else: continue diff --git a/headphones/searcher.py b/headphones/searcher.py index 254412d6..1ccd9337 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -299,11 +299,21 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.ORPHEUS or headphones.CONFIG.REDACTED) + BANDCAMP = 1 if (headphones.CONFIG.BANDCAMP and + headphones.CONFIG.BANDCAMP_DIR) else 0 + + SOULSEEK = 1 if (headphones.CONFIG.SOULSEEK and + headphones.CONFIG.SOULSEEK_API_URL and + headphones.CONFIG.SOULSEEK_API_KEY and + headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR and + headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR) else 0 + results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + # NZBs if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: results = searchNZB(album, new, losslessOnly, albumlength) @@ -311,9 +321,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if not results and TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and headphones.CONFIG.BANDCAMP: + if not results and BANDCAMP: results = searchBandcamp(album, new, albumlength) + if not results and SOULSEEK: + results = searchSoulseek(album, new, losslessOnly, albumlength) + + # Torrents elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) @@ -321,17 +335,32 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: results = searchNZB(album, new, losslessOnly, albumlength) - if not results and headphones.CONFIG.BANDCAMP: - results = searchBandcamp(album, new, albumlength) + if not results and BANDCAMP: + results = searchBandcamp(album, new, albumlength) + if not results and SOULSEEK: + results = searchSoulseek(album, new, losslessOnly, albumlength) + + # Soulseek elif headphones.CONFIG.PREFER_TORRENTS == 2 and not choose_specific_download: results = searchSoulseek(album, new, losslessOnly, albumlength) + if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: + results = searchNZB(album, new, losslessOnly, albumlength) + + if not results and TORRENT_PROVIDERS: + results = searchTorrent(album, new, losslessOnly, albumlength) + + if not results and BANDCAMP: + results = searchBandcamp(album, new, albumlength) + else: + # No Preference nzb_results = [] torrent_results = [] bandcamp_results = [] + soulseek_results = [] if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, @@ -341,10 +370,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if headphones.CONFIG.BANDCAMP: + if BANDCAMP: bandcamp_results = searchBandcamp(album, new, albumlength) - results = nzb_results + torrent_results + bandcamp_results + if SOULSEEK: + soulseek_results = searchSoulseek(album, new, losslessOnly, albumlength) + + results = nzb_results + torrent_results + bandcamp_results + soulseek_results if choose_specific_download: return results @@ -876,8 +908,14 @@ def send_to_downloader(data, result, album): logger.info("Setting folder_name to: {}".format(folder_name)) elif kind == 'soulseek': - soulseek.download(user=result.user, filelist=result.files) - folder_name = result.folder + try: + soulseek.download(user=result.user, filelist=result.files) + folder_name = '{' + result.user + '}' + result.folder + logger.info(f"Soulseek folder name: {result.folder}") + except Exception as e: + logger.error(f"Soulseek error, check server logs: {e}") + return + else: folder_name = '%s - %s [%s]' % ( @@ -1953,12 +1991,33 @@ def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None): num_tracks = get_album_track_count(album['AlbumID']) year = get_year_from_release_date(album['ReleaseDate']) - cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip() - cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip() + cleanalbum = unidecode(replace_all(album['AlbumTitle'], replacements)).strip() + cleanartist = unidecode(replace_all(album['ArtistName'], replacements)).strip() - results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, num_tracks=num_tracks) + # If Preferred Bitrate and High Limit and Allow Lossless then get both lossy and lossless + if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: + allow_lossless = True + else: + allow_lossless = False - return results + if headphones.CONFIG.PREFERRED_QUALITY == 3 : + losslessOnly = True + elif headphones.CONFIG.PREFERRED_QUALITY == 1: + allow_lossless = True + + if album['SearchTerm']: + term = album['SearchTerm'] + else: + term = '' + + try: + results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, + allow_lossless=allow_lossless, num_tracks=num_tracks, user_search_term=term) + if not results: + logger.info("No valid results found from Soulseek") + return results + except Exception as e: + logger.error(f"Soulseek error, check server logs: {e}") def get_album_track_count(album_id): diff --git a/headphones/soulseek.py b/headphones/soulseek.py index 2802c35c..27bb3571 100644 --- a/headphones/soulseek.py +++ b/headphones/soulseek.py @@ -14,40 +14,50 @@ def initialize_soulseek_client(): return slskd_api.SlskdClient(host=host, api_key=api_key) # Search logic, calling search and processing fucntions -def search(artist, album, year, num_tracks, losslessOnly): +def search(artist, album, year, num_tracks, losslessOnly, allow_lossless, user_search_term): client = initialize_soulseek_client() + + # override search string with user provided search term if entered + if user_search_term: + artist = user_search_term + album = '' + year = '' # Stage 1: Search with artist, album, year, and num_tracks - results = execute_search(client, artist, album, year, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks) - if processed_results: + logger.info(f"Searching Soulseek using term: {artist} {album} {year}") + results = execute_search(client, artist, album, year, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks) + if processed_results or user_search_term or album.lower() == artist.lower(): return processed_results # Stage 2: If Stage 1 fails, search with artist, album, and num_tracks (excluding year) logger.info("Soulseek search stage 1 did not meet criteria. Retrying without year...") - results = execute_search(client, artist, album, None, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks) - if processed_results: + results = execute_search(client, artist, album, None, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks) + if processed_results or artist == "Various Artists": return processed_results # Stage 3: Final attempt, search only with artist and album logger.info("Soulseek search stage 2 did not meet criteria. Final attempt with only artist and album.") - results = execute_search(client, artist, album, None, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks, ignore_track_count=True) - + results = execute_search(client, artist, album, None, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=True) + return processed_results -def execute_search(client, artist, album, year, losslessOnly): +def execute_search(client, artist, album, year, losslessOnly, allow_lossless): search_text = f"{artist} {album}" if year: search_text += f" {year}" + if losslessOnly: - search_text += ".flac" + search_text += " flac" + elif not allow_lossless: + search_text += " mp3" # Actual search search_response = client.searches.search_text(searchText=search_text, filterResponses=True) search_id = search_response.get('id') - + # Wait for search completion and return response while not client.searches.state(id=search_id).get('isComplete'): time.sleep(2) @@ -55,8 +65,15 @@ def execute_search(client, artist, album, year, losslessOnly): return client.searches.search_responses(id=search_id) # Processing the search result passed -def process_results(results, losslessOnly, num_tracks, ignore_track_count=False): - valid_extensions = {'.flac'} if losslessOnly else {'.mp3', '.flac'} +def process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=False): + + if losslessOnly: + valid_extensions = {'.flac'} + elif allow_lossless: + valid_extensions = {'.mp3', '.flac'} + else: + valid_extensions = {'.mp3'} + albums = defaultdict(lambda: {'files': [], 'user': None, 'hasFreeUploadSlot': None, 'queueLength': None, 'uploadSpeed': None}) # Extract info from the api response and combine files at album level @@ -71,7 +88,8 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) filename = file.get('filename') file_extension = os.path.splitext(filename)[1].lower() if file_extension in valid_extensions: - album_directory = os.path.dirname(filename) + #album_directory = os.path.dirname(filename) + album_directory = filename.rsplit('\\', 1)[0] albums[album_directory]['files'].append(file) # Update metadata only once per album_directory @@ -86,8 +104,9 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) # Filter albums based on num_tracks, add bunch of useful info to the compiled album final_results = [] for directory, album_data in albums.items(): - if ignore_track_count or len(album_data['files']) == num_tracks: - album_title = os.path.basename(directory) + if ignore_track_count and len(album_data['files']) > 1 or len(album_data['files']) == num_tracks: + #album_title = os.path.basename(directory) + album_title = directory.rsplit('\\', 1)[1] total_size = sum(file.get('size', 0) for file in album_data['files']) final_results.append(Result( title=album_title, @@ -102,7 +121,8 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) files=album_data['files'], kind='soulseek', url='http://thisisnot.needed', # URL is needed in other parts of the program. - folder=os.path.basename(directory) + #folder=os.path.basename(directory) + folder = album_title )) return final_results @@ -163,13 +183,13 @@ def download_completed(): username = file_data.get('username', '') success = client.transfers.cancel_download(username, file_id) if not success: - print(f"Failed to cancel download for file ID: {file_id}") + logger.debug(f"Soulseek failed to cancel download for file ID: {file_id}") # Clear completed/canceled/errored stuff from client downloads try: client.transfers.remove_completed_downloads() except Exception as e: - print(f"Failed to remove completed downloads: {e}") + logger.debug(f"Soulseek failed to remove completed downloads: {e}") # Identify completed albums completed_albums = {album for album, counts in album_completion_tracker.items() if counts['total'] == counts['completed']} @@ -178,6 +198,53 @@ def download_completed(): return completed_albums, errored_albums +def download_completed_album(username, foldername): + client = initialize_soulseek_client() + downloads = client.transfers.get_downloads(username) + + # Anything older than 24 hours will be canceled + cutoff_time = datetime.now() - timedelta(hours=24) + + total_count = 0 + completed_count = 0 + errored_count = 0 + file_ids = [] + + # Identify errored and completed album + directories = downloads.get('directories', []) + for directory in directories: + album_part = directory.get('directory', '').split('\\')[-1] + if album_part == foldername: + files = directory.get('files', []) + for file_data in files: + state = file_data.get('state', '') + requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00') + requested_at = parse_datetime(requested_at_str) + + total_count += 1 + file_id = file_data.get('id', '') + file_ids.append(file_id) + + if 'Completed, Succeeded' in state: + completed_count += 1 + elif 'Completed, Errored' in state or requested_at < cutoff_time: + errored_count += 1 + break + + completed = True if completed_count == total_count else False + errored = True if errored_count else False + + # Cancel downloads for errored album + if errored: + for file_id in file_ids: + try: + success = client.transfers.cancel_download(username, file_id, remove=True) + except Exception as e: + logger.debug(f"Soulseek failed to cancel download for folder with file ID: {foldername} {file_id}") + + return completed, errored + + def parse_datetime(datetime_string): # Parse the datetime api response if '.' in datetime_string: diff --git a/headphones/webserve.py b/headphones/webserve.py index 005f4146..26a993a4 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1491,7 +1491,7 @@ class WebInterface(object): "songkick_enabled", "songkick_filter_enabled", "mpc_enabled", "email_enabled", "email_ssl", "email_tls", "email_onsnatch", "customauth", "idtag", "deluge_paused", - "join_enabled", "join_onsnatch", "use_bandcamp" + "join_enabled", "join_onsnatch", "use_bandcamp", "use_soulseek" ] for checked_config in checked_configs: if checked_config not in kwargs: From e5beb5291dab8b19586869f7d2683f8cdc0cccee Mon Sep 17 00:00:00 2001 From: AdeHub Date: Wed, 19 Jun 2024 20:33:40 +1200 Subject: [PATCH 3/5] Soulseek tweaks # 2 push results through filters --- headphones/searcher.py | 25 +++++++++++++++++++------ headphones/soulseek.py | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 1ccd9337..3f57bca9 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -373,8 +373,10 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if BANDCAMP: bandcamp_results = searchBandcamp(album, new, albumlength) - if SOULSEEK: - soulseek_results = searchSoulseek(album, new, losslessOnly, albumlength) + # TODO: get this working + # if SOULSEEK: + # soulseek_results = searchSoulseek(album, new, losslessOnly, + # albumlength, choose_specific_download) results = nzb_results + torrent_results + bandcamp_results + soulseek_results @@ -1451,7 +1453,6 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, title = item.title.get_text() url = item.find("link").next_sibling.strip() seeders = int(item.find("torznab:attr", attrs={"name": "seeders"}).get('value')) - # Torrentech hack - size currently not returned, make it up if 'torrentech' in torznab_host[0]: if albumlength: @@ -1973,7 +1974,8 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, return results -def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None): +def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None, + choose_specific_download=False): # Not using some of the input stuff for now or ever replacements = { '...': '', @@ -2011,13 +2013,24 @@ def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None): term = '' try: - results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, + resultlist = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, allow_lossless=allow_lossless, num_tracks=num_tracks, user_search_term=term) - if not results: + + if not resultlist: logger.info("No valid results found from Soulseek") + + # filter results + results = [result for result in resultlist if verifyresult(result.title, cleanartist, term, losslessOnly)] + + # Additional filtering for size etc + if results and not choose_specific_download: + results = more_filtering(results, album, albumlength, new) + return results + except Exception as e: logger.error(f"Soulseek error, check server logs: {e}") + return None def get_album_track_count(album_id): diff --git a/headphones/soulseek.py b/headphones/soulseek.py index 27bb3571..cd090b6a 100644 --- a/headphones/soulseek.py +++ b/headphones/soulseek.py @@ -120,7 +120,7 @@ def process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_tr queueLength=album_data['queueLength'], files=album_data['files'], kind='soulseek', - url='http://thisisnot.needed', # URL is needed in other parts of the program. + url='http://' + album_data['user'] + album_title, # URL is needed in other parts of the program. #folder=os.path.basename(directory) folder = album_title )) From 37c4e1966340991df4ac994802438b16db391873 Mon Sep 17 00:00:00 2001 From: AdeHub Date: Fri, 5 Jul 2024 19:26:30 +1200 Subject: [PATCH 4/5] Torznab changes -Search using parent category 3000 some providers lump everything in to the parent Music Category 3000 instead of the Sub Category (e.g. 3040 lossless). search using 3000 then filter with Category (if returned) or Headphones filtering overall this should return more results across indexers -Allow Prowlarr indexers to be manually entered -Searcher fixups --- data/interfaces/default/config.html | 2 +- headphones/searcher.py | 120 ++++++++++++++++------------ 2 files changed, 71 insertions(+), 51 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 77d3f27c..92ff7791 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -749,7 +749,7 @@
- +
diff --git a/headphones/searcher.py b/headphones/searcher.py index 3f57bca9..b704497a 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -187,13 +187,13 @@ def get_seed_ratio(provider): seed_ratio = headphones.CONFIG.OLDPIRATEBAY_RATIO elif provider == 'Waffles.ch': seed_ratio = headphones.CONFIG.WAFFLES_RATIO - elif provider.startswith("Jackett_"): - provider = provider.split("Jackett_")[1] - if provider in headphones.CONFIG.TORZNAB_HOST: + elif provider.startswith("Torznab"): + host = provider.split('|')[2] + if host == headphones.CONFIG.TORZNAB_HOST: seed_ratio = headphones.CONFIG.TORZNAB_RATIO else: for torznab in headphones.CONFIG.get_extra_torznabs(): - if provider in torznab[0]: + if host == torznab[0]: seed_ratio = torznab[2] break else: @@ -208,6 +208,21 @@ def get_seed_ratio(provider): return seed_ratio +def get_provider_name(provider): + """ + Return the provider name for the provider + """ + + if provider.startswith("Torznab"): + provider_name = "Torznab " + provider.split("|")[1] + elif provider.startswith(("http://", "https://")): + provider_name = provider.split("//")[1] + else: + provider_name = provider + + return provider_name + + def searchforalbum(albumid=None, new=False, losslessOnly=False, choose_specific_download=False): logger.info('Searching for wanted albums') @@ -394,7 +409,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): logger.info( "Making sure we can download the best result: " - f"{sorted_search_results[0].title} from {sorted_search_results[0].provider}" + f"{sorted_search_results[0].title} from {get_provider_name(sorted_search_results[0].provider)}" ) (data, result) = preprocess(sorted_search_results) @@ -423,10 +438,10 @@ def more_filtering(results, album, albumlength, new): targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128 logger.info('Target size: %s' % bytes_to_mb(targetsize)) if headphones.CONFIG.PREFERRED_BITRATE_LOW_BUFFER: - low_size_limit = targetsize * int( + low_size_limit = targetsize - targetsize * int( headphones.CONFIG.PREFERRED_BITRATE_LOW_BUFFER) / 100 if headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER: - high_size_limit = targetsize * int( + high_size_limit = targetsize + targetsize * int( headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER) / 100 if headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: allow_lossless = True @@ -437,15 +452,15 @@ def more_filtering(results, album, albumlength, new): if low_size_limit and result.size < low_size_limit: logger.info( - f"{result.title} from {result.provider} is too small for this album. " - f"(Size: {result.size}, MinSize: {bytes_to_mb(low_size_limit)})" + f"{result.title} from {get_provider_name(result.provider)} is too small for this album. " + f"(Size: {bytes_to_mb(result.size)}, MinSize: {bytes_to_mb(low_size_limit)})" ) continue if high_size_limit and result.size > high_size_limit: logger.info( - f"{result.title} from {result.provider} is too large for this album. " - f"(Size: {result.size}, MaxSize: {bytes_to_mb(high_size_limit)})" + f"{result.title} from {get_provider_name(result.provider)} is too large for this album. " + f"(Size: {bytes_to_mb(result.size)}, MaxSize: {bytes_to_mb(high_size_limit)})" ) # Keep lossless results if there are no good lossy matches if not (allow_lossless and 'flac' in result.title.lower()): @@ -458,7 +473,7 @@ def more_filtering(results, album, albumlength, new): if len(alreadydownloaded): logger.info( f"{result.title} has already been downloaded from " - f"{result.provider}. Skipping." + f"{get_provider_name(result.provider)}. Skipping." ) continue @@ -517,12 +532,13 @@ def sort_search_results(resultlist, album, new, albumlength): delta = abs(targetsize - result.size) lossy_results_with_delta.append((result, priority, delta)) - return list(map(lambda x: x[0], - sorted( - lossy_results_with_delta, - key=lambda x: (-x[0].matches, -x[1], x[2]) - ) - )) + if len(lossy_results_with_delta): + return list(map(lambda x: x[0], + sorted( + lossy_results_with_delta, + key=lambda x: (-x[0].matches, -x[1], x[2]) + ) + )) if ( not len(lossy_results_with_delta) @@ -534,7 +550,7 @@ def sort_search_results(resultlist, album, new, albumlength): "(and at least one lossless match), going to use " "lossless instead" ) - return sort_by_priority_then_size(results_with_priority) + return sort_by_priority_then_size(lossless_results) except Exception: logger.exception('Unhandled exception') @@ -854,7 +870,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, def send_to_downloader(data, result, album): logger.info( - f"Found best result from {result.provider}: " + f"Found best result from {get_provider_name(result.provider)}: " f"{result.title} - {bytes_to_mb(result.size)}" ) # Get rid of any dodgy chars here so we can prevent sab from renaming our downloads @@ -1155,9 +1171,7 @@ def send_to_downloader(data, result, album): albumname = album[2] rgid = album[6] title = artist + ' - ' + albumname - provider = result.provider - if provider.startswith(("http://", "https://")): - provider = provider.split("//")[1] + provider = get_provider_name(result.provider) name = folder_name if folder_name else None if headphones.CONFIG.GROWL_ENABLED and headphones.CONFIG.GROWL_ONSNATCH: @@ -1252,8 +1266,8 @@ def verifyresult(title, artistterm, term, lossless): return False # Filter out FLAC if we're not specifically looking for it - if headphones.CONFIG.PREFERRED_QUALITY == ( - 0 or '0') and 'flac' in title.lower() and not lossless: + if (headphones.CONFIG.PREFERRED_QUALITY == 0 or headphones.CONFIG.PREFERRED_QUALITY == '0') \ + and 'flac' in title.lower() and not lossless: logger.info( "Removed %s from results because it's a lossless album and we're not looking for a lossless album right now.", title) @@ -1285,7 +1299,7 @@ def verifyresult(title, artistterm, term, lossless): if headphones.CONFIG.IGNORE_CLEAN_RELEASES: for each_word in ['clean', 'edited', 'censored']: - logger.debug("Checking if '%s' is in search result: '%s'", each_word, title) + # logger.debug("Checking if '%s' is in search result: '%s'", each_word, title) if each_word.lower() in title.lower() and each_word.lower() not in term.lower(): logger.info("Removed '%s' from results because it contains clean album word: '%s'", title, each_word) @@ -1403,6 +1417,8 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, if torznab_host[3] == '1' or torznab_host[3] == 1: torznab_hosts.append(torznab_host) + parent_category = "3000" + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" maxsize = 10000000000 @@ -1417,23 +1433,28 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, categories = "3030" logger.info("Album type is audiobook/spokenword. Using audiobook category") + categories = categories + "," + parent_category + for torznab_host in torznab_hosts: provider = torznab_host[0] + provider_name = torznab_host[0] # Format Jackett provider if "api/v2.0/indexers" in torznab_host[0]: - provider = "Jackett_" + provider.split("/indexers/", 1)[1].split('/', 1)[0] + provider_name = provider.split("/indexers/", 1)[1].split('/', 1)[0] + provider = "Torznab" + '|' + provider_name + '|' + torznab_host[0] # Request results - logger.info('Parsing results from %s using search term: %s' % (provider, term)) + logger.info('Parsing results from Torznab %s using search term: %s' % (provider_name, term)) headers = {'User-Agent': USER_AGENT} params = { "t": "search", "apikey": torznab_host[1], - "cat": categories, - "maxage": headphones.CONFIG.USENET_RETENTION, + #"cat": categories, + "cat": parent_category, # search using '3000' and filter below + #"maxage": headphones.CONFIG.USENET_RETENTION, "q": term } @@ -1446,39 +1467,38 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, if data: items = data.find_all('item') if not items: - logger.info("No results found from %s for %s", provider, term) + logger.info("No results found from %s for %s", provider_name, term) else: for item in items: try: title = item.title.get_text() url = item.find("link").next_sibling.strip() seeders = int(item.find("torznab:attr", attrs={"name": "seeders"}).get('value')) - # Torrentech hack - size currently not returned, make it up - if 'torrentech' in torznab_host[0]: - if albumlength: - if 'Lossless' in title: - size = albumlength / 1000 * 800 * 128 - elif 'MP3' in title: - size = albumlength / 1000 * 320 * 128 - else: - size = albumlength / 1000 * 256 * 128 - else: - logger.info('Skipping %s, could not determine size' % title) - continue - elif item.size: + if item.size: size = int(item.size.string) else: size = int(item.find("torznab:attr", attrs={"name": "size"}).get('value')) + category = item.find("torznab:attr", attrs={"name": "category"}).get('value') + if category not in categories: + logger.info(f"Skipping {title}, size {bytes_to_mb(size)}, incorrect category {category}") + continue + if all(word.lower() in title.lower() for word in term.split()): if size < maxsize and minimumseeders < seeders: logger.info('Found %s. Size: %s' % (title, bytes_to_mb(size))) + if item.prowlarrindexer: + provider = "Torznab" + '|' + item.prowlarrindexer.get_text() + '|' + \ + torznab_host[0] + elif item.jackettindexer: + provider = "Torznab" + '|' + item.jackettindexer.get_text() + '|' + \ + torznab_host[0] resultlist.append(Result(title, size, url, provider, 'torrent', True)) else: logger.info( '%s is larger than the maxsize or has too little seeders for this category, ' - 'skipping. (Size: %i bytes, Seeders: %d)', - title, size, seeders) + 'skipping. (Size: %s, Seeders: %d)', + title, bytes_to_mb(size), seeders) else: logger.info('Skipping %s, not all search term words found' % title) @@ -2056,15 +2076,15 @@ def preprocess(resultlist): if result.provider == 'rutracker.org': return ruobj.get_torrent_data(result.url), result - # Jackett sometimes redirects - if result.provider.startswith('Jackett_') or 'torznab' in result.provider.lower(): + # Torznab sometimes redirects + if result.provider.startswith("Torznab") or 'torznab' in result.provider.lower(): r = request.request_response(url=result.url, headers=headers, allow_redirects=False) if r: link = r.headers.get('Location') if link and link != result.url: if link.startswith('magnet:'): result = Result( - result.url, + result.title, result.size, link, result.provider, @@ -2074,7 +2094,7 @@ def preprocess(resultlist): return "d10:magnet-uri%d:%se" % (len(link), link), result else: result = Result( - result.url, + result.title, result.size, link, result.provider, From 7b9e32f525600f9482901501d3fee89adf6aa14a Mon Sep 17 00:00:00 2001 From: dsm1212 Date: Sun, 14 Jul 2024 20:55:05 -0400 Subject: [PATCH 5/5] fix duplicate file rename --- headphones/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/headphones/helpers.py b/headphones/helpers.py index ea8960f2..54bc3f06 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -860,6 +860,8 @@ def smartMove(src, dest, delete=True): try: os.rename(src, os.path.join(source_dir, newfile)) filename = newfile + source_path = os.path.join(source_dir, filename) + dest_path = os.path.join(dest, filename) except Exception as e: logger.warn(f"Error renaming {src}: {e}") break