From 59b4cefd53bab74cc95d708e04616aafb3dc0f97 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 17:02:34 +0200 Subject: [PATCH 1/6] Creating two functions from duplicated code --- headphones/postprocessor.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index c80fc267..0c053495 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,6 +30,47 @@ from headphones import logger, helpers, request, mb, music_encoder postprocessor_lock = threading.Lock() + +def find_in_path(albumpath, extra_formats=None, use_MF=True): + """ Takes a path and optionally extra formats. + Finds files matching the MEDIA_FORMATS and returns them as a list. + If use_MF is disabled MEDIA_FORMATS will be ignored. + """ + found_tracks = [] + if extra_formats: + if type(extra_formats) is not list: + extra_formats = [extra_formats] + if not use_MF: + mf = extra_formats + else: + mf = headphones.MEDIA_FORMATS + extra_formats + else: + if not use_MF: + # if not using MF then extra_formats must be set + return False + mf = headphones.MEDIA_FORMATS + for r, d, f in os.walk(albumpath): + for file in f: + if any(file.lower().endswith('.' + x.lower()) for x in mf): + found_tracks.append(os.path.join(r, file)) + return found_tracks + + +def count_matches(track_list, match, inverted=False): + """ Takes a list of tracks and a pattern to match. + Returns the number of matched items and a list of the matches. + If inverted is set to True the returned list will be inverted and only + contain the NON-matches. + """ + if not track_list or not match: + return False, False + new_list = [x for x in track_list if x.lower().endswith(match)] + nr_of_matches = len(new_list) + if inverted: + new_list = [x for x in track_list if not x.lower().endswith(match)] + return nr_of_matches, new_list + + def checkFolder(): with postprocessor_lock: From 405ebe5ee2b6ca71053659b1e6cdf2759fccd26c Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 17:46:01 +0200 Subject: [PATCH 2/6] Removing duplications, replacing with functions --- headphones/postprocessor.py | 71 ++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 0c053495..38c2c2fb 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -193,20 +193,19 @@ def verify(albumid, albumpath, Kind=None, forced=False): release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) - downloaded_track_list = [] - downloaded_cuecount = 0 - - for r,d,f in os.walk(albumpath): - for files in f: - if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, files)) - elif files.lower().endswith('.cue'): - downloaded_cuecount += 1 - # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: - logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") - return + all_matched = find_in_path(albumpath, ["cue", "part", "upart"]) + incomplete_matches, remaining_matches = count_matches(all_matched, + (".part", ".upart"), + inverted = True) + if incomplete_matches and not forced: + _a = os.path.basename(albumpath).decode(headphones.SYS_ENCODING, + 'replace') + logger.info("Looks like " + _a + " isn't complete yet. Will try again on the next run") + return + downloaded_cuecount, downloaded_track_list = count_matches(remaining_matches, + ".cue", + inverted = True) # use xld to split cue @@ -248,11 +247,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): # count files, should now be more than original if xld successfully split - new_downloaded_track_list_count = 0 - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - new_downloaded_track_list_count += 1 + new_downloaded_track_list_count = len(find_in_path(albumpath)) if new_downloaded_track_list_count > len(downloaded_track_list): @@ -261,12 +256,8 @@ def verify(albumid, albumpath, Kind=None, forced=False): os.rename(downloaded_track, downloaded_track + '.original') #reload + downloaded_track_list = find_in_path(albumpath) - downloaded_track_list = [] - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, file)) # test #1: metadata - usually works logger.debug('Verifying metadata...') @@ -371,15 +362,12 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # Need to update the downloaded track list with the new location. # Could probably just throw in the "headphones-modified" folder, # but this is good to make sure we're not counting files that may have failed to move - downloaded_track_list = [] - downloaded_cuecount = 0 - for r,d,f in os.walk(albumpath): - for files in f: - if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, files)) - elif files.lower().endswith('.cue'): - downloaded_cuecount += 1 + all_matches = find_in_path(albumpath, "cue") + downloaded_cuecount, downloaded_track_list = count_matches(all_matches, + ".cue", + True) + # Check if files are valid media files and are writeable, before the steps # below are executed. This simplifies errors and prevents unfinished steps. @@ -618,15 +606,18 @@ def addAlbumArt(artwork, albumpath, release): def cleanupFiles(albumpath): logger.info('Cleaning up files') - for r,d,f in os.walk(albumpath): - for files in f: - if not any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - if not (headphones.KEEP_NFO and files.lower().endswith('.nfo')): - logger.debug('Removing: %s' % files) - try: - os.remove(os.path.join(r, files)) - except Exception, e: - logger.error(u'Could not remove file: %s. Error: %s' % (files.decode(headphones.SYS_ENCODING, 'replace'), e)) + if headphones.KEEP_NFO: + files = find_in_path(albumpath, extra_formats="nfo", inverted=True) + else: + files = find_in_path(albumpath, inverted=True) + for _f in files: + logger.debug('Removing: %s' % _f) + try: + os.remove(_f) + except Exception, e: + _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace') + logger.error(u'Could not remove file: %s. Error: %s' % (_f_decoded, + e)) def renameNFO(albumpath): for r,d,f in os.walk(albumpath): From 08f483d77d9b6838b074300afd3c1beba46e665b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 18:57:44 +0200 Subject: [PATCH 3/6] Removing duplications, replacing with functions --- headphones/postprocessor.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 38c2c2fb..f0debe29 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -620,15 +620,16 @@ def cleanupFiles(albumpath): e)) def renameNFO(albumpath): - for r,d,f in os.walk(albumpath): - for file in f: - if file.lower().endswith('.nfo'): - logger.debug('Renaming: "%s" to "%s"' % (file.decode(headphones.SYS_ENCODING, 'replace'), file.decode(headphones.SYS_ENCODING, 'replace') + '-orig')) - try: - new_file_name = os.path.join(r, file)[:-3] + 'orig.nfo' - os.rename(os.path.join(r, file), new_file_name) - except Exception, e: - logger.error(u'Could not rename file: %s. Error: %s' % (os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e)) + files = find_in_path(albumpath, extra_formats="nfo", use_MF=False) + for _f in files: + _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace') + logger.debug('Renaming: "%s" to "%s"' % (_f_decoded, _f_decoded + '-orig')) + try: + new_file_name = _f[:-3] + 'orig.nfo' + os.rename(_f, new_file_name) + except Exception, e: + logger.error(u'Could not rename file: %s. Error: %s' % (_f_decoded, + e)) def moveFiles(albumpath, release, tracks): From ef3cd00ef1724ae234add7d35dc063db379e43ad Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 19:12:02 +0200 Subject: [PATCH 4/6] Removing duplications, replacing with functions --- headphones/postprocessor.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index f0debe29..a4f65674 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -695,13 +695,14 @@ def moveFiles(albumpath, release, tracks): lossy_media = False lossless_media = False - for r,d,f in os.walk(albumpath): - for files in f: - files_to_move.append(os.path.join(r, files)) - if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSY_MEDIA_FORMATS): - lossy_media = True - if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSLESS_MEDIA_FORMATS): - lossless_media = True + if find_in_path(albumpath, + extra_formats=headphones.LOSSY_MEDIA_FORMATS, + use_MF=False): + lossy_media = True + if find_in_path(albumpath, + extra_formats=headphones.LOSSLESS_MEDIA_FORMATS, + use_MF=False): + lossless_media = True # Do some sanity checking to see what directories we need to create: make_lossy_folder = False From 17ef29887e48c4e9e35b63896429214f78cf4795 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 16 Aug 2014 01:57:05 +0200 Subject: [PATCH 5/6] Use _file instead of file since file is a built-in function --- headphones/postprocessor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index a4f65674..e102a4d2 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -50,9 +50,9 @@ def find_in_path(albumpath, extra_formats=None, use_MF=True): return False mf = headphones.MEDIA_FORMATS for r, d, f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in mf): - found_tracks.append(os.path.join(r, file)) + for _file in f: + if any(_file.lower().endswith('.' + x.lower()) for x in mf): + found_tracks.append(os.path.join(r, _file)) return found_tracks @@ -420,7 +420,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, if headphones.KEEP_NFO: renameNFO(albumpath) - + if headphones.ADD_ALBUM_ART and artwork: addAlbumArt(artwork, albumpath, release) @@ -655,12 +655,12 @@ def moveFiles(albumpath, release, tracks): firstchar = '0-9' else: firstchar = sortname[0] - + for r,d,f in os.walk(albumpath): try: origfolder = os.path.basename(os.path.normpath(r)) except: - origfolder = '' + origfolder = '' values = { '$Artist': artist, '$SortArtist': sortname, From 1b65df6c4574b094d9d6115eede6b4547bd80eac Mon Sep 17 00:00:00 2001 From: David Date: Sun, 17 Aug 2014 19:48:47 +0200 Subject: [PATCH 6/6] Return false on non existing dirs --- headphones/postprocessor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e102a4d2..b4381934 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -36,6 +36,8 @@ def find_in_path(albumpath, extra_formats=None, use_MF=True): Finds files matching the MEDIA_FORMATS and returns them as a list. If use_MF is disabled MEDIA_FORMATS will be ignored. """ + if not os.path.isdir(albumpath): + return False found_tracks = [] if extra_formats: if type(extra_formats) is not list: