From 7e9bd432ce052cd7e245229dee1db903752816ce Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Fri, 26 Feb 2016 17:43:36 +0100 Subject: [PATCH 01/57] postprocessor.py: Safeguard against lack or ReleaseDate in metadata --- headphones/postprocessor.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 507a5571..194cb62c 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -610,16 +610,23 @@ def addAlbumArt(artwork, albumpath, release): logger.info('Adding album art to folder') try: - year = release['ReleaseDate'][:4] + date = release['ReleaseDate'] except TypeError: - year = '' + date = u'' + + if date is not None: + year = date[:4] + else: + year = u'' values = {'$Artist': release['ArtistName'], '$Album': release['AlbumTitle'], '$Year': year, + '$Date': date, '$artist': release['ArtistName'].lower(), '$album': release['AlbumTitle'].lower(), - '$year': year + '$year': year, + '$date': date } album_art_name = helpers.replace_all(headphones.CONFIG.ALBUM_ART_FORMAT.strip(), @@ -679,7 +686,12 @@ def moveFiles(albumpath, release, tracks): date = release['ReleaseDate'] except TypeError: date = u'' - year = date[:4] + + if date is not None: + year = date[:4] + else: + year = u'' + artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') if headphones.CONFIG.FILE_UNDERSCORES: @@ -1072,7 +1084,11 @@ def renameFiles(albumpath, downloaded_track_list, release): date = release['ReleaseDate'] except TypeError: date = u'' - year = date[:4] + + if date is not None: + year = date[:4] + else: + year = u'' # Until tagging works better I'm going to rely on the already provided metadata From 72b764cc51db1513c4497e0c442aaca184fde6bf Mon Sep 17 00:00:00 2001 From: satreix Date: Fri, 26 Feb 2016 19:25:21 +0100 Subject: [PATCH 02/57] pep: fix e502 --- .pep8 | 3 +-- headphones/__init__.py | 2 +- headphones/helpers.py | 6 +++--- headphones/logger.py | 2 +- headphones/postprocessor.py | 12 ++++++------ headphones/searcher.py | 14 +++++++------- headphones/transmission.py | 2 +- headphones/webstart.py | 4 ++-- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.pep8 b/.pep8 index daa763d1..92ad73e6 100644 --- a/.pep8 +++ b/.pep8 @@ -10,6 +10,5 @@ # E262 inline comment should start with '# ' # E265 block comment should start with '# ' # E501 line too long (312 > 160 characters) -# E502 the backslash is redundant between brackets -ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E501,E502 +ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E501 max-line-length = 160 diff --git a/headphones/__init__.py b/headphones/__init__.py index 4b36acf7..0ed027af 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -132,7 +132,7 @@ def initialize(config_file): CONFIG.LOG_DIR = None if not QUIET: - sys.stderr.write("Unable to create the log directory. " \ + sys.stderr.write("Unable to create the log directory. " "Logging to screen only.\n") # Start the logger, disable console if needed diff --git a/headphones/helpers.py b/headphones/helpers.py index 5e836178..adfced66 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -538,8 +538,8 @@ def preserve_torrent_directory(albumpath): shutil.copytree(albumpath, new_folder) return new_folder except Exception as e: - logger.warn("Cannot copy/move files to temp folder: " + \ - new_folder.decode(headphones.SYS_ENCODING, 'replace') + \ + logger.warn("Cannot copy/move files to temp folder: " + + new_folder.decode(headphones.SYS_ENCODING, 'replace') + ". Not continuing. Error: " + str(e)) return None @@ -684,7 +684,7 @@ def walk_directory(basedir, followlinks=True): real_path = os.path.abspath(os.readlink(path)) if real_path in traversed: - logger.debug("Skipping '%s' since it is a symlink to " \ + logger.debug("Skipping '%s' since it is a symlink to " "'%s', which is already visited.", path, real_path) else: traversed.append(real_path) diff --git a/headphones/logger.py b/headphones/logger.py index b01e880f..1eaac7cc 100644 --- a/headphones/logger.py +++ b/headphones/logger.py @@ -72,7 +72,7 @@ def listener(): # http://stackoverflow.com/questions/2009278 for more information. if e.errno == errno.EACCES: logger.warning("Multiprocess logging disabled, because " - "current user cannot map shared memory. You won't see any" \ + "current user cannot map shared memory. You won't see any" "logging generated by the worker processed.") # Multiprocess logging may be disabled. diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 507a5571..2bc8e7c6 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -108,8 +108,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal [release_dict['artist_id'], release_dict['artist_name']]) if not artist: - logger.warn("Continuing would add new artist '%s' (ID %s), " \ - "but database is frozen. Will skip postprocessing for " \ + logger.warn("Continuing would add new artist '%s' (ID %s), " + "but database is frozen. Will skip postprocessing for " "album with rgid: %s", release_dict['artist_name'], release_dict['artist_id'], albumid) @@ -348,7 +348,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # this test is just to keep pyflakes from complaining about an unused variable return except (FileTypeError, UnreadableFileError): - logger.error("Track file is not a valid media file: %s. Not " \ + logger.error("Track file is not a valid media file: %s. Not " "continuing.", downloaded_track.decode( headphones.SYS_ENCODING, "replace")) return @@ -371,7 +371,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, fp.seek(0) except IOError as e: logger.debug("Write check exact error: %s", e) - logger.error("Track file is not writable. This is required " \ + logger.error("Track file is not writable. This is required " "for some post processing steps: %s. Not continuing.", downloaded_track.decode(headphones.SYS_ENCODING, "replace")) if new_folder: @@ -1420,7 +1420,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig logger.info('No match found on MusicBrainz for: %s - %s', name, album) # Fail here - logger.info("Couldn't parse '%s' into any valid format. If adding " \ - "albums from another source, they must be in an 'Artist - Album " \ + logger.info("Couldn't parse '%s' into any valid format. If adding " + "albums from another source, they must be in an 'Artist - Album " "[Year]' format, or end with the musicbrainz release group id.", folder_basename) diff --git a/headphones/searcher.py b/headphones/searcher.py index 5c8b4144..66611f8f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -119,10 +119,10 @@ def read_torrent_name(torrent_file, default_name=None): return torrent_info["info"]["name"] except KeyError: if default_name: - logger.warning("Couldn't get name from torrent file: %s. " \ + logger.warning("Couldn't get name from torrent file: %s. " "Defaulting to '%s'", e, default_name) else: - logger.warning("Couldn't get name from torrent file: %s. No " \ + logger.warning("Couldn't get name from torrent file: %s. No " "default given", e) # Return default @@ -143,7 +143,7 @@ def calculate_torrent_hash(link, data=None): info = bdecode(data)["info"] torrent_hash = sha1(bencode(info)).hexdigest() else: - raise ValueError("Cannot calculate torrent hash without magnet link " \ + raise ValueError("Cannot calculate torrent hash without magnet link " "or data") return torrent_hash.upper() @@ -842,16 +842,16 @@ def send_to_downloader(data, bestqual, album): break else: # No service succeeded - logger.warning("Unable to convert magnet with hash " \ + logger.warning("Unable to convert magnet with hash " "'%s' into a torrent file.", torrent_hash) return elif headphones.CONFIG.MAGNET_LINKS == 3: torrent_to_file(download_path, data) return else: - logger.error("Cannot save magnet link in blackhole. " \ - "Please switch your torrent downloader to " \ - "Transmission, uTorrent or Deluge, or allow Headphones " \ + logger.error("Cannot save magnet link in blackhole. " + "Please switch your torrent downloader to " + "Transmission, uTorrent or Deluge, or allow Headphones " "to open or convert magnet links") return else: diff --git a/headphones/transmission.py b/headphones/transmission.py index b62e2fac..3e436617 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -164,7 +164,7 @@ def torrentAction(method, arguments): whitelist_status_code=[401, 409]) if response.status_code == 401: if auth: - logger.error("Username and/or password not accepted by " \ + logger.error("Username and/or password not accepted by " "Transmission") else: logger.error("Transmission authorization required") diff --git a/headphones/webstart.py b/headphones/webstart.py index a4ab7113..cc639221 100644 --- a/headphones/webstart.py +++ b/headphones/webstart.py @@ -35,12 +35,12 @@ def initialize(options): if not (https_cert and os.path.exists(https_cert)) or not ( https_key and os.path.exists(https_key)): if not create_https_certificates(https_cert, https_key): - logger.warn("Unable to create certificate and key. Disabling " \ + logger.warn("Unable to create certificate and key. Disabling " "HTTPS") enable_https = False if not (os.path.exists(https_cert) and os.path.exists(https_key)): - logger.warn("Disabled HTTPS because of missing certificate and " \ + logger.warn("Disabled HTTPS because of missing certificate and " "key.") enable_https = False From a4c9990df1244de7d72cb531b280bc50b6f1c745 Mon Sep 17 00:00:00 2001 From: satreix Date: Fri, 26 Feb 2016 19:33:30 +0100 Subject: [PATCH 03/57] pep: fix e262 --- .pep8 | 3 +-- headphones/searcher.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pep8 b/.pep8 index daa763d1..548f124a 100644 --- a/.pep8 +++ b/.pep8 @@ -7,9 +7,8 @@ # E127 continuation line over-indented for visual indent # E128 continuation line under-indented for visual indent # E261 at least two spaces before inline comment -# E262 inline comment should start with '# ' # E265 block comment should start with '# ' # E501 line too long (312 > 160 characters) # E502 the backslash is redundant between brackets -ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E501,E502 +ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E265,E501,E502 max-line-length = 160 diff --git a/headphones/searcher.py b/headphones/searcher.py index 5c8b4144..01578d70 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -1779,7 +1779,7 @@ def preprocess(resultlist): return ruobj.get_torrent_data(result[2]), result # Get out of here if we're using Transmission - if headphones.CONFIG.TORRENT_DOWNLOADER == 1: ## if not a magnet link still need the .torrent to generate hash... uTorrent support labeling + if headphones.CONFIG.TORRENT_DOWNLOADER == 1: # if not a magnet link still need the .torrent to generate hash... uTorrent support labeling return True, result # Get out of here if it's a magnet link if result[2].lower().startswith("magnet:"): From 55eed7c378723cf31963f85e23bf6fd2df7297ad Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Sat, 27 Feb 2016 17:00:34 +0100 Subject: [PATCH 04/57] pathrender: Add unit tests module for pathrender.py --- headphones/pathrender.py | 68 ++++++++++++++++------- headphones/pathrender_test.py | 100 ++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 19 deletions(-) create mode 100644 headphones/pathrender_test.py diff --git a/headphones/pathrender.py b/headphones/pathrender.py index 162c6956..48798dd7 100644 --- a/headphones/pathrender.py +++ b/headphones/pathrender.py @@ -13,22 +13,23 @@ # # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -'''Path pattern substitution module, see details below for syntax. +""" +Path pattern substitution module, see details below for syntax. - The pattern matching is loosely based on foobar2000 pattern syntax, - i.e. the notion of escaping characters with \' and optional elements - enclosed in square brackets [] is taken from there while the - substitution variable names are Perl-ish or sh-ish. The following - syntax elements are supported: - * escaped literal strings, that is everything that is enclosed - within single quotes (like \'this\'); - * substitution variables, which start with dollar sign ($) and - extend until next non-alphanumeric+underscore character - (like $This and $5_that). - * optional elements enclosed in curly braces, which render - nonempty value only if any variable or optional inside returned - nonempty value, ignoring literals (like {\'[\'$That\']\'}). -''' +The pattern matching is loosely based on foobar2000 pattern syntax, +i.e. the notion of escaping characters with \' and optional elements +enclosed in square brackets [] is taken from there while the +substitution variable names are Perl-ish or sh-ish. The following +syntax elements are supported: +* escaped literal strings, that is everything that is enclosed + within single quotes (like 'this'); +* substitution variables, which start with dollar sign ($) and + extend until next non-alphanumeric+underscore character + (like $This and $5_that). +* optional elements enclosed in curly braces, which render + nonempty value only if any variable or optional inside returned + nonempty value, ignoring literals (like {'{'$That'}'}). +""" from __future__ import print_function from enum import Enum @@ -42,6 +43,9 @@ class _PatternElement(object): '''Format this _PatternElement into string using provided substitution dictionary.''' raise NotImplementedError() + def __ne__(self, other): + return not self == other + class _Generator(_PatternElement): # pylint: disable=abstract-method @@ -57,11 +61,23 @@ class _Replacement(_Generator): def render(self, replacement): # type: (Mapping[str,str]) -> str - return replacement.get(self._pattern, self._pattern) + res = replacement.get(self._pattern, self._pattern) + if res is None: + return '' + else: + return res def __str__(self): return self._pattern + @property + def pattern(self): + return self._pattern + + def __eq__(self, other): + return isinstance(other, _Replacement) and \ + self._pattern == other.pattern + class _LiteralText(_PatternElement): '''Just a plain piece of text to be rendered "as is".''' @@ -76,6 +92,13 @@ class _LiteralText(_PatternElement): def __str__(self): return self._text + @property + def text(self): + return self._text + + def __eq__(self, other): + return isinstance(other, _LiteralText) and self._text == other.text + class _OptionalBlock(_Generator): '''Optional block will render its contents only if any _Generator in its scope did return non-empty result.''' @@ -87,11 +110,17 @@ class _OptionalBlock(_Generator): def render(self, replacement): # type: (Mapping[str,str]) -> str res = [(isinstance(x, _Generator), x.render(replacement)) for x in self._scope] - if any((t[0] and len(t[1]) != 0) for t in res): + if any((t[0] and t[1] is not None and len(t[1]) != 0) for t in res): return u"".join(t[1] for t in res) else: return u"" + def __eq__(self, other): + """ + :type other: _OptionalBlock + """ + return isinstance(other, _OptionalBlock) and self._scope == other._scope + _OPTIONAL_START = u'{' _OPTIONAL_END = u'}' @@ -230,8 +259,9 @@ def render(pattern, replacement): p = Pattern(pattern) return p(replacement), p.warnings + if __name__ == "__main__": # primitive test ;) - p = Pattern(u"[$Disc.]$Track - $Artist - $Title[ '['$Year']'") + p = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]}") d = {'$Disc': '', '$Track': '05', '$Artist': u'Grzegżółka', '$Title': u'Błona kapłona', '$Year': '2019'} - print(p(d).encode('utf8'), p.warnings) + assert p(d) == u"05 - Grzegżółka - Błona kapłona [2019]" diff --git a/headphones/pathrender_test.py b/headphones/pathrender_test.py new file mode 100644 index 00000000..4c24b99f --- /dev/null +++ b/headphones/pathrender_test.py @@ -0,0 +1,100 @@ +# encoding=utf8 +# This file is part of Headphones. +# +# Headphones is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Headphones is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Headphones. If not, see . +""" +Test module for pathrender. +""" +import headphones.pathrender as _pr +from headphones.pathrender import Pattern, Warnings + +from unittestcompat import TestCase + + +__author__ = "Andrzej Ciarkowski " + + +class PathRenderTest(TestCase): + """ + Tests for pathrender module. + """ + + def test_parsing(self): + """pathrender: pattern parsing""" + pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]}") + expected = [ + _pr._OptionalBlock([ + _pr._Replacement(u"$Disc"), + _pr._LiteralText(u".") + ]), + _pr._Replacement(u"$Track"), + _pr._LiteralText(u" - "), + _pr._Replacement(u"$Artist"), + _pr._LiteralText(u" - "), + _pr._Replacement(u"$Title"), + _pr._OptionalBlock([ + _pr._LiteralText(u" ["), + _pr._Replacement(u"$Year"), + _pr._LiteralText(u"]") + ]) + ] + self.assertEqual(expected, pattern._pattern) + self.assertItemsEqual([], pattern.warnings) + + def test_parsing_warnings(self): + """pathrender: pattern parsing with warnings""" + pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]") + self.assertEqual(set([Warnings.UNCLOSED_OPTIONAL]), pattern.warnings) + pattern = Pattern(u"{$Disc.}$Track - $Artist - $Title{ [$Year]'}") + self.assertEqual(set([ + Warnings.UNCLOSED_ESCAPE, + Warnings.UNCLOSED_OPTIONAL + ]), + pattern.warnings) + + def test_replacement(self): + """pathrender: _Replacement variable substitution""" + r = _pr._Replacement(u"$Title") + subst = {'$Title': 'foo', '$Track': 'bar'} + res = r.render(subst) + self.assertEqual(res, u'foo', 'check valid replacement') + subst = {} + res = r.render(subst) + self.assertEqual(res, u'$Title', 'check missing replacement') + subst = {'$Title': None} + res = r.render(subst) + self.assertEqual(res, '', 'check render() works with None') + + def test_literal(self): + """pathrender: _Literal text rendering""" + l = _pr._LiteralText(u"foo") + subst = {'$foo': 'bar'} + res = l.render(subst) + self.assertEqual(res, 'foo') + + def test_optional(self): + """pathrender: _OptionalBlock element processing""" + o = _pr._OptionalBlock([ + _pr._Replacement(u"$Title"), + _pr._LiteralText(u".foobar") + ]) + subst = {'$Title': 'foo', '$Track': 'bar'} + res = o.render(subst) + self.assertEqual(res, u'foo.foobar', 'check non-empty replacement') + subst = {'$Title': ''} + res = o.render(subst) + self.assertEqual(res, '', 'check empty replacement') + subst = {'$Title': None} + res = o.render(subst) + self.assertEqual(res, '', 'check render() works with None') From 97e405ee8c6d087f19124d3a8fb9356f565dd00e Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Sat, 27 Feb 2016 00:37:49 +0100 Subject: [PATCH 05/57] metadata.py: Use arbitrary variables from existing tags or database in renamer. This commit adds the possiblity to use variables from tags already present in the downloaded media files in file, folder and album art renaming routines. The variable names translate directly to MediaFile field names, so it is now possible to use variables like $mb_albumid, $genre, $bitrate, $samplerate, etc. The full list can be read from MediaFile.readable_fields(). --- headphones/metadata.py | 352 ++++++++++++++++++++++++++++++++++++ headphones/metadata_test.py | 148 +++++++++++++++ headphones/postprocessor.py | 185 ++++--------------- 3 files changed, 534 insertions(+), 151 deletions(-) create mode 100644 headphones/metadata.py create mode 100644 headphones/metadata_test.py diff --git a/headphones/metadata.py b/headphones/metadata.py new file mode 100644 index 00000000..3a423b32 --- /dev/null +++ b/headphones/metadata.py @@ -0,0 +1,352 @@ +# encoding=utf8 +# This file is part of Headphones. +# +# Headphones is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Headphones is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Headphones. If not, see . +""" +Track/album metadata handling routines. +""" + +from __future__ import print_function +from beets.mediafile import MediaFile, UnreadableFileError +import headphones +from headphones import logger +import os.path +import datetime + +__author__ = "Andrzej Ciarkowski " + + +class MetadataDict(dict): + """ + Dictionary which allows for case-insensitive, but case-preserving lookup, + allowing to put different values under $Album and $album, but still + finding some value if only single key is present and called with any + variation of the name's case. + + Keeps case-sensitive mapping in superclass dict, and case-insensitive ( + lowercase) in member variable self._lower. If case-sensitive lookup + fails, another case-insensitive attempt is made. + """ + def __setitem__(self, key, value): + super(MetadataDict, self).__setitem__(key, value) + self._lower.__setitem__(key.lower(), value) + + def add_items(self, items): + # type: (Iterable[Tuple[Any,Any]])->None + """ + Add (key,value) pairs to this dictionary using iterable as an input. + :param items: input items. + """ + for key, value in items: + self.__setitem__(key, value) + + def __init__(self, seq=None, **kwargs): + if isinstance(seq, MetadataDict): + super(MetadataDict, self).__init__(seq) + self._lower = dict(seq._lower) + else: + super(MetadataDict, self).__init__() + self._lower = {} + if seq is not None: + try: + self.add_items(seq.iteritems()) + except KeyError: + self.add_items(seq) + + def __getitem__(self, item): + try: + return super(MetadataDict, self).__getitem__(item) + except KeyError: + return self._lower.__getitem__(item.lower()) + + def __contains__(self, item): + return self._lower.__contains__(item.lower()) + + +class Vars: + """ + Metadata $variable names (only ones set explicitly by headphones). + """ + DISC = '$Disc' + TRACK = '$Track' + TITLE = '$Title' + ARTIST = '$Artist' + SORT_ARTIST = '$SortArtist' + ALBUM = '$Album' + YEAR = '$Year' + DATE = '$Date' + EXTENSION = '$Extension' + ORIGINAL_FOLDER = '$OriginalFolder' + FIRST_LETTER = '$First' + TYPE = '$Type' + TITLE_LOWER = TITLE.lower() + ARTIST_LOWER = ARTIST.lower() + SORT_ARTIST_LOWER = SORT_ARTIST.lower() + ALBUM_LOWER = ALBUM.lower() + ORIGINAL_FOLDER_LOWER = ORIGINAL_FOLDER.lower() + FIRST_LETTER_LOWER = FIRST_LETTER.lower() + TYPE_LOWER = TYPE.lower() + + +def _verify_var_type(val): + """ + Check if type of value is allowed as a variable in pathname substitution. + """ + return isinstance(val, (basestring, int, float, datetime.date)) + + +def _as_str(val): + if isinstance(val, basestring): + return val + else: + return str(val) + + +def _media_file_to_dict(mf, d): + """ + Populate dict with tags read from media file. + """ + for fld in mf.readable_fields(): + if 'art' == fld: + # skip embedded artwork as it's a BLOB + continue + val = getattr(mf, fld) + if val is None: + val = '' + # include only types with meaningful string representation + if _verify_var_type(val): + d['$' + fld] = _as_str(val) + + +def _row_to_dict(row, d): + """ + Populate dict with database row fields. + """ + for fld in row.keys(): + val = row[fld] + if val is None: + val = '' + if _verify_var_type(val): + d['$' + fld] = _as_str(val) + + +def _date_year(release): + # type: (sqlite3.Row)->Tuple[str,str] + """ + Extract release date and year from database row + """ + try: + date = release['ReleaseDate'] + except TypeError: + date = '' + + if date is not None: + year = date[:4] + else: + year = '' + return date, year + + +def file_metadata(path, release): + # type: (str,sqlite3.Row)->Tuple[Mapping[str,str],bool] + """ + Prepare metadata dictionary for path substitution, based on file name, + the tags stored within it and release info from the db. + :param path: media file path + :param release: database row with release info + :return: pair (dict,boolean indicating if Vars.TITLE is taken from tags or + file name). (None,None) if unable to parse the media file. + """ + try: + f = MediaFile(path) + except UnreadableFileError as ex: + logger.info("MediaFile couldn't parse: %s (%s)", + path.decode(headphones.SYS_ENCODING, 'replace'), + str(ex)) + return None, None + + res = MetadataDict() + # add existing tags first, these will get overwritten by musicbrainz from db + _media_file_to_dict(f, res) + # raw database fields come next + _row_to_dict(release, res) + + date, year = _date_year(release) + if not f.disc: + disc_number = '' + else: + disc_number = '%d' % f.disc + + if not f.track: + track_number = '' + else: + track_number = '%02d' % f.track + + if not f.title: + basename = os.path.basename( + path.decode(headphones.SYS_ENCODING, 'replace')) + title = os.path.splitext(basename)[0] + from_metadata = False + else: + title = f.title + from_metadata = True + + ext = os.path.splitext(path)[1] + if release['ArtistName'] == "Various Artists" and f.artist: + artist_name = f.artist + else: + artist_name = release['ArtistName'] + + if artist_name.startswith('The '): + sort_name = artist_name[4:] + ", The" + else: + sort_name = artist_name + + album_title = release['AlbumTitle'] + override_values = { + Vars.DISC: disc_number, + Vars.TRACK: track_number, + Vars.TITLE: title, + Vars.ARTIST: artist_name, + Vars.SORT_ARTIST: sort_name, + Vars.ALBUM: album_title, + Vars.YEAR: year, + Vars.DATE: date, + Vars.EXTENSION: ext, + Vars.TITLE_LOWER: title.lower(), + Vars.ARTIST_LOWER: artist_name.lower(), + Vars.SORT_ARTIST_LOWER: sort_name.lower(), + Vars.ALBUM_LOWER: album_title.lower(), + } + res.add_items(override_values.iteritems()) + return res, from_metadata + + +def _intersect(d1, d2): + # type: (Mapping,Mapping)->Mapping + """ + Create intersection (common part) of two dictionaries. + """ + res = {} + for key, val in d1.iteritems(): + if key in d2 and d2[key] == val: + res[key] = val + return res + + +def album_metadata(path, release, common_tags): + # type: (str,sqlite3.Row,Mapping[str,str])->Mapping[str,str] + """ + Prepare metadata dictionary for path substitution of album folder. + :param path: album path to prepare metadata for. + :param release: database row with release properties. + :param common_tags: common set of tags gathered from media files. + :return: metadata dictionary with substitution variables for rendering path. + """ + date, year = _date_year(release) + artist = release['ArtistName'].replace('/', '_') + album = release['AlbumTitle'].replace('/', '_') + release_type = release['Type'].replace('/', '_') + + if release['ArtistName'].startswith('The '): + sort_name = release['ArtistName'][4:] + ", The" + else: + sort_name = release['ArtistName'] + + if sort_name[0].isdigit(): + first_char = u'0-9' + else: + first_char = sort_name[0] + + for r, d, f in os.walk(path): + try: + orig_folder = os.path.basename( + os.path.normpath(r).decode(headphones.SYS_ENCODING, 'replace')) + except: + orig_folder = u'' + + override_values = { + Vars.ARTIST: artist, + Vars.SORT_ARTIST: sort_name, + Vars.ALBUM: album, + Vars.YEAR: year, + Vars.DATE: date, + Vars.TYPE: release_type, + Vars.ORIGINAL_FOLDER: orig_folder, + Vars.FIRST_LETTER: first_char.upper(), + Vars.ARTIST_LOWER: artist.lower(), + Vars.SORT_ARTIST_LOWER: sort_name.lower(), + Vars.ALBUM_LOWER: album.lower(), + Vars.TYPE_LOWER: release_type.lower(), + Vars.FIRST_LETTER_LOWER: first_char.lower(), + Vars.ORIGINAL_FOLDER_LOWER: orig_folder.lower() + } + res = MetadataDict(common_tags) + res.add_items(override_values.iteritems()) + return res + + +def albumart_metadata(release, common_tags): + # type: (sqlite3.Row,Mapping)->Mapping + """ + Prepare metadata dictionary for path subtitution of album art file. + :param release: database row with release properties. + :param common_tags: common set of tags gathered from media files. + :return: metadata dictionary with substitution variables for rendering path. + """ + date, year = _date_year(release) + override_values = { + Vars.ARTIST: release['ArtistName'], + Vars.ALBUM: release['AlbumTitle'], + Vars.YEAR: year, + Vars.DATE: date, + Vars.ARTIST_LOWER: release['ArtistName'].lower(), + Vars.ALBUM_LOWER: release['AlbumTitle'].lower() + } + res = MetadataDict(common_tags) + res.add_items(override_values.iteritems()) + return res + + +class AlbumMetadataBuilder(object): + """ + Facilitates building of album metadata as a common set of tags retrieved + from media files. + """ + + def __init__(self): + self._common = None + + def add_media_file(self, mf): + # type: (Mapping)->None + """ + Add metadata tags read from media file to album metadata. + :param mf: MediaFile + """ + md = {} + _media_file_to_dict(mf, md) + if self._common is None: + self._common = md + else: + self._common = _intersect(self._common, md) + + def build(self): + # type: (None)->Mapping + """ + Build case-insensitive, case-preserving dict from gathered metadata + tags. + :return: dictinary-like object filled with $variables based on common + tags. + """ + return MetadataDict(self._common) diff --git a/headphones/metadata_test.py b/headphones/metadata_test.py new file mode 100644 index 00000000..c4e6bb47 --- /dev/null +++ b/headphones/metadata_test.py @@ -0,0 +1,148 @@ +# encoding=utf8 +# This file is part of Headphones. +# +# Headphones is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Headphones is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Headphones. If not, see . +""" +Test module for metadata. +""" +import headphones.metadata as _md +from headphones.metadata import MetadataDict +import datetime + +from unittestcompat import TestCase + + +__author__ = "Andrzej Ciarkowski " + + +class _MockMediaFile(object): + + def __init__(self, artist, album, year, track, title, label): + self.artist = artist + self.album = album + self.year = year + self.track = track + self.title = title + self.label = label + self.art = 'THIS IS ART BLOB' + + @classmethod + def readable_fields(cls): + return 'artist', 'album', 'year', 'track', 'title', 'label', 'art' + + +class _MockDatabaseRow(object): + + def __init__(self, d): + self._dict = dict(d) + + def keys(self): + return self._dict.iterkeys() + + def __getitem__(self, item): + return self._dict[item] + + +class MetadataTest(TestCase): + """ + Tests for metadata module. + """ + + def test_metadata_dict_ci(self): + """MetadataDict: case-insensitive lookup""" + expected = u'naïve' + key_var = '$TitlE' + m = MetadataDict({key_var.lower(): u'naïve'}) + self.assertFalse('$track' in m) + self.assertTrue('$tITLe' in m, "cross-case lookup with 'in'") + self.assertEqual(m[key_var], expected, "cross-case lookup success") + self.assertEqual(m[key_var.lower()], expected, "same-case lookup " + "succes") + + def test_metadata_dict_cs(self): + """MetadataDice: case-preserving lookup""" + expected_var = u'NaïVe' + key_var = '$TitlE' + m = MetadataDict({ + key_var.lower(): expected_var.lower(), + key_var: expected_var + }) + self.assertFalse('$track' in m) + self.assertTrue('$tITLe' in m, "cross-case lookup with 'in'") + self.assertEqual(m[key_var.lower()], expected_var.lower(), + "case-preserving lookup lower") + self.assertEqual(m[key_var], expected_var, + "case-preserving lookup variable") + + def test_dict_intersect(self): + """metadata: check dictionary intersect function validity""" + d1 = { + 'one': 'one', + 'two': 'two', + 'three': 'zonk' + } + d2 = { + 'two': 'two', + 'three': 'three' + } + expected = { + 'two': 'two' + } + self.assertItemsEqual( + expected, _md._intersect(d1, d2), "check dictionary intersection " + "is common part indeed" + ) + del d1['two'] + expected = {} + self.assertItemsEqual( + expected, _md._intersect(d1, d2), "check intersection empty" + ) + + def test_album_metadata_builder(self): + """AlbumMetadataBuilder: check validity""" + mb = _md.AlbumMetadataBuilder() + f1 = _MockMediaFile('artist', 'album', 2000, 1, 'track1', 'Ant-Zen') + mb.add_media_file(f1) + f2 = _MockMediaFile('artist', 'album', 2000, 2, 'track2', 'Ant-Zen') + mb.add_media_file(f2) + + md = mb.build() + expected = { + _md.Vars.ARTIST_LOWER: 'artist', + _md.Vars.ALBUM_LOWER: 'album', + _md.Vars.YEAR.lower(): 2000, + '$label': 'Ant-Zen' + } + self.assertItemsEqual( + expected, md, "check AlbumMetadataBuilder validity" + ) + + def test_populate_from_row(self): + """metadata: check populating metadata from database row""" + row = _MockDatabaseRow({ + 'ArtistName': 'artist', + 'AlbumTitle': 'album', + 'ReleaseDate': datetime.date(2004, 11, 28), + 'Variation': 5, + 'WrongTyped': complex(1, -1) + }) + md = _md.MetadataDict() + _md._row_to_dict(row, md) + expected = { + '$ArtistName': 'artist', + '$AlbumTitle': 'album', + '$ReleaseDate': '2004-11-28', + '$Variation': '5' + } + self.assertItemsEqual(expected, md, "check _row_to_dict() valid") diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 194cb62c..3f50109b 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,6 +30,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge from headphones import db, albumart, librarysync from headphones import logger, helpers, request, mb, music_encoder +from headphones import metadata postprocessor_lock = threading.Lock() @@ -339,14 +340,13 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, files)) + builder = metadata.AlbumMetadataBuilder() # Check if files are valid media files and are writable, before the steps # below are executed. This simplifies errors and prevents unfinished steps. for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) - if f is None: - # this test is just to keep pyflakes from complaining about an unused variable - return + builder.add_media_file(f) except (FileTypeError, UnreadableFileError): logger.error("Track file is not a valid media file: %s. Not " \ "continuing.", downloaded_track.decode( @@ -378,6 +378,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, shutil.rmtree(new_folder) return + metadata_dict = builder.build() # start encoding if headphones.CONFIG.MUSIC_ENCODER: downloaded_track_list = music_encoder.encode(albumpath) @@ -413,7 +414,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, renameNFO(albumpath) if headphones.CONFIG.ADD_ALBUM_ART and artwork: - addAlbumArt(artwork, albumpath, release) + addAlbumArt(artwork, albumpath, release, metadata_dict) if headphones.CONFIG.CORRECT_METADATA: correctedMetadata = correctMetadata(albumid, release, downloaded_track_list) @@ -433,7 +434,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, 'No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to') albumpaths = [albumpath] elif headphones.CONFIG.MOVE_FILES and headphones.CONFIG.DESTINATION_DIR: - albumpaths = moveFiles(albumpath, release, tracks) + albumpaths = moveFiles(albumpath, release, metadata_dict) else: albumpaths = [albumpath] @@ -606,34 +607,20 @@ def embedAlbumArt(artwork, downloaded_track_list): continue -def addAlbumArt(artwork, albumpath, release): +def addAlbumArt(artwork, albumpath, release, metadata_dict): logger.info('Adding album art to folder') + md = metadata.album_metadata(albumpath, release, metadata_dict) - try: - date = release['ReleaseDate'] - except TypeError: - date = u'' + ext = ".jpg" + # PNGs are possibe here too + if artwork[:4] == '\x89PNG': + ext = ".png" - if date is not None: - year = date[:4] - else: - year = u'' + album_art_name = helpers.replace_all( + headphones.CONFIG.ALBUM_ART_FORMAT.strip(), md) + ext - values = {'$Artist': release['ArtistName'], - '$Album': release['AlbumTitle'], - '$Year': year, - '$Date': date, - '$artist': release['ArtistName'].lower(), - '$album': release['AlbumTitle'].lower(), - '$year': year, - '$date': date - } - - album_art_name = helpers.replace_all(headphones.CONFIG.ALBUM_ART_FORMAT.strip(), - values) + ".jpg" - - album_art_name = helpers.replace_illegal_chars(album_art_name).encode(headphones.SYS_ENCODING, - 'replace') + album_art_name = helpers.replace_illegal_chars(album_art_name).encode( + headphones.SYS_ENCODING, 'replace') if headphones.CONFIG.FILE_UNDERSCORES: album_art_name = album_art_name.replace(' ', '_') @@ -680,62 +667,15 @@ def renameNFO(albumpath): os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e)) -def moveFiles(albumpath, release, tracks): +def moveFiles(albumpath, release, metadata_dict): logger.info("Moving files: %s" % albumpath) - try: - date = release['ReleaseDate'] - except TypeError: - date = u'' - if date is not None: - year = date[:4] - else: - year = u'' + md = metadata.album_metadata(albumpath, release, metadata_dict) + folder = helpers.replace_all( + headphones.CONFIG.FOLDER_FORMAT.strip(), md, normalize=True) - artist = release['ArtistName'].replace('/', '_') - album = release['AlbumTitle'].replace('/', '_') if headphones.CONFIG.FILE_UNDERSCORES: - artist = artist.replace(' ', '_') - album = album.replace(' ', '_') - - releasetype = release['Type'].replace('/', '_') - - if release['ArtistName'].startswith('The '): - sortname = release['ArtistName'][4:] + ", The" - else: - sortname = release['ArtistName'] - - if sortname[0].isdigit(): - firstchar = u'0-9' - else: - firstchar = sortname[0] - - for r, d, f in os.walk(albumpath): - try: - origfolder = os.path.basename( - os.path.normpath(r).decode(headphones.SYS_ENCODING, 'replace')) - except: - origfolder = u'' - - values = {'$Artist': artist, - '$SortArtist': sortname, - '$Album': album, - '$Year': year, - '$Date': date, - '$Type': releasetype, - '$OriginalFolder': origfolder, - '$First': firstchar.upper(), - '$artist': artist.lower(), - '$sortartist': sortname.lower(), - '$album': album.lower(), - '$year': year, - '$date': date, - '$type': releasetype.lower(), - '$first': firstchar.lower(), - '$originalfolder': origfolder.lower() - } - - folder = helpers.replace_all(headphones.CONFIG.FOLDER_FORMAT.strip(), values, normalize=True) + folder = folder.replace(' ', '_') folder = helpers.replace_illegal_chars(folder, type="folder") folder = folder.replace('./', '_/').replace('/.', '/_') @@ -1080,82 +1020,25 @@ def embedLyrics(downloaded_track_list): def renameFiles(albumpath, downloaded_track_list, release): logger.info('Renaming files') - try: - date = release['ReleaseDate'] - except TypeError: - date = u'' - - if date is not None: - year = date[:4] - else: - year = u'' - # Until tagging works better I'm going to rely on the already provided metadata for downloaded_track in downloaded_track_list: - try: - f = MediaFile(downloaded_track) - except: - logger.info("MediaFile couldn't parse: %s", - downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) + md, from_metadata = metadata.file_metadata(downloaded_track, release) + if md is None: + # unable to parse media file, skip file continue - if not f.disc: - discnumber = '' - else: - discnumber = '%d' % f.disc - - if not f.track: - tracknumber = '' - else: - tracknumber = '%02d' % f.track - - if not f.title: - - basename = os.path.basename(downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) - title = os.path.splitext(basename)[0] - ext = os.path.splitext(basename)[1] - + ext = md[metadata.Vars.EXTENSION] + if not from_metadata: + title = md[metadata.Vars.TITLE] new_file_name = helpers.cleanTitle(title) + ext - else: - title = f.title + new_file_name = helpers.replace_all( + headphones.CONFIG.FILE_FORMAT.strip(), md + ).replace('/', '_') + ext - if release['ArtistName'] == "Various Artists" and f.artist: - artistname = f.artist - else: - artistname = release['ArtistName'] - - if artistname.startswith('The '): - sortname = artistname[4:] + ", The" - else: - sortname = artistname - - values = {'$Disc': discnumber, - '$Track': tracknumber, - '$Title': title, - '$Artist': artistname, - '$SortArtist': sortname, - '$Album': release['AlbumTitle'], - '$Year': year, - '$Date': date, - '$disc': discnumber, - '$track': tracknumber, - '$title': title.lower(), - '$artist': artistname.lower(), - '$sortartist': sortname.lower(), - '$album': release['AlbumTitle'].lower(), - '$year': year, - '$date': date - } - - ext = os.path.splitext(downloaded_track)[1] - - new_file_name = helpers.replace_all(headphones.CONFIG.FILE_FORMAT.strip(), - values).replace('/', '_') + ext - - new_file_name = helpers.replace_illegal_chars(new_file_name).encode(headphones.SYS_ENCODING, - 'replace') + new_file_name = helpers.replace_illegal_chars(new_file_name).encode( + headphones.SYS_ENCODING, 'replace') if headphones.CONFIG.FILE_UNDERSCORES: new_file_name = new_file_name.replace(' ', '_') @@ -1166,8 +1049,8 @@ def renameFiles(albumpath, downloaded_track_list, release): new_file = os.path.join(albumpath, new_file_name) if downloaded_track == new_file_name: - logger.debug("Renaming for: " + downloaded_track.decode(headphones.SYS_ENCODING, - 'replace') + " is not neccessary") + logger.debug("Renaming for: " + downloaded_track.decode( + headphones.SYS_ENCODING, 'replace') + " is not neccessary") continue logger.debug('Renaming %s ---> %s', From ef08a5067314445c3e0bca02d1dafe00528db45c Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 12:37:44 +0200 Subject: [PATCH 06/57] Deluge SSL and More Logging - Added option to use Deluge WebUI SSL using certificate file path - Added logging in order to understand other bugs --- data/interfaces/default/config.html | 6 + headphones/config.py | 1 + headphones/deluge.py | 216 +++++++++++++++++++--------- headphones/webserve.py | 1 + 4 files changed, 153 insertions(+), 71 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 9ba9f2aa..14254619 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -392,6 +392,12 @@ Usually http://localhost:8112 (requires WebUI plugin) +
+ + + Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match + hostname). +
diff --git a/headphones/config.py b/headphones/config.py index dd1ae9b6..a4dc5d27 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -68,6 +68,7 @@ _CONFIG_DEFINITIONS = { 'CUSTOMUSER': (str, 'General', ''), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), + 'DELUGE_CERT': (str, 'Deluge', ''), 'DELUGE_PASSWORD': (str, 'Deluge', ''), 'DELUGE_LABEL': (str, 'Deluge', ''), 'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''), diff --git a/headphones/deluge.py b/headphones/deluge.py index 5f79d29e..8d283ffa 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -48,6 +48,7 @@ import traceback delugeweb_auth = {} delugeweb_url = '' +deluge_verify_cert = False def addTorrent(link, data=None): @@ -158,14 +159,16 @@ def getTorrentFolder(result): ["total_done"] ], "id": 22}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] tries = 0 while result['total_done'] == 0 and tries < 10: tries += 1 time.sleep(5) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] post_data = json.dumps({"method": "web.get_torrent_status", @@ -183,7 +186,8 @@ def getTorrentFolder(result): ], "id": 23}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['save_path'] = json.loads(response.text)['result']['save_path'] result['name'] = json.loads(response.text)['result']['name'] @@ -198,30 +202,47 @@ def removeTorrent(torrentid, remove_data=False): if not any(delugeweb_auth): _get_auth() - result = False - post_data = json.dumps({"method": "core.remove_torrent", - "params": [ - torrentid, - remove_data - ], - "id": 25}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - result = json.loads(response.text)['result'] + try: + result = False + post_data = json.dumps({"method": "core.remove_torrent", + "params": [ + torrentid, + remove_data + ], + "id": 25}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + result = json.loads(response.text)['result'] - return result + return result + except Exception as e: + logger.error('Deluge: Removing torrent failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def _get_auth(): logger.debug('Deluge: Authenticating...') - global delugeweb_auth, delugeweb_url + global delugeweb_auth, delugeweb_url, deluge_verify_cert delugeweb_auth = {} delugeweb_host = headphones.CONFIG.DELUGE_HOST + delugeweb_cert = headphones.CONFIG.DELUGE_CERT delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD + logger.debug('Deluge: Using password %s***%s' % (delugeweb_password[0], delugeweb_password[-1])) if not delugeweb_host.startswith('http'): delugeweb_host = 'http://%s' % delugeweb_host + if delugeweb_cert is None or delugeweb_cert.strip() == '': + deluge_verify_cert = False + logger.debug('Deluge: No SSL certificate configured') + else: + deluge_verify_cert = delugeweb_cert + delugeweb_host = delugeweb_host.replace('http:', 'https:') + logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, delugeweb_host)) + if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] @@ -231,33 +252,47 @@ def _get_auth(): "params": [delugeweb_password], "id": 1}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None auth = json.loads(response.text)["result"] + auth_error = json.loads(response.text)["error"] + logger.debug('Deluge: Authentication result: %s, Error: %s' % (auth, auth_error)) delugeweb_auth = response.cookies + logger.debug('Deluge: Authentication cookies: %s' % str(delugeweb_auth.get_dict())) post_data = json.dumps({"method": "web.connected", "params": [], "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None connected = json.loads(response.text)['result'] + connected_error = json.loads(response.text)['error'] + logger.debug('Deluge: Connection result: %s, Error: %s' % (connected, connected_error)) if not connected: post_data = json.dumps({"method": "web.get_hosts", "params": [], "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None delugeweb_hosts = json.loads(response.text)['result'] @@ -270,9 +305,12 @@ def _get_auth(): "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None post_data = json.dumps({"method": "web.connected", @@ -280,9 +318,12 @@ def _get_auth(): "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None connected = json.loads(response.text)['result'] @@ -302,12 +343,16 @@ def _add_torrent_magnet(result): post_data = json.dumps({"method": "core.add_torrent_magnet", "params": [result['url'], {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent magnet failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None ''' def _add_torrent_url(result): @@ -318,7 +363,8 @@ def _add_torrent_url(result): post_data = json.dumps({"method": "web.download_torrent_from_url", "params": [result['url'], {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] @@ -336,7 +382,8 @@ def _add_torrent_file(result): post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] @@ -344,6 +391,7 @@ def _add_torrent_file(result): logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) + return None def setTorrentLabel(result): @@ -361,7 +409,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.get_labels', "params": [], "id": 3}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) labels = json.loads(response.text)['result'] if labels is not None: @@ -371,7 +420,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.add', "params": [label], "id": 4}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) logger.debug('Deluge: %s label added to Deluge' % label) except Exception as e: logger.error('Deluge: Setting label failed: %s' % str(e)) @@ -382,7 +432,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.set_torrent', "params": [result['hash'], label], "id": 5}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) logger.debug('Deluge: %s label added to torrent' % label) else: logger.debug('Deluge: Label plugin not detected') @@ -400,19 +451,27 @@ def setSeedRatio(result): if result['ratio']: ratio = result['ratio'] - if ratio: - post_data = json.dumps({"method": "core.set_torrent_stop_at_ratio", - "params": [result['hash'], True], - "id": 5}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - post_data = json.dumps({"method": "core.set_torrent_stop_ratio", - "params": [result['hash'], float(ratio)], - "id": 6}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if ratio: + post_data = json.dumps({"method": "core.set_torrent_stop_at_ratio", + "params": [result['hash'], True], + "id": 5}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + post_data = json.dumps({"method": "core.set_torrent_stop_ratio", + "params": [result['hash'], float(ratio)], + "id": 6}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting seed ratio failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def setTorrentPath(result): @@ -420,28 +479,36 @@ def setTorrentPath(result): if not any(delugeweb_auth): _get_auth() - if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: - post_data = json.dumps({"method": "core.set_torrent_move_completed", - "params": [result['hash'], True], - "id": 7}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: + post_data = json.dumps({"method": "core.set_torrent_move_completed", + "params": [result['hash'], True], + "id": 7}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: - move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY - else: - move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR + if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY + else: + move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR - if not os.path.exists(move_to): - logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to) - os.makedirs(move_to) - post_data = json.dumps({"method": "core.set_torrent_move_completed_path", - "params": [result['hash'], move_to], - "id": 8}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + if not os.path.exists(move_to): + logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to) + os.makedirs(move_to) + post_data = json.dumps({"method": "core.set_torrent_move_completed_path", + "params": [result['hash'], move_to], + "id": 8}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting torrent move-to directory failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def setTorrentPause(result): @@ -449,12 +516,19 @@ def setTorrentPause(result): if not any(delugeweb_auth): _get_auth() - if headphones.CONFIG.DELUGE_PAUSED: - post_data = json.dumps({"method": "core.pause_torrent", - "params": [[result['hash']]], - "id": 9}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if headphones.CONFIG.DELUGE_PAUSED: + post_data = json.dumps({"method": "core.pause_torrent", + "params": [[result['hash']]], + "id": 9}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting torrent paused failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None diff --git a/headphones/webserve.py b/headphones/webserve.py index 741a37f4..d22f0185 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1157,6 +1157,7 @@ class WebInterface(object): "transmission_username": headphones.CONFIG.TRANSMISSION_USERNAME, "transmission_password": headphones.CONFIG.TRANSMISSION_PASSWORD, "deluge_host": headphones.CONFIG.DELUGE_HOST, + "deluge_cert": headphones.CONFIG.DELUGE_CERT, "deluge_password": headphones.CONFIG.DELUGE_PASSWORD, "deluge_label": headphones.CONFIG.DELUGE_LABEL, "deluge_done_directory": headphones.CONFIG.DELUGE_DONE_DIRECTORY, From e2097fdb31f71dc4e5917f1e7f238e1a457b23f5 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 13:42:34 +0200 Subject: [PATCH 07/57] Expanded SSL Description --- data/interfaces/default/config.html | 2 +- headphones/deluge.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 14254619..a50035d3 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -396,7 +396,7 @@ Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match - hostname). + hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux.
diff --git a/headphones/deluge.py b/headphones/deluge.py index 8d283ffa..d48f0248 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -237,7 +237,7 @@ def _get_auth(): if delugeweb_cert is None or delugeweb_cert.strip() == '': deluge_verify_cert = False - logger.debug('Deluge: No SSL certificate configured') + logger.debug('Deluge: FYI no SSL certificate configured') else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') From 06ae4e97a08c731bf887e9072b3397000cb0d780 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 16:47:17 +0200 Subject: [PATCH 08/57] Handle UnicodeDecodeError - Attempt at handling encoding errors with specific torrent files --- headphones/deluge.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index d48f0248..770c7073 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -198,7 +198,7 @@ def getTorrentFolder(result): def removeTorrent(torrentid, remove_data=False): - + logger.debug('Deluge: Remove torrent %s' % torrentid) if not any(delugeweb_auth): _get_auth() @@ -387,6 +387,22 @@ def _add_torrent_file(result): result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] + except UnicodeDecodeError: + try: + # content is torrent file contents that needs to be encoded to base64 + post_data = json.dumps({"method": "core.add_torrent_file", + "params": [result['name'] + '.torrent', b64encode(result['content'].decode('utf-8').encode('utf8')), {}], + "id": 2}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + result['hash'] = json.loads(response.text)['result'] + logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + return json.loads(response.text)['result'] + except Exception as e: + logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None except Exception as e: logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() From 2c9c4d32db09972134d65aa79121f53073006120 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 22:48:37 +0200 Subject: [PATCH 09/57] Another Fix For Encoding Issues - Apparently some torrents are already proper UTF8 so we don't need to encode them... --- headphones/deluge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 770c7073..809298f7 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -390,8 +390,9 @@ def _add_torrent_file(result): except UnicodeDecodeError: try: # content is torrent file contents that needs to be encoded to base64 + # this time let's try leaving the encoding alone post_data = json.dumps({"method": "core.add_torrent_file", - "params": [result['name'] + '.torrent', b64encode(result['content'].decode('utf-8').encode('utf8')), {}], + "params": [result['name'] + '.torrent', b64encode(result['content']), {}], "id": 2}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) From c166e36c4ed75d7c4ae72b7ce1e00380ba11e3e4 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 11:54:35 +0200 Subject: [PATCH 10/57] Log Scrubber - Attempt to remove sensitive information from logs --- headphones/deluge.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 809298f7..2df96ea7 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -49,7 +49,20 @@ import traceback delugeweb_auth = {} delugeweb_url = '' deluge_verify_cert = False +scrub_logs = True +def _scrubber(text): + if scrub_logs: + try: + # URL parameter values + text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) + # Local host with port + text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) + #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) + except Exception as e: + logger.debug('Deluge: Scrubber failed: %s' % str(e)) + return text def addTorrent(link, data=None): try: @@ -57,13 +70,13 @@ def addTorrent(link, data=None): retid = False if link.startswith('magnet:'): - logger.debug('Deluge: Got a magnet link: %s' % link) + logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) result = {'type': 'magnet', 'url': link} retid = _add_torrent_magnet(result) elif link.startswith('http://') or link.startswith('https://'): - logger.debug('Deluge: Got a URL: %s' % link) + logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} torrentfile = '' @@ -73,16 +86,13 @@ def addTorrent(link, data=None): if r.status_code == 200: logger.debug('Deluge: 200 OK') torrentfile = r.text - #for chunk in r.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # torrentfile = torrentfile + chunk else: - logger.debug('Deluge: Trying to GET %s returned status %d' % (link, r.status_code)) + logger.debug('Deluge: Trying to GET %s returned status %d' % (_scrubber(link), r.status_code)) return False except Exception as e: logger.debug('Deluge: Download failed: %s' % str(e)) if 'announce' not in torrentfile[:40]: - logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % link) + logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False # Extract torrent name from .torrent try: @@ -241,7 +251,7 @@ def _get_auth(): else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') - logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, delugeweb_host)) + logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, _scrubber(delugeweb_host))) if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] From b89dcfde65791fc87f3a71e7aef58e631c734576 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 13:45:30 +0200 Subject: [PATCH 11/57] More Scrubbing and Logging and Auto-Switch to HTTPS - If Deluge is set to use HTTPS but there is no certificate configured in HP, use HTTPS without verifying the certificate - connection will still be encrypted but the certificate won't be validated - this will show up as an ERROR in the logs - Scrubbing session cookies --- headphones/deluge.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 2df96ea7..869e0ceb 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -58,6 +58,10 @@ def _scrubber(text): text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + # Session cookie + text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) + # Local Windows path + # TODO #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) except Exception as e: @@ -148,7 +152,7 @@ def addTorrent(link, data=None): logger.info('Deluge: Torrent sent to Deluge successfully (%s)' % retid) return retid else: - logger.info('Deluge returned status %s' % retid) + logger.info('Deluge: Returned status %s' % retid) return False except Exception as e: @@ -251,7 +255,7 @@ def _get_auth(): else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') - logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, _scrubber(delugeweb_host))) + logger.debug('Deluge: Using certificate %s, host is now %s' % (_scrubber(deluge_verify_cert), _scrubber(delugeweb_host))) if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] @@ -264,6 +268,19 @@ def _get_auth(): try: response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) + except ConnectionError: + try: + logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') + response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + # If the previous line didn't fail, change delugeweb_url for the rest of this session + logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!') + delugeweb_url = delugeweb_url.replace('http:', 'https:') + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() @@ -274,8 +291,7 @@ def _get_auth(): auth_error = json.loads(response.text)["error"] logger.debug('Deluge: Authentication result: %s, Error: %s' % (auth, auth_error)) delugeweb_auth = response.cookies - logger.debug('Deluge: Authentication cookies: %s' % str(delugeweb_auth.get_dict())) - + logger.debug('Deluge: Authentication cookies: %s' % _scrubber(str(delugeweb_auth.get_dict()))) post_data = json.dumps({"method": "web.connected", "params": [], "id": 10}) @@ -356,13 +372,13 @@ def _add_torrent_magnet(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent magnet failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False ''' def _add_torrent_url(result): @@ -376,7 +392,7 @@ def _add_torrent_url(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent URL failed: %s' % str(e)) @@ -395,30 +411,30 @@ def _add_torrent_file(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except UnicodeDecodeError: try: # content is torrent file contents that needs to be encoded to base64 - # this time let's try leaving the encoding alone + # this time let's try leaving the encoding as is post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content']), {}], "id": 2}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False except Exception as e: logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False def setTorrentLabel(result): From 9e943a9681fff1789d9d8a3d6cd96a9cd9fc28b9 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 16:26:34 +0200 Subject: [PATCH 12/57] Handle ConnectionError Properly - except requests.ConnectionError... --- headphones/deluge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 869e0ceb..72aa76ff 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -268,7 +268,7 @@ def _get_auth(): try: response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) - except ConnectionError: + except requests.ConnectionError: try: logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, From 8ead17e25e35d31849198616b8aa1db0010126c2 Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 10:15:36 +0200 Subject: [PATCH 13/57] Try Different User-Agent for Specific Sites - DLing not working for some users, possibly because of the site - maybe a different UA will make a difference - Make links lowercase before testing them for content --- headphones/deluge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 72aa76ff..79104bc9 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,16 +72,20 @@ def addTorrent(link, data=None): try: result = {} retid = False + special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] - if link.startswith('magnet:'): + if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) result = {'type': 'magnet', 'url': link} retid = _add_torrent_magnet(result) - elif link.startswith('http://') or link.startswith('https://'): + elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) - user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' + if link.lower().startswith(tuple(special_treatment_sites)): + user_agent = 'Headphones' + else: + user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} torrentfile = '' logger.debug('Deluge: Trying to download (GET)') @@ -118,7 +122,7 @@ def addTorrent(link, data=None): retid = _add_torrent_file(result) # elif link.endswith('.torrent') or data: - elif not (link.startswith('http://') or link.startswith('https://')): + elif not (link.lower().startswith('http://') or link.lower().startswith('https://')): if data: logger.debug('Deluge: Getting .torrent data') torrentfile = data From 3ac15d880f89dd6e6f938bfa6bd189e1352e01ae Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 10:40:22 +0200 Subject: [PATCH 14/57] Scrub Local Windows User Path - Scrub Windows user path from logs --- headphones/deluge.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 79104bc9..bde7856f 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -60,8 +60,10 @@ def _scrubber(text): text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) - # Local Windows path - # TODO + # Local Windows user path + if text.lower().startswith('c:\\users\\'): + k = text.split('\\') + text = '\\'.join([k[0], k[1], '.....', k[-1]]) #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) except Exception as e: From a8476e89f4a9976839b16517cf66621652db110e Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 12:17:13 +0200 Subject: [PATCH 15/57] Log User-Agent Switch --- headphones/deluge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/headphones/deluge.py b/headphones/deluge.py index bde7856f..e79482e2 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -85,6 +85,7 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) if link.lower().startswith(tuple(special_treatment_sites)): + logger.debug('Deluge: Trying different user-agent for this site') user_agent = 'Headphones' else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' From aff82e47554f6c63247fa29f33341adc9bfa6d8d Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 13:16:33 +0200 Subject: [PATCH 16/57] Always Authenticate When Adding Torrents - Avoid expired session cookies, just authenticate always --- headphones/deluge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/headphones/deluge.py b/headphones/deluge.py index e79482e2..1e4ddf19 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,6 +72,10 @@ def _scrubber(text): def addTorrent(link, data=None): try: + # Authenticate anyway + logger.debug('Deluge: addTorrent Authentication') + _get_auth() + result = {} retid = False special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] From 69405d4f0df093e0cca513c66b4cedaadaf0867d Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 14:54:29 +0200 Subject: [PATCH 17/57] Let Deluge Handle Certain Downloads - Use Deluge's web.download_torrent_from_url for specific sites --- headphones/deluge.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 1e4ddf19..b42cd8dc 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -89,8 +89,13 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) if link.lower().startswith(tuple(special_treatment_sites)): - logger.debug('Deluge: Trying different user-agent for this site') - user_agent = 'Headphones' + #logger.debug('Deluge: Trying different user-agent for this site') + #user_agent = 'Headphones' + # This method will make Deluge download the file + logger.debug('Deluge: Letting Deluge download this') + local_torrent_path = _add_torrent_url({'url': link}) + logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) + return addTorrent(local_torrent_path) else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} @@ -391,7 +396,7 @@ def _add_torrent_magnet(result): logger.error('; '.join(formatted_lines)) return False -''' + def _add_torrent_url(result): logger.debug('Deluge: Adding URL') if not any(delugeweb_auth): @@ -399,15 +404,17 @@ def _add_torrent_url(result): try: post_data = json.dumps({"method": "web.download_torrent_from_url", "params": [result['url'], {}], - "id": 2}) + "id": 32}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) - result['hash'] = json.loads(response.text)['result'] + result['location'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent URL failed: %s' % str(e)) -''' + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return False def _add_torrent_file(result): @@ -430,7 +437,7 @@ def _add_torrent_file(result): # this time let's try leaving the encoding as is post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content']), {}], - "id": 2}) + "id": 22}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] From ac91e17f868fc1e6007f7d2a8568158ebec27443 Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 23:39:38 +0200 Subject: [PATCH 18/57] More Special Sites --- headphones/deluge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index b42cd8dc..933871e8 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -78,7 +78,7 @@ def addTorrent(link, data=None): result = {} retid = False - special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] + special_treatment_sites = ['https://what.cd/', 'http://what.cd/', 'https://waffles.fm/', 'http://waffles.fm/'] if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) From 73a55e81b8442e393072e702d44b73d40032bd13 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 15:12:18 +0200 Subject: [PATCH 19/57] Better Downloading of Files - Reading downloaded files now uses response.content instead of response.text which caused loss of file data due to encoding shenanigans. This fixes the WHAT issue. - Fix URL for WFFLS downloads - Special User-Agent for WHAT downloads, still not sure if this is necessary --- headphones/deluge.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 933871e8..3c178d68 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -78,7 +78,8 @@ def addTorrent(link, data=None): result = {} retid = False - special_treatment_sites = ['https://what.cd/', 'http://what.cd/', 'https://waffles.fm/', 'http://waffles.fm/'] + url_what = ['https://what.cd/', 'http://what.cd/'] + url_waffles = ['https://waffles.fm/', 'http://waffles.fm/'] if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) @@ -88,14 +89,17 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) - if link.lower().startswith(tuple(special_treatment_sites)): - #logger.debug('Deluge: Trying different user-agent for this site') - #user_agent = 'Headphones' + if link.lower().startswith(tuple(url_waffles)): + if 'rss=' not in link: + link = link + '&rss=1' + if link.lower().startswith(tuple(url_what)): + logger.debug('Deluge: Using different User-Agent for this site') + user_agent = 'Headphones' # This method will make Deluge download the file - logger.debug('Deluge: Letting Deluge download this') - local_torrent_path = _add_torrent_url({'url': link}) - logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) - return addTorrent(local_torrent_path) + #logger.debug('Deluge: Letting Deluge download this') + #local_torrent_path = _add_torrent_url({'url': link}) + #logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) + #return addTorrent(local_torrent_path) else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} @@ -105,21 +109,22 @@ def addTorrent(link, data=None): r = requests.get(link, headers=headers) if r.status_code == 200: logger.debug('Deluge: 200 OK') - torrentfile = r.text + # .text will ruin the encoding for some torrents + torrentfile = r.content else: logger.debug('Deluge: Trying to GET %s returned status %d' % (_scrubber(link), r.status_code)) return False except Exception as e: logger.debug('Deluge: Download failed: %s' % str(e)) - if 'announce' not in torrentfile[:40]: + if 'announce' not in str(torrentfile)[:40]: logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False # Extract torrent name from .torrent try: logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0]) + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length] + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] except Exception as e: logger.debug('Deluge: Could not get torrent name, getting file name') # get last part of link/path (name only) @@ -127,7 +132,7 @@ def addTorrent(link, data=None): # remove '.torrent' suffix if name[-len('.torrent'):] == '.torrent': name = name[:-len('.torrent')] - logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40])) + logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, 'content': torrentfile} @@ -145,9 +150,9 @@ def addTorrent(link, data=None): # Extract torrent name from .torrent try: logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0]) + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length] + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] except Exception as e: logger.debug('Deluge: Could not get torrent name, getting file name') # get last part of link/path (name only) @@ -155,7 +160,7 @@ def addTorrent(link, data=None): # remove '.torrent' suffix if name[-len('.torrent'):] == '.torrent': name = name[:-len('.torrent')] - logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40])) + logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, 'content': torrentfile} @@ -260,7 +265,7 @@ def _get_auth(): delugeweb_host = headphones.CONFIG.DELUGE_HOST delugeweb_cert = headphones.CONFIG.DELUGE_CERT delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD - logger.debug('Deluge: Using password %s***%s' % (delugeweb_password[0], delugeweb_password[-1])) + logger.debug('Deluge: Using password %s******%s' % (delugeweb_password[0], delugeweb_password[-1])) if not delugeweb_host.startswith('http'): delugeweb_host = 'http://%s' % delugeweb_host From 1a56e24def62963197906465d59ae9379e28dd4c Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 15:35:20 +0200 Subject: [PATCH 20/57] Added SSL/TLS Disclaimer - Currently self-signed certificates can't be verified --- data/interfaces/default/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index a50035d3..b6e41303 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -396,7 +396,7 @@ Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match - hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux. + hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux. Leave this blank if you are using a self-signed certificate.
From a7f2329fee2ea63ae062db417c56003ad3188803 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 23:19:30 +0200 Subject: [PATCH 21/57] Travis Fixes --- headphones/deluge.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 3c178d68..33467783 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -51,13 +51,14 @@ delugeweb_url = '' deluge_verify_cert = False scrub_logs = True + def _scrubber(text): if scrub_logs: try: # URL parameter values text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port - text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) # Local Windows user path @@ -70,6 +71,7 @@ def _scrubber(text): logger.debug('Deluge: Scrubber failed: %s' % str(e)) return text + def addTorrent(link, data=None): try: # Authenticate anyway @@ -287,12 +289,12 @@ def _get_auth(): "params": [delugeweb_password], "id": 1}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except requests.ConnectionError: try: logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') - response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) # If the previous line didn't fail, change delugeweb_url for the rest of this session logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!') @@ -317,7 +319,7 @@ def _get_auth(): "params": [], "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -334,7 +336,7 @@ def _get_auth(): "params": [], "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -352,7 +354,7 @@ def _get_auth(): "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -365,7 +367,7 @@ def _get_auth(): "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) From 30a3421710da3fa25c12f0d1e6a923e5a47c2e57 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 23:32:13 +0200 Subject: [PATCH 22/57] More Travis - Not sure what this is but I think this should fix it... --- headphones/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 4b36acf7..daf067a5 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -143,7 +143,7 @@ def initialize(config_file): SOFT_CHROOT = SoftChroot(str(CONFIG.SOFT_CHROOT)) if SOFT_CHROOT.isEnabled(): logger.info("Soft-chroot enabled for dir: %s", str(CONFIG.SOFT_CHROOT)) - except exceptions.SoftChrootError as e: + except headphones.exceptions.SoftChrootError as e: logger.error("SoftChroot error: %s", e) raise e From c9a6125d9208a5a90af6a3f06f0a78059019af2a Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Thu, 3 Mar 2016 08:51:24 +0100 Subject: [PATCH 23/57] __init__.py: Fix pyflakes failing due to invalid use of headphones.exceptions. --- headphones/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 4b36acf7..dcb0f67d 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -30,7 +30,7 @@ from apscheduler.triggers.interval import IntervalTrigger from headphones import versioncheck, logger import headphones.config from headphones.softchroot import SoftChroot -import headphones.exceptions +from headphones import exceptions # (append new extras to the end) POSSIBLE_EXTRAS = [ From 147f62d87a1c5e4d9fd95f7737c43b3114784d8c Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 3 Mar 2016 14:23:12 +0200 Subject: [PATCH 24/57] Removed Folder Name - Folder name (folder_name) is set to Artist - Album by default - use that instead --- headphones/searcher.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 5c8b4144..a0d7ad8d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -921,12 +921,13 @@ def send_to_downloader(data, bestqual, album): deluge.setTorrentPath({'hash': torrentid}) # I only just realized this function is useless... - folder_name = deluge.getTorrentFolder({'hash': torrentid}) - if folder_name: - logger.info('Torrent folder name: %s' % folder_name) - else: - logger.error('Torrent folder name could not be determined') - return + # Hadn't realized folder_name was already being set to Artist - Album + #folder_name = deluge.getTorrentFolder({'hash': torrentid}) + #if folder_name: + # logger.info('Torrent folder name: %s' % folder_name) + #else: + # logger.error('Torrent folder name could not be determined') + # return except Exception as e: logger.error('Error sending torrent to Deluge: %s' % str(e)) From e05699e7ba73ebf8b3f1f5b9bd1ab7497e4fe4fa Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 3 Mar 2016 14:37:55 +0200 Subject: [PATCH 25/57] Specify Name When Sending to Deluge Module - Searcher should send "Artist - Album" string to Deluge so we don't have to guess the name --- headphones/deluge.py | 56 ++++++++++++++++++++++-------------------- headphones/searcher.py | 4 +-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 33467783..1acdd851 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,7 +72,7 @@ def _scrubber(text): return text -def addTorrent(link, data=None): +def addTorrent(link, data=None, name=None): try: # Authenticate anyway logger.debug('Deluge: addTorrent Authentication') @@ -121,19 +121,20 @@ def addTorrent(link, data=None): if 'announce' not in str(torrentfile)[:40]: logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False - # Extract torrent name from .torrent - try: - logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) - logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] - except Exception as e: - logger.debug('Deluge: Could not get torrent name, getting file name') - # get last part of link/path (name only) - name = link.split('\\')[-1].split('/')[-1] - # remove '.torrent' suffix - if name[-len('.torrent'):] == '.torrent': - name = name[:-len('.torrent')] + if not name: + # Extract torrent name from .torrent + try: + logger.debug('Deluge: Getting torrent name length') + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) + logger.debug('Deluge: Getting torrent name') + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] + except Exception as e: + logger.debug('Deluge: Could not get torrent name, getting file name') + # get last part of link/path (name only) + name = link.split('\\')[-1].split('/')[-1] + # remove '.torrent' suffix + if name[-len('.torrent'):] == '.torrent': + name = name[:-len('.torrent')] logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, @@ -149,19 +150,20 @@ def addTorrent(link, data=None): logger.debug('Deluge: Getting .torrent file') with open(link, 'rb') as f: torrentfile = f.read() - # Extract torrent name from .torrent - try: - logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) - logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] - except Exception as e: - logger.debug('Deluge: Could not get torrent name, getting file name') - # get last part of link/path (name only) - name = link.split('\\')[-1].split('/')[-1] - # remove '.torrent' suffix - if name[-len('.torrent'):] == '.torrent': - name = name[:-len('.torrent')] + if not name: + # Extract torrent name from .torrent + try: + logger.debug('Deluge: Getting torrent name length') + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) + logger.debug('Deluge: Getting torrent name') + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] + except Exception as e: + logger.debug('Deluge: Could not get torrent name, getting file name') + # get last part of link/path (name only) + name = link.split('\\')[-1].split('/')[-1] + # remove '.torrent' suffix + if name[-len('.torrent'):] == '.torrent': + name = name[:-len('.torrent')] logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, diff --git a/headphones/searcher.py b/headphones/searcher.py index a0d7ad8d..d4a2b9c9 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -895,9 +895,9 @@ def send_to_downloader(data, bestqual, album): try: # Add torrent if bestqual[3] == 'rutracker.org': - torrentid = deluge.addTorrent('', data) + torrentid = deluge.addTorrent('', data, name=folder_name) else: - torrentid = deluge.addTorrent(bestqual[2]) + torrentid = deluge.addTorrent(bestqual[2], name=folder_name) if not torrentid: logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?") From fd8fb4529c11a07f45be574de170bd9ffac45935 Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Sat, 27 Feb 2016 23:31:16 +0100 Subject: [PATCH 26/57] helpers: Replace cleanName() implementation to get much higher match ratio. Track matching is performed using 'CleanName' which up to now was obtained in convoluted way, which effectively removed any non-ascii alphanumeric characters but at the same time left some trash preventing the names to be matched due to whitespace differences. Current implementation performs most of the transliteration using Unicode NFD decomposition to remove diacritical marks from characters in Latin scripts, leaving the others intact. Only alphanumeric chars are included in resulting string and all the spaces are coalesced. Based on observations on several-tens GiB library, this allows for much better ratio of automatic track matches. --- headphones/helpers.py | 103 +++++++++++++++++++++++++++++++++++-- headphones/helpers_test.py | 48 +++++++++++++++++ headphones/importer.py | 4 +- headphones/librarysync.py | 10 ++-- headphones/mb.py | 2 +- headphones/webserve.py | 28 +++++----- 6 files changed, 169 insertions(+), 26 deletions(-) create mode 100644 headphones/helpers_test.py diff --git a/headphones/helpers.py b/headphones/helpers.py index 5e836178..076468fd 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # This file is part of Headphones. # # Headphones is free software: you can redistribute it and/or modify @@ -219,12 +220,104 @@ def replace_illegal_chars(string, type="file"): return string -def cleanName(string): - pass1 = latinToAscii(string).lower() - out_string = re.sub('[\.\-\/\!\@\#\$\%\^\&\*\(\)\+\-\"\'\,\;\:\[\]\{\}\<\>\=\_]', '', - pass1).encode('utf-8') +_CN_RE1 = re.compile(ur'[^\w]+', re.UNICODE) +_CN_RE2 = re.compile(ur'[\s_]+', re.UNICODE) - return out_string + +_XLATE_GRAPHICAL_AND_DIACRITICAL = { + # Translation table. + # Covers the following letters, for which NFD fails because of lack of + # combining character: + # ©ª«®²³¹»¼½¾ÆÐØÞßæðøþĐđĦħıIJijĸĿŀŁłŒœŦŧDŽDždžLJLjljNJNjnjǤǥDZDzdzȤȥ. This + # includes also some graphical symbols which can be easily replaced and + # usually are written by people who don't have appropriate keyboard layout. + u'©': '(C)', u'ª': 'a.', u'«': '<<', u'®': '(R)', u'²': '2', u'³': '3', + u'¹': '1', u'»': '>>', u'¼': ' 1/4 ', u'½': ' 1/2 ', u'¾': ' 3/4 ', + u'Æ': 'AE', u'Ð': 'D', u'Ø': 'O', u'Þ': 'Th', u'ß': 'ss', u'æ': 'ae', + u'ð': 'd', u'ø': 'o', u'þ': 'th', u'Đ': 'D', u'đ': 'd', u'Ħ': 'H', + u'ħ': 'h', u'ı': 'i', u'IJ': 'IJ', u'ij': 'ij', u'ĸ': 'q', u'Ŀ': 'L', + u'ŀ': 'l', u'Ł': 'L', u'ł': 'l', u'Œ': 'OE', u'œ': 'oe', u'Ŧ': 'T', + u'ŧ': 't', u'DŽ': 'DZ', u'Dž': 'Dz', u'LJ': 'LJ', u'Lj': 'Lj', + u'lj': 'lj', u'NJ': 'NJ', u'Nj': 'Nj', u'nj': 'nj', + u'Ǥ': 'G', u'ǥ': 'g', u'DZ': 'DZ', u'Dz': 'Dz', u'dz': 'dz', + u'Ȥ': 'Z', u'ȥ': 'z', u'№': 'No.', + u'º': 'o.', # normalize Nº abbrev (popular w/ classical music), + # this is 'masculine ordering indicator', not degree +} + +_XLATE_SPECIAL = { + # Translation table. + # Cover additional special characters processing normalization. + u"'": '', # replace apostrophe with nothing + u'&': ' and ', # expand & to ' and ' +} + + +def _translate(s, dictionary): + # type: (basestring,Mapping[basestring,basestring])->basestring + return ''.join(dictionary.get(x, x) for x in s) + + +_COMBINING_RANGES = ( + (0x0300, 0x036f), # Combining Diacritical Marks + (0x1ab0, 0x1aff), # Combining Diacritical Marks Extended + (0x20d0, 0x20ff), # Combining Diacritical Marks for Symbols + (0x1dc0, 0x1dff) # Combining Diacritical Marks Supplement +) + + +def _is_unicode_combining(u): + # type: (unicode)->bool + """ + Check if input unicode is combining diacritical mark. + """ + i = ord(u) + for r in _COMBINING_RANGES: + if r[0] <= i <= r[1]: + return True + return False + + +def _transliterate(u, xlate): + # type: (unicode)->unicode + """ + Perform transliteration using the specified dictionary + """ + u = unicodedata.normalize('NFD', u) + u = u''.join([u'' if _is_unicode_combining(x) else x for x in u]) + u = _translate(u, xlate) + # at this point output is either unicode, or plain ascii + return unicode(u) + + +def clean_name(s): + # type: (basestring)->unicode + """Remove non-alphanumeric characters from the string, perform + normalization and substitution of some special characters; coalesce spaces. + :param s: string to clean up, possibly unicode one. + :return: cleaned-up version of input string. + """ + if not isinstance(s, unicode): + # ignore extended chars if someone was dumb enough to pass non-ascii + # narrow string here, use only unicode for meaningful texts + u = unicode(s, 'ascii', 'replace') + else: + u = s + # 1. don't bother doing normalization NFKC, rather transliterate + # using special translation table + u = _transliterate(u, _XLATE_GRAPHICAL_AND_DIACRITICAL) + # 2. normalize NFKC the result + u = unicodedata.normalize('NFKC', u) + # 3. translate spacials + u = _translate(u, _XLATE_SPECIAL) + # 4. replace any non-alphanumeric character sequences by spaces + u = _CN_RE1.sub(u' ', u) + # 5. coalesce interleaved space/underscore sequences + u = _CN_RE2.sub(u' ', u) + # 6. trim + u = u.strip() + # 7. lowercase + return u def cleanTitle(title): diff --git a/headphones/helpers_test.py b/headphones/helpers_test.py new file mode 100644 index 00000000..2033aab4 --- /dev/null +++ b/headphones/helpers_test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +from unittestcompat import TestCase +from headphones.helpers import clean_name + + +class HelpersTest(TestCase): + + def test_clean_name(self): + """helpers: check correctness of clean_name() function""" + cases = { + u' Weiße & rose ': 'Weisse and rose', + u'Multiple / spaces': 'Multiple spaces', + u'Kevin\'s m²': 'Kevins m2', + u'Symphonęy Nº9': 'Symphoney No.9', + u'ÆæßðÞIJij': u'AeaessdThIJıj', + u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' + 'apoplexy remix', + u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' + 'zbojow', + u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', + u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', + u'L\'Arc~en~Ciel': 'larc en ciel', + u'Orquesta de la Luz (オルケスタ・デ・ラ・ルス)': + u'Orquesta de la Luz オルケスタ デ ラ ルス' + + } + for first, second in cases.iteritems(): + nf = clean_name(first).lower() + ns = clean_name(second).lower() + self.assertEqual( + nf, ns, u"check cleaning of case (%s," + u"%s)" % (nf, ns) + ) + + def test_clean_name_nonunicode(self): + """helpers: check if clean_name() works on non-unicode input""" + input = 'foo $ bar/BAZ' + test = clean_name(input).lower() + expected = 'foo bar baz' + self.assertEqual( + test, expected, "check clean_name() works on non-unicode" + ) + input = 'fóó $ BAZ' + test = clean_name(input).lower() + expected = clean_name('%fóó baz ').lower() + self.assertEqual( + test, expected, "check clean_name() with narrow non-ascii input" + ) diff --git a/headphones/importer.py b/headphones/importer.py index 3cc7f141..9a02ce49 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -374,7 +374,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): for track in hybridrelease['Tracks']: - cleanname = helpers.cleanName( + cleanname = helpers.clean_name( artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], @@ -710,7 +710,7 @@ def addReleaseById(rid, rgid=None): myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: - cleanname = helpers.cleanName( + cleanname = helpers.clean_name( release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 367d160c..e4e7a53f 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -138,7 +138,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: - CleanName = helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) + CleanName = helpers.clean_name(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None @@ -332,15 +332,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [ x for x in unique_artists - if helpers.cleanName(x).lower() not in [ - helpers.cleanName(y[0]).lower() + if helpers.clean_name(x).lower() not in [ + helpers.clean_name(y[0]).lower() for y in current_artists ] ] artists_checked = [ x for x in unique_artists - if helpers.cleanName(x).lower() in [ - helpers.cleanName(y[0]).lower() + if helpers.clean_name(x).lower() in [ + helpers.clean_name(y[0]).lower() for y in current_artists ] ] diff --git a/headphones/mb.py b/headphones/mb.py index 38b952e6..619908ba 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -637,7 +637,7 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False): for track in release['Tracks']: - cleanname = helpers.cleanName( + cleanname = helpers.clean_name( release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], diff --git a/headphones/webserve.py b/headphones/webserve.py index 741a37f4..fc7b2d72 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -29,7 +29,7 @@ import urllib2 import os import re from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers -from headphones.helpers import checked, radio, today, cleanName +from headphones.helpers import checked, radio, today, clean_name from mako.lookup import TemplateLookup from mako import exceptions import headphones @@ -577,7 +577,7 @@ class WebInterface(object): for albums in have_albums: # Have to skip over manually matched tracks if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: - original_clean = helpers.cleanName( + original_clean = helpers.clean_name( albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle']) # else: # original_clean = None @@ -595,10 +595,12 @@ class WebInterface(object): # unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] check = set( - [(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in + [(clean_name(d['ArtistName']).lower(), + clean_name(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) unmatchedalbums = [d for d in have_album_dictionary if ( - cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] + clean_name(d['ArtistName']).lower(), + clean_name(d['AlbumTitle']).lower()) not in check] return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) @@ -622,8 +624,8 @@ class WebInterface(object): (artist, album)) elif action == "matchArtist": - existing_artist_clean = helpers.cleanName(existing_artist).lower() - new_artist_clean = helpers.cleanName(new_artist).lower() + existing_artist_clean = helpers.clean_name(existing_artist).lower() + new_artist_clean = helpers.clean_name(new_artist).lower() if new_artist_clean != existing_artist_clean: have_tracks = myDB.action( 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', @@ -668,10 +670,10 @@ class WebInterface(object): "Artist %s already named appropriately; nothing to modify" % existing_artist) elif action == "matchAlbum": - existing_artist_clean = helpers.cleanName(existing_artist).lower() - new_artist_clean = helpers.cleanName(new_artist).lower() - existing_album_clean = helpers.cleanName(existing_album).lower() - new_album_clean = helpers.cleanName(new_album).lower() + existing_artist_clean = helpers.clean_name(existing_artist).lower() + new_artist_clean = helpers.clean_name(new_artist).lower() + existing_album_clean = helpers.clean_name(existing_album).lower() + new_album_clean = helpers.clean_name(new_album).lower() existing_clean_string = existing_artist_clean + " " + existing_album_clean new_clean_string = new_artist_clean + " " + new_album_clean if existing_clean_string != new_clean_string: @@ -728,7 +730,7 @@ class WebInterface(object): 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have') for albums in manualalbums: if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: - original_clean = helpers.cleanName( + original_clean = helpers.clean_name( albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle']) if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums[ 'CleanName'] != original_clean: @@ -769,7 +771,7 @@ class WebInterface(object): [artist]) update_count = 0 for tracks in update_clean: - original_clean = helpers.cleanName( + original_clean = helpers.clean_name( tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[ 'TrackTitle']).lower() album = tracks['AlbumTitle'] @@ -797,7 +799,7 @@ class WebInterface(object): (artist, album)) update_count = 0 for tracks in update_clean: - original_clean = helpers.cleanName( + original_clean = helpers.clean_name( tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[ 'TrackTitle']).lower() track_title = tracks['TrackTitle'] From 5bf465b88379623e7fedcb3f532ce7822ff20904 Mon Sep 17 00:00:00 2001 From: Noam Date: Fri, 4 Mar 2016 15:29:52 +0200 Subject: [PATCH 27/57] Go Back to Previous Folder Name - Go back to previous folder name method --- headphones/deluge.py | 5 +++-- headphones/searcher.py | 19 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 1acdd851..e07cc7c1 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -58,7 +58,8 @@ def _scrubber(text): # URL parameter values text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port - text = re.sub('\:\/\/.*\:', '://REMOVED:', text) + # text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # just host + text = re.sub('\:\/\/.*\:[0-9]*', '://REMOVED:', text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) # Local Windows user path @@ -197,7 +198,7 @@ def getTorrentFolder(result): result['hash'], ["total_done"] ], - "id": 22}) + "id": 21}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] diff --git a/headphones/searcher.py b/headphones/searcher.py index d4a2b9c9..ee561823 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -895,9 +895,9 @@ def send_to_downloader(data, bestqual, album): try: # Add torrent if bestqual[3] == 'rutracker.org': - torrentid = deluge.addTorrent('', data, name=folder_name) + torrentid = deluge.addTorrent('', data) else: - torrentid = deluge.addTorrent(bestqual[2], name=folder_name) + torrentid = deluge.addTorrent(bestqual[2]) if not torrentid: logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?") @@ -920,14 +920,13 @@ def send_to_downloader(data, bestqual, album): if headphones.CONFIG.DELUGE_DONE_DIRECTORY: deluge.setTorrentPath({'hash': torrentid}) - # I only just realized this function is useless... - # Hadn't realized folder_name was already being set to Artist - Album - #folder_name = deluge.getTorrentFolder({'hash': torrentid}) - #if folder_name: - # logger.info('Torrent folder name: %s' % folder_name) - #else: - # logger.error('Torrent folder name could not be determined') - # return + # Get folder name from Deluge, it's usually the torrent name + folder_name = deluge.getTorrentFolder({'hash': torrentid}) + if folder_name: + logger.info('Torrent folder name: %s' % folder_name) + else: + logger.error('Torrent folder name could not be determined') + return except Exception as e: logger.error('Error sending torrent to Deluge: %s' % str(e)) From 9b9d728ce380b88f636a5181ccedfda3c94f6eab Mon Sep 17 00:00:00 2001 From: Noam Date: Fri, 4 Mar 2016 17:44:44 +0200 Subject: [PATCH 28/57] Deluge Move-To Directory - Deluge move-to directory only relevant if Deluge is the chosen torrent downloader --- headphones/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 507a5571..ef87624f 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -46,7 +46,7 @@ def checkFolder(): if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR else: - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY else: download_dir = headphones.CONFIG.DOWNLOAD_TORRENT_DIR From f1e70789022fc2de63cbfc770870d877d4cb13cd Mon Sep 17 00:00:00 2001 From: satreix Date: Fri, 4 Mar 2016 12:47:51 -0800 Subject: [PATCH 29/57] pep: fix e261 --- .pep8 | 3 +-- headphones/postprocessor.py | 2 +- headphones/searcher.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pep8 b/.pep8 index daa763d1..8889c362 100644 --- a/.pep8 +++ b/.pep8 @@ -6,10 +6,9 @@ # E126 continuation line over-indented for hanging indent # E127 continuation line over-indented for visual indent # E128 continuation line under-indented for visual indent -# E261 at least two spaces before inline comment # E262 inline comment should start with '# ' # E265 block comment should start with '# ' # E501 line too long (312 > 160 characters) # E502 the backslash is redundant between brackets -ignore = E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E501,E502 +ignore = E121,E122,E123,E124,E125,E126,E127,E128,E262,E265,E501,E502 max-line-length = 160 diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 507a5571..e9748f5b 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -457,7 +457,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, release['ArtistName'], release['AlbumTitle'])) if headphones.CONFIG.TORRENT_DOWNLOADER == 1: torrent_removed = transmission.removeTorrent(hash, True) - elif headphones.CONFIG.TORRENT_DOWNLOADER == 3: # Deluge + elif headphones.CONFIG.TORRENT_DOWNLOADER == 3: # Deluge torrent_removed = deluge.removeTorrent(hash, True) else: torrent_removed = utorrent.removeTorrent(hash, True) diff --git a/headphones/searcher.py b/headphones/searcher.py index 5c8b4144..4dbbf0b2 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -889,7 +889,7 @@ def send_to_downloader(data, bestqual, album): if seed_ratio is not None: transmission.setSeedRatio(torrentid, seed_ratio) - elif headphones.CONFIG.TORRENT_DOWNLOADER == 3: # Deluge + elif headphones.CONFIG.TORRENT_DOWNLOADER == 3: # Deluge logger.info("Sending torrent to Deluge") try: From 479a9548c14108c2cf1bd99d3f1fbc63a49f56a2 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 9 Mar 2016 00:01:05 +0200 Subject: [PATCH 30/57] Set Move-To Path for General Setting - setTorrentPath was called only in case DELUGE_DONE_DIRECTORY was set, forgot about the more general DOWNLOAD_TORRENT_DIR option. Thought about it in deluge.py, forgot about it in searcher.py. This should fix #2557 --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index ee561823..c3e5d72d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -917,7 +917,7 @@ def send_to_downloader(data, bestqual, album): deluge.setSeedRatio({'hash': torrentid, 'ratio': seed_ratio}) # Set move-to directory - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: deluge.setTorrentPath({'hash': torrentid}) # Get folder name from Deluge, it's usually the torrent name From 5693e509b45a880c643f29b594c6e56a769dc835 Mon Sep 17 00:00:00 2001 From: Federico Brigante Date: Tue, 8 Mar 2016 19:54:43 -0800 Subject: [PATCH 31/57] Custom build badges in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87eb2f36..ff1e214b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ##![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones -**Master Branch:** [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=master)](https://travis-ci.org/rembo10/headphones) -**Develop Branch:** [![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=develop)](https://travis-ci.org/rembo10/headphones) +[![Build Status](https://travis-ci.org/rembo10/headphones.svg?branch=master)](https://travis-ci.org/rembo10/headphones) +[![Build Status](https://img.shields.io/travis/rembo10/headphones/develop.svg?label=develop%20branch%20build)](https://travis-ci.org/rembo10/headphones) Headphones is an automated music downloader for NZB and Torrent, written in Python. It supports SABnzbd, NZBget, Transmission, µTorrent, Deluge and Blackhole. From 33baf66371a5dd881cc07d97675edd3cf69ec834 Mon Sep 17 00:00:00 2001 From: Andrzej Ciarkowski Date: Wed, 9 Mar 2016 21:03:37 +0100 Subject: [PATCH 32/57] pathrender: Update explanation text. --- data/interfaces/default/config.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 9ba9f2aa..c107c7b5 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -888,7 +888,7 @@
as .jpg
- Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in square brackets, use single-quote marks to escape square brackets literally ('[', ']'). + Use $Artist/$artist, $Album/$album, $Year/$year, put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').