From ef08a5067314445c3e0bca02d1dafe00528db45c Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 12:37:44 +0200 Subject: [PATCH 01/22] Deluge SSL and More Logging - Added option to use Deluge WebUI SSL using certificate file path - Added logging in order to understand other bugs --- data/interfaces/default/config.html | 6 + headphones/config.py | 1 + headphones/deluge.py | 216 +++++++++++++++++++--------- headphones/webserve.py | 1 + 4 files changed, 153 insertions(+), 71 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 9ba9f2aa..14254619 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -392,6 +392,12 @@ Usually http://localhost:8112 (requires WebUI plugin) +
+ + + Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match + hostname). +
diff --git a/headphones/config.py b/headphones/config.py index dd1ae9b6..a4dc5d27 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -68,6 +68,7 @@ _CONFIG_DEFINITIONS = { 'CUSTOMUSER': (str, 'General', ''), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), + 'DELUGE_CERT': (str, 'Deluge', ''), 'DELUGE_PASSWORD': (str, 'Deluge', ''), 'DELUGE_LABEL': (str, 'Deluge', ''), 'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''), diff --git a/headphones/deluge.py b/headphones/deluge.py index 5f79d29e..8d283ffa 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -48,6 +48,7 @@ import traceback delugeweb_auth = {} delugeweb_url = '' +deluge_verify_cert = False def addTorrent(link, data=None): @@ -158,14 +159,16 @@ def getTorrentFolder(result): ["total_done"] ], "id": 22}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] tries = 0 while result['total_done'] == 0 and tries < 10: tries += 1 time.sleep(5) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] post_data = json.dumps({"method": "web.get_torrent_status", @@ -183,7 +186,8 @@ def getTorrentFolder(result): ], "id": 23}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['save_path'] = json.loads(response.text)['result']['save_path'] result['name'] = json.loads(response.text)['result']['name'] @@ -198,30 +202,47 @@ def removeTorrent(torrentid, remove_data=False): if not any(delugeweb_auth): _get_auth() - result = False - post_data = json.dumps({"method": "core.remove_torrent", - "params": [ - torrentid, - remove_data - ], - "id": 25}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - result = json.loads(response.text)['result'] + try: + result = False + post_data = json.dumps({"method": "core.remove_torrent", + "params": [ + torrentid, + remove_data + ], + "id": 25}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + result = json.loads(response.text)['result'] - return result + return result + except Exception as e: + logger.error('Deluge: Removing torrent failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def _get_auth(): logger.debug('Deluge: Authenticating...') - global delugeweb_auth, delugeweb_url + global delugeweb_auth, delugeweb_url, deluge_verify_cert delugeweb_auth = {} delugeweb_host = headphones.CONFIG.DELUGE_HOST + delugeweb_cert = headphones.CONFIG.DELUGE_CERT delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD + logger.debug('Deluge: Using password %s***%s' % (delugeweb_password[0], delugeweb_password[-1])) if not delugeweb_host.startswith('http'): delugeweb_host = 'http://%s' % delugeweb_host + if delugeweb_cert is None or delugeweb_cert.strip() == '': + deluge_verify_cert = False + logger.debug('Deluge: No SSL certificate configured') + else: + deluge_verify_cert = delugeweb_cert + delugeweb_host = delugeweb_host.replace('http:', 'https:') + logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, delugeweb_host)) + if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] @@ -231,33 +252,47 @@ def _get_auth(): "params": [delugeweb_password], "id": 1}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None auth = json.loads(response.text)["result"] + auth_error = json.loads(response.text)["error"] + logger.debug('Deluge: Authentication result: %s, Error: %s' % (auth, auth_error)) delugeweb_auth = response.cookies + logger.debug('Deluge: Authentication cookies: %s' % str(delugeweb_auth.get_dict())) post_data = json.dumps({"method": "web.connected", "params": [], "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None connected = json.loads(response.text)['result'] + connected_error = json.loads(response.text)['error'] + logger.debug('Deluge: Connection result: %s, Error: %s' % (connected, connected_error)) if not connected: post_data = json.dumps({"method": "web.get_hosts", "params": [], "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None delugeweb_hosts = json.loads(response.text)['result'] @@ -270,9 +305,12 @@ def _get_auth(): "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None post_data = json.dumps({"method": "web.connected", @@ -280,9 +318,12 @@ def _get_auth(): "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - # , verify=TORRENT_VERIFY_CERT) - except Exception: + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) return None connected = json.loads(response.text)['result'] @@ -302,12 +343,16 @@ def _add_torrent_magnet(result): post_data = json.dumps({"method": "core.add_torrent_magnet", "params": [result['url'], {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent magnet failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None ''' def _add_torrent_url(result): @@ -318,7 +363,8 @@ def _add_torrent_url(result): post_data = json.dumps({"method": "web.download_torrent_from_url", "params": [result['url'], {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] @@ -336,7 +382,8 @@ def _add_torrent_file(result): post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content'].encode('utf8')), {}], "id": 2}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] @@ -344,6 +391,7 @@ def _add_torrent_file(result): logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) + return None def setTorrentLabel(result): @@ -361,7 +409,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.get_labels', "params": [], "id": 3}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) labels = json.loads(response.text)['result'] if labels is not None: @@ -371,7 +420,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.add', "params": [label], "id": 4}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) logger.debug('Deluge: %s label added to Deluge' % label) except Exception as e: logger.error('Deluge: Setting label failed: %s' % str(e)) @@ -382,7 +432,8 @@ def setTorrentLabel(result): post_data = json.dumps({"method": 'label.set_torrent', "params": [result['hash'], label], "id": 5}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) logger.debug('Deluge: %s label added to torrent' % label) else: logger.debug('Deluge: Label plugin not detected') @@ -400,19 +451,27 @@ def setSeedRatio(result): if result['ratio']: ratio = result['ratio'] - if ratio: - post_data = json.dumps({"method": "core.set_torrent_stop_at_ratio", - "params": [result['hash'], True], - "id": 5}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) - post_data = json.dumps({"method": "core.set_torrent_stop_ratio", - "params": [result['hash'], float(ratio)], - "id": 6}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if ratio: + post_data = json.dumps({"method": "core.set_torrent_stop_at_ratio", + "params": [result['hash'], True], + "id": 5}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + post_data = json.dumps({"method": "core.set_torrent_stop_ratio", + "params": [result['hash'], float(ratio)], + "id": 6}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting seed ratio failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def setTorrentPath(result): @@ -420,28 +479,36 @@ def setTorrentPath(result): if not any(delugeweb_auth): _get_auth() - if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: - post_data = json.dumps({"method": "core.set_torrent_move_completed", - "params": [result['hash'], True], - "id": 7}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: + post_data = json.dumps({"method": "core.set_torrent_move_completed", + "params": [result['hash'], True], + "id": 7}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: - move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY - else: - move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR + if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY + else: + move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR - if not os.path.exists(move_to): - logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to) - os.makedirs(move_to) - post_data = json.dumps({"method": "core.set_torrent_move_completed_path", - "params": [result['hash'], move_to], - "id": 8}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + if not os.path.exists(move_to): + logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to) + os.makedirs(move_to) + post_data = json.dumps({"method": "core.set_torrent_move_completed_path", + "params": [result['hash'], move_to], + "id": 8}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting torrent move-to directory failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None def setTorrentPause(result): @@ -449,12 +516,19 @@ def setTorrentPause(result): if not any(delugeweb_auth): _get_auth() - if headphones.CONFIG.DELUGE_PAUSED: - post_data = json.dumps({"method": "core.pause_torrent", - "params": [[result['hash']]], - "id": 9}) - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth) + try: + if headphones.CONFIG.DELUGE_PAUSED: + post_data = json.dumps({"method": "core.pause_torrent", + "params": [[result['hash']]], + "id": 9}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) - return not json.loads(response.text)['error'] + return not json.loads(response.text)['error'] - return True + return True + except Exception as e: + logger.error('Deluge: Setting torrent paused failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None diff --git a/headphones/webserve.py b/headphones/webserve.py index 741a37f4..d22f0185 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1157,6 +1157,7 @@ class WebInterface(object): "transmission_username": headphones.CONFIG.TRANSMISSION_USERNAME, "transmission_password": headphones.CONFIG.TRANSMISSION_PASSWORD, "deluge_host": headphones.CONFIG.DELUGE_HOST, + "deluge_cert": headphones.CONFIG.DELUGE_CERT, "deluge_password": headphones.CONFIG.DELUGE_PASSWORD, "deluge_label": headphones.CONFIG.DELUGE_LABEL, "deluge_done_directory": headphones.CONFIG.DELUGE_DONE_DIRECTORY, From e2097fdb31f71dc4e5917f1e7f238e1a457b23f5 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 13:42:34 +0200 Subject: [PATCH 02/22] Expanded SSL Description --- data/interfaces/default/config.html | 2 +- headphones/deluge.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 14254619..a50035d3 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -396,7 +396,7 @@ Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match - hostname). + hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux.
diff --git a/headphones/deluge.py b/headphones/deluge.py index 8d283ffa..d48f0248 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -237,7 +237,7 @@ def _get_auth(): if delugeweb_cert is None or delugeweb_cert.strip() == '': deluge_verify_cert = False - logger.debug('Deluge: No SSL certificate configured') + logger.debug('Deluge: FYI no SSL certificate configured') else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') From 06ae4e97a08c731bf887e9072b3397000cb0d780 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 16:47:17 +0200 Subject: [PATCH 03/22] Handle UnicodeDecodeError - Attempt at handling encoding errors with specific torrent files --- headphones/deluge.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index d48f0248..770c7073 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -198,7 +198,7 @@ def getTorrentFolder(result): def removeTorrent(torrentid, remove_data=False): - + logger.debug('Deluge: Remove torrent %s' % torrentid) if not any(delugeweb_auth): _get_auth() @@ -387,6 +387,22 @@ def _add_torrent_file(result): result['hash'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) return json.loads(response.text)['result'] + except UnicodeDecodeError: + try: + # content is torrent file contents that needs to be encoded to base64 + post_data = json.dumps({"method": "core.add_torrent_file", + "params": [result['name'] + '.torrent', b64encode(result['content'].decode('utf-8').encode('utf8')), {}], + "id": 2}) + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + result['hash'] = json.loads(response.text)['result'] + logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + return json.loads(response.text)['result'] + except Exception as e: + logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None except Exception as e: logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() From 2c9c4d32db09972134d65aa79121f53073006120 Mon Sep 17 00:00:00 2001 From: Noam Date: Sun, 28 Feb 2016 22:48:37 +0200 Subject: [PATCH 04/22] Another Fix For Encoding Issues - Apparently some torrents are already proper UTF8 so we don't need to encode them... --- headphones/deluge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 770c7073..809298f7 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -390,8 +390,9 @@ def _add_torrent_file(result): except UnicodeDecodeError: try: # content is torrent file contents that needs to be encoded to base64 + # this time let's try leaving the encoding alone post_data = json.dumps({"method": "core.add_torrent_file", - "params": [result['name'] + '.torrent', b64encode(result['content'].decode('utf-8').encode('utf8')), {}], + "params": [result['name'] + '.torrent', b64encode(result['content']), {}], "id": 2}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) From c166e36c4ed75d7c4ae72b7ce1e00380ba11e3e4 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 11:54:35 +0200 Subject: [PATCH 05/22] Log Scrubber - Attempt to remove sensitive information from logs --- headphones/deluge.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 809298f7..2df96ea7 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -49,7 +49,20 @@ import traceback delugeweb_auth = {} delugeweb_url = '' deluge_verify_cert = False +scrub_logs = True +def _scrubber(text): + if scrub_logs: + try: + # URL parameter values + text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) + # Local host with port + text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) + #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) + except Exception as e: + logger.debug('Deluge: Scrubber failed: %s' % str(e)) + return text def addTorrent(link, data=None): try: @@ -57,13 +70,13 @@ def addTorrent(link, data=None): retid = False if link.startswith('magnet:'): - logger.debug('Deluge: Got a magnet link: %s' % link) + logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) result = {'type': 'magnet', 'url': link} retid = _add_torrent_magnet(result) elif link.startswith('http://') or link.startswith('https://'): - logger.debug('Deluge: Got a URL: %s' % link) + logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} torrentfile = '' @@ -73,16 +86,13 @@ def addTorrent(link, data=None): if r.status_code == 200: logger.debug('Deluge: 200 OK') torrentfile = r.text - #for chunk in r.iter_content(chunk_size=1024): - # if chunk: # filter out keep-alive new chunks - # torrentfile = torrentfile + chunk else: - logger.debug('Deluge: Trying to GET %s returned status %d' % (link, r.status_code)) + logger.debug('Deluge: Trying to GET %s returned status %d' % (_scrubber(link), r.status_code)) return False except Exception as e: logger.debug('Deluge: Download failed: %s' % str(e)) if 'announce' not in torrentfile[:40]: - logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % link) + logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False # Extract torrent name from .torrent try: @@ -241,7 +251,7 @@ def _get_auth(): else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') - logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, delugeweb_host)) + logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, _scrubber(delugeweb_host))) if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] From b89dcfde65791fc87f3a71e7aef58e631c734576 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 13:45:30 +0200 Subject: [PATCH 06/22] More Scrubbing and Logging and Auto-Switch to HTTPS - If Deluge is set to use HTTPS but there is no certificate configured in HP, use HTTPS without verifying the certificate - connection will still be encrypted but the certificate won't be validated - this will show up as an ERROR in the logs - Scrubbing session cookies --- headphones/deluge.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 2df96ea7..869e0ceb 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -58,6 +58,10 @@ def _scrubber(text): text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + # Session cookie + text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) + # Local Windows path + # TODO #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) except Exception as e: @@ -148,7 +152,7 @@ def addTorrent(link, data=None): logger.info('Deluge: Torrent sent to Deluge successfully (%s)' % retid) return retid else: - logger.info('Deluge returned status %s' % retid) + logger.info('Deluge: Returned status %s' % retid) return False except Exception as e: @@ -251,7 +255,7 @@ def _get_auth(): else: deluge_verify_cert = delugeweb_cert delugeweb_host = delugeweb_host.replace('http:', 'https:') - logger.debug('Deluge: Using certificate %s, host is now %s' % (deluge_verify_cert, _scrubber(delugeweb_host))) + logger.debug('Deluge: Using certificate %s, host is now %s' % (_scrubber(deluge_verify_cert), _scrubber(delugeweb_host))) if delugeweb_host.endswith('/'): delugeweb_host = delugeweb_host[:-1] @@ -264,6 +268,19 @@ def _get_auth(): try: response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) + except ConnectionError: + try: + logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') + response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, + verify=deluge_verify_cert) + # If the previous line didn't fail, change delugeweb_url for the rest of this session + logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!') + delugeweb_url = delugeweb_url.replace('http:', 'https:') + except Exception as e: + logger.error('Deluge: Authentication failed: %s' % str(e)) + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return None except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() @@ -274,8 +291,7 @@ def _get_auth(): auth_error = json.loads(response.text)["error"] logger.debug('Deluge: Authentication result: %s, Error: %s' % (auth, auth_error)) delugeweb_auth = response.cookies - logger.debug('Deluge: Authentication cookies: %s' % str(delugeweb_auth.get_dict())) - + logger.debug('Deluge: Authentication cookies: %s' % _scrubber(str(delugeweb_auth.get_dict()))) post_data = json.dumps({"method": "web.connected", "params": [], "id": 10}) @@ -356,13 +372,13 @@ def _add_torrent_magnet(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent magnet failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False ''' def _add_torrent_url(result): @@ -376,7 +392,7 @@ def _add_torrent_url(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent URL failed: %s' % str(e)) @@ -395,30 +411,30 @@ def _add_torrent_file(result): response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except UnicodeDecodeError: try: # content is torrent file contents that needs to be encoded to base64 - # this time let's try leaving the encoding alone + # this time let's try leaving the encoding as is post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content']), {}], "id": 2}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] - logger.debug('Deluge: Response was %s' % str(json.loads(response.text)['result'])) + logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False except Exception as e: logger.error('Deluge: Adding torrent file failed: %s' % str(e)) formatted_lines = traceback.format_exc().splitlines() logger.error('; '.join(formatted_lines)) - return None + return False def setTorrentLabel(result): From 9e943a9681fff1789d9d8a3d6cd96a9cd9fc28b9 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 29 Feb 2016 16:26:34 +0200 Subject: [PATCH 07/22] Handle ConnectionError Properly - except requests.ConnectionError... --- headphones/deluge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 869e0ceb..72aa76ff 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -268,7 +268,7 @@ def _get_auth(): try: response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) - except ConnectionError: + except requests.ConnectionError: try: logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, From 8ead17e25e35d31849198616b8aa1db0010126c2 Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 10:15:36 +0200 Subject: [PATCH 08/22] Try Different User-Agent for Specific Sites - DLing not working for some users, possibly because of the site - maybe a different UA will make a difference - Make links lowercase before testing them for content --- headphones/deluge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 72aa76ff..79104bc9 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,16 +72,20 @@ def addTorrent(link, data=None): try: result = {} retid = False + special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] - if link.startswith('magnet:'): + if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) result = {'type': 'magnet', 'url': link} retid = _add_torrent_magnet(result) - elif link.startswith('http://') or link.startswith('https://'): + elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) - user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' + if link.lower().startswith(tuple(special_treatment_sites)): + user_agent = 'Headphones' + else: + user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} torrentfile = '' logger.debug('Deluge: Trying to download (GET)') @@ -118,7 +122,7 @@ def addTorrent(link, data=None): retid = _add_torrent_file(result) # elif link.endswith('.torrent') or data: - elif not (link.startswith('http://') or link.startswith('https://')): + elif not (link.lower().startswith('http://') or link.lower().startswith('https://')): if data: logger.debug('Deluge: Getting .torrent data') torrentfile = data From 3ac15d880f89dd6e6f938bfa6bd189e1352e01ae Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 10:40:22 +0200 Subject: [PATCH 09/22] Scrub Local Windows User Path - Scrub Windows user path from logs --- headphones/deluge.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 79104bc9..bde7856f 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -60,8 +60,10 @@ def _scrubber(text): text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) - # Local Windows path - # TODO + # Local Windows user path + if text.lower().startswith('c:\\users\\'): + k = text.split('\\') + text = '\\'.join([k[0], k[1], '.....', k[-1]]) #partial_link = re.sub('(auth.*?)=.*&','\g<1>=SECRETZ&', link) #partial_link = re.sub('(\w)=[0-9a-zA-Z]*&*','\g<1>=REMOVED&', link) except Exception as e: From a8476e89f4a9976839b16517cf66621652db110e Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 12:17:13 +0200 Subject: [PATCH 10/22] Log User-Agent Switch --- headphones/deluge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/headphones/deluge.py b/headphones/deluge.py index bde7856f..e79482e2 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -85,6 +85,7 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) if link.lower().startswith(tuple(special_treatment_sites)): + logger.debug('Deluge: Trying different user-agent for this site') user_agent = 'Headphones' else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' From aff82e47554f6c63247fa29f33341adc9bfa6d8d Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 13:16:33 +0200 Subject: [PATCH 11/22] Always Authenticate When Adding Torrents - Avoid expired session cookies, just authenticate always --- headphones/deluge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/headphones/deluge.py b/headphones/deluge.py index e79482e2..1e4ddf19 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,6 +72,10 @@ def _scrubber(text): def addTorrent(link, data=None): try: + # Authenticate anyway + logger.debug('Deluge: addTorrent Authentication') + _get_auth() + result = {} retid = False special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] From 69405d4f0df093e0cca513c66b4cedaadaf0867d Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 14:54:29 +0200 Subject: [PATCH 12/22] Let Deluge Handle Certain Downloads - Use Deluge's web.download_torrent_from_url for specific sites --- headphones/deluge.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 1e4ddf19..b42cd8dc 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -89,8 +89,13 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) if link.lower().startswith(tuple(special_treatment_sites)): - logger.debug('Deluge: Trying different user-agent for this site') - user_agent = 'Headphones' + #logger.debug('Deluge: Trying different user-agent for this site') + #user_agent = 'Headphones' + # This method will make Deluge download the file + logger.debug('Deluge: Letting Deluge download this') + local_torrent_path = _add_torrent_url({'url': link}) + logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) + return addTorrent(local_torrent_path) else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} @@ -391,7 +396,7 @@ def _add_torrent_magnet(result): logger.error('; '.join(formatted_lines)) return False -''' + def _add_torrent_url(result): logger.debug('Deluge: Adding URL') if not any(delugeweb_auth): @@ -399,15 +404,17 @@ def _add_torrent_url(result): try: post_data = json.dumps({"method": "web.download_torrent_from_url", "params": [result['url'], {}], - "id": 2}) + "id": 32}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) - result['hash'] = json.loads(response.text)['result'] + result['location'] = json.loads(response.text)['result'] logger.debug('Deluge: Response was %s' % str(json.loads(response.text))) return json.loads(response.text)['result'] except Exception as e: logger.error('Deluge: Adding torrent URL failed: %s' % str(e)) -''' + formatted_lines = traceback.format_exc().splitlines() + logger.error('; '.join(formatted_lines)) + return False def _add_torrent_file(result): @@ -430,7 +437,7 @@ def _add_torrent_file(result): # this time let's try leaving the encoding as is post_data = json.dumps({"method": "core.add_torrent_file", "params": [result['name'] + '.torrent', b64encode(result['content']), {}], - "id": 2}) + "id": 22}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['hash'] = json.loads(response.text)['result'] From ac91e17f868fc1e6007f7d2a8568158ebec27443 Mon Sep 17 00:00:00 2001 From: Noam Date: Tue, 1 Mar 2016 23:39:38 +0200 Subject: [PATCH 13/22] More Special Sites --- headphones/deluge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index b42cd8dc..933871e8 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -78,7 +78,7 @@ def addTorrent(link, data=None): result = {} retid = False - special_treatment_sites = ['https://what.cd/', 'http://what.cd/'] + special_treatment_sites = ['https://what.cd/', 'http://what.cd/', 'https://waffles.fm/', 'http://waffles.fm/'] if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) From 73a55e81b8442e393072e702d44b73d40032bd13 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 15:12:18 +0200 Subject: [PATCH 14/22] Better Downloading of Files - Reading downloaded files now uses response.content instead of response.text which caused loss of file data due to encoding shenanigans. This fixes the WHAT issue. - Fix URL for WFFLS downloads - Special User-Agent for WHAT downloads, still not sure if this is necessary --- headphones/deluge.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 933871e8..3c178d68 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -78,7 +78,8 @@ def addTorrent(link, data=None): result = {} retid = False - special_treatment_sites = ['https://what.cd/', 'http://what.cd/', 'https://waffles.fm/', 'http://waffles.fm/'] + url_what = ['https://what.cd/', 'http://what.cd/'] + url_waffles = ['https://waffles.fm/', 'http://waffles.fm/'] if link.lower().startswith('magnet:'): logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link)) @@ -88,14 +89,17 @@ def addTorrent(link, data=None): elif link.lower().startswith('http://') or link.lower().startswith('https://'): logger.debug('Deluge: Got a URL: %s' % _scrubber(link)) - if link.lower().startswith(tuple(special_treatment_sites)): - #logger.debug('Deluge: Trying different user-agent for this site') - #user_agent = 'Headphones' + if link.lower().startswith(tuple(url_waffles)): + if 'rss=' not in link: + link = link + '&rss=1' + if link.lower().startswith(tuple(url_what)): + logger.debug('Deluge: Using different User-Agent for this site') + user_agent = 'Headphones' # This method will make Deluge download the file - logger.debug('Deluge: Letting Deluge download this') - local_torrent_path = _add_torrent_url({'url': link}) - logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) - return addTorrent(local_torrent_path) + #logger.debug('Deluge: Letting Deluge download this') + #local_torrent_path = _add_torrent_url({'url': link}) + #logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path)) + #return addTorrent(local_torrent_path) else: user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' headers = {'User-Agent': user_agent} @@ -105,21 +109,22 @@ def addTorrent(link, data=None): r = requests.get(link, headers=headers) if r.status_code == 200: logger.debug('Deluge: 200 OK') - torrentfile = r.text + # .text will ruin the encoding for some torrents + torrentfile = r.content else: logger.debug('Deluge: Trying to GET %s returned status %d' % (_scrubber(link), r.status_code)) return False except Exception as e: logger.debug('Deluge: Download failed: %s' % str(e)) - if 'announce' not in torrentfile[:40]: + if 'announce' not in str(torrentfile)[:40]: logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False # Extract torrent name from .torrent try: logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0]) + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length] + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] except Exception as e: logger.debug('Deluge: Could not get torrent name, getting file name') # get last part of link/path (name only) @@ -127,7 +132,7 @@ def addTorrent(link, data=None): # remove '.torrent' suffix if name[-len('.torrent'):] == '.torrent': name = name[:-len('.torrent')] - logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40])) + logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, 'content': torrentfile} @@ -145,9 +150,9 @@ def addTorrent(link, data=None): # Extract torrent name from .torrent try: logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', torrentfile)[0]) + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', torrentfile)[0][:name_length] + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] except Exception as e: logger.debug('Deluge: Could not get torrent name, getting file name') # get last part of link/path (name only) @@ -155,7 +160,7 @@ def addTorrent(link, data=None): # remove '.torrent' suffix if name[-len('.torrent'):] == '.torrent': name = name[:-len('.torrent')] - logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, torrentfile[:40])) + logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, 'content': torrentfile} @@ -260,7 +265,7 @@ def _get_auth(): delugeweb_host = headphones.CONFIG.DELUGE_HOST delugeweb_cert = headphones.CONFIG.DELUGE_CERT delugeweb_password = headphones.CONFIG.DELUGE_PASSWORD - logger.debug('Deluge: Using password %s***%s' % (delugeweb_password[0], delugeweb_password[-1])) + logger.debug('Deluge: Using password %s******%s' % (delugeweb_password[0], delugeweb_password[-1])) if not delugeweb_host.startswith('http'): delugeweb_host = 'http://%s' % delugeweb_host From 1a56e24def62963197906465d59ae9379e28dd4c Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 15:35:20 +0200 Subject: [PATCH 15/22] Added SSL/TLS Disclaimer - Currently self-signed certificates can't be verified --- data/interfaces/default/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index a50035d3..b6e41303 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -396,7 +396,7 @@ Path to the certificate file. Make sure to use a valid certificate ("Issued To" field must match - hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux. + hostname) which is not the case with the default certificate. Path is usually %appdata%\deluge\ssl on Windows, ~/.config/deluge/ssl/ on Linux. Leave this blank if you are using a self-signed certificate.
From a7f2329fee2ea63ae062db417c56003ad3188803 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 23:19:30 +0200 Subject: [PATCH 16/22] Travis Fixes --- headphones/deluge.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 3c178d68..33467783 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -51,13 +51,14 @@ delugeweb_url = '' deluge_verify_cert = False scrub_logs = True + def _scrubber(text): if scrub_logs: try: # URL parameter values text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port - text = re.sub('\:\/\/.*\:' , '://REMOVED:' , text) + text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) # Local Windows user path @@ -70,6 +71,7 @@ def _scrubber(text): logger.debug('Deluge: Scrubber failed: %s' % str(e)) return text + def addTorrent(link, data=None): try: # Authenticate anyway @@ -287,12 +289,12 @@ def _get_auth(): "params": [delugeweb_password], "id": 1}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except requests.ConnectionError: try: logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case') - response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) # If the previous line didn't fail, change delugeweb_url for the rest of this session logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!') @@ -317,7 +319,7 @@ def _get_auth(): "params": [], "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -334,7 +336,7 @@ def _get_auth(): "params": [], "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -352,7 +354,7 @@ def _get_auth(): "id": 11}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) @@ -365,7 +367,7 @@ def _get_auth(): "id": 10}) try: - response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, + response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) except Exception as e: logger.error('Deluge: Authentication failed: %s' % str(e)) From 30a3421710da3fa25c12f0d1e6a923e5a47c2e57 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 2 Mar 2016 23:32:13 +0200 Subject: [PATCH 17/22] More Travis - Not sure what this is but I think this should fix it... --- headphones/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 4b36acf7..daf067a5 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -143,7 +143,7 @@ def initialize(config_file): SOFT_CHROOT = SoftChroot(str(CONFIG.SOFT_CHROOT)) if SOFT_CHROOT.isEnabled(): logger.info("Soft-chroot enabled for dir: %s", str(CONFIG.SOFT_CHROOT)) - except exceptions.SoftChrootError as e: + except headphones.exceptions.SoftChrootError as e: logger.error("SoftChroot error: %s", e) raise e From 147f62d87a1c5e4d9fd95f7737c43b3114784d8c Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 3 Mar 2016 14:23:12 +0200 Subject: [PATCH 18/22] Removed Folder Name - Folder name (folder_name) is set to Artist - Album by default - use that instead --- headphones/searcher.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 5c8b4144..a0d7ad8d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -921,12 +921,13 @@ def send_to_downloader(data, bestqual, album): deluge.setTorrentPath({'hash': torrentid}) # I only just realized this function is useless... - folder_name = deluge.getTorrentFolder({'hash': torrentid}) - if folder_name: - logger.info('Torrent folder name: %s' % folder_name) - else: - logger.error('Torrent folder name could not be determined') - return + # Hadn't realized folder_name was already being set to Artist - Album + #folder_name = deluge.getTorrentFolder({'hash': torrentid}) + #if folder_name: + # logger.info('Torrent folder name: %s' % folder_name) + #else: + # logger.error('Torrent folder name could not be determined') + # return except Exception as e: logger.error('Error sending torrent to Deluge: %s' % str(e)) From e05699e7ba73ebf8b3f1f5b9bd1ab7497e4fe4fa Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 3 Mar 2016 14:37:55 +0200 Subject: [PATCH 19/22] Specify Name When Sending to Deluge Module - Searcher should send "Artist - Album" string to Deluge so we don't have to guess the name --- headphones/deluge.py | 56 ++++++++++++++++++++++-------------------- headphones/searcher.py | 4 +-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 33467783..1acdd851 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -72,7 +72,7 @@ def _scrubber(text): return text -def addTorrent(link, data=None): +def addTorrent(link, data=None, name=None): try: # Authenticate anyway logger.debug('Deluge: addTorrent Authentication') @@ -121,19 +121,20 @@ def addTorrent(link, data=None): if 'announce' not in str(torrentfile)[:40]: logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link)) return False - # Extract torrent name from .torrent - try: - logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) - logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] - except Exception as e: - logger.debug('Deluge: Could not get torrent name, getting file name') - # get last part of link/path (name only) - name = link.split('\\')[-1].split('/')[-1] - # remove '.torrent' suffix - if name[-len('.torrent'):] == '.torrent': - name = name[:-len('.torrent')] + if not name: + # Extract torrent name from .torrent + try: + logger.debug('Deluge: Getting torrent name length') + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) + logger.debug('Deluge: Getting torrent name') + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] + except Exception as e: + logger.debug('Deluge: Could not get torrent name, getting file name') + # get last part of link/path (name only) + name = link.split('\\')[-1].split('/')[-1] + # remove '.torrent' suffix + if name[-len('.torrent'):] == '.torrent': + name = name[:-len('.torrent')] logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, @@ -149,19 +150,20 @@ def addTorrent(link, data=None): logger.debug('Deluge: Getting .torrent file') with open(link, 'rb') as f: torrentfile = f.read() - # Extract torrent name from .torrent - try: - logger.debug('Deluge: Getting torrent name length') - name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) - logger.debug('Deluge: Getting torrent name') - name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] - except Exception as e: - logger.debug('Deluge: Could not get torrent name, getting file name') - # get last part of link/path (name only) - name = link.split('\\')[-1].split('/')[-1] - # remove '.torrent' suffix - if name[-len('.torrent'):] == '.torrent': - name = name[:-len('.torrent')] + if not name: + # Extract torrent name from .torrent + try: + logger.debug('Deluge: Getting torrent name length') + name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0]) + logger.debug('Deluge: Getting torrent name') + name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length] + except Exception as e: + logger.debug('Deluge: Could not get torrent name, getting file name') + # get last part of link/path (name only) + name = link.split('\\')[-1].split('/')[-1] + # remove '.torrent' suffix + if name[-len('.torrent'):] == '.torrent': + name = name[:-len('.torrent')] logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40])) result = {'type': 'torrent', 'name': name, diff --git a/headphones/searcher.py b/headphones/searcher.py index a0d7ad8d..d4a2b9c9 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -895,9 +895,9 @@ def send_to_downloader(data, bestqual, album): try: # Add torrent if bestqual[3] == 'rutracker.org': - torrentid = deluge.addTorrent('', data) + torrentid = deluge.addTorrent('', data, name=folder_name) else: - torrentid = deluge.addTorrent(bestqual[2]) + torrentid = deluge.addTorrent(bestqual[2], name=folder_name) if not torrentid: logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?") From 5bf465b88379623e7fedcb3f532ce7822ff20904 Mon Sep 17 00:00:00 2001 From: Noam Date: Fri, 4 Mar 2016 15:29:52 +0200 Subject: [PATCH 20/22] Go Back to Previous Folder Name - Go back to previous folder name method --- headphones/deluge.py | 5 +++-- headphones/searcher.py | 19 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/headphones/deluge.py b/headphones/deluge.py index 1acdd851..e07cc7c1 100644 --- a/headphones/deluge.py +++ b/headphones/deluge.py @@ -58,7 +58,8 @@ def _scrubber(text): # URL parameter values text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text) # Local host with port - text = re.sub('\:\/\/.*\:', '://REMOVED:', text) + # text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # just host + text = re.sub('\:\/\/.*\:[0-9]*', '://REMOVED:', text) # Session cookie text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text) # Local Windows user path @@ -197,7 +198,7 @@ def getTorrentFolder(result): result['hash'], ["total_done"] ], - "id": 22}) + "id": 21}) response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth, verify=deluge_verify_cert) result['total_done'] = json.loads(response.text)['result']['total_done'] diff --git a/headphones/searcher.py b/headphones/searcher.py index d4a2b9c9..ee561823 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -895,9 +895,9 @@ def send_to_downloader(data, bestqual, album): try: # Add torrent if bestqual[3] == 'rutracker.org': - torrentid = deluge.addTorrent('', data, name=folder_name) + torrentid = deluge.addTorrent('', data) else: - torrentid = deluge.addTorrent(bestqual[2], name=folder_name) + torrentid = deluge.addTorrent(bestqual[2]) if not torrentid: logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?") @@ -920,14 +920,13 @@ def send_to_downloader(data, bestqual, album): if headphones.CONFIG.DELUGE_DONE_DIRECTORY: deluge.setTorrentPath({'hash': torrentid}) - # I only just realized this function is useless... - # Hadn't realized folder_name was already being set to Artist - Album - #folder_name = deluge.getTorrentFolder({'hash': torrentid}) - #if folder_name: - # logger.info('Torrent folder name: %s' % folder_name) - #else: - # logger.error('Torrent folder name could not be determined') - # return + # Get folder name from Deluge, it's usually the torrent name + folder_name = deluge.getTorrentFolder({'hash': torrentid}) + if folder_name: + logger.info('Torrent folder name: %s' % folder_name) + else: + logger.error('Torrent folder name could not be determined') + return except Exception as e: logger.error('Error sending torrent to Deluge: %s' % str(e)) From 9b9d728ce380b88f636a5181ccedfda3c94f6eab Mon Sep 17 00:00:00 2001 From: Noam Date: Fri, 4 Mar 2016 17:44:44 +0200 Subject: [PATCH 21/22] Deluge Move-To Directory - Deluge move-to directory only relevant if Deluge is the chosen torrent downloader --- headphones/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 507a5571..ef87624f 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -46,7 +46,7 @@ def checkFolder(): if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR else: - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY else: download_dir = headphones.CONFIG.DOWNLOAD_TORRENT_DIR From 479a9548c14108c2cf1bd99d3f1fbc63a49f56a2 Mon Sep 17 00:00:00 2001 From: Noam Date: Wed, 9 Mar 2016 00:01:05 +0200 Subject: [PATCH 22/22] Set Move-To Path for General Setting - setTorrentPath was called only in case DELUGE_DONE_DIRECTORY was set, forgot about the more general DOWNLOAD_TORRENT_DIR option. Thought about it in deluge.py, forgot about it in searcher.py. This should fix #2557 --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index ee561823..c3e5d72d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -917,7 +917,7 @@ def send_to_downloader(data, bestqual, album): deluge.setSeedRatio({'hash': torrentid, 'ratio': seed_ratio}) # Set move-to directory - if headphones.CONFIG.DELUGE_DONE_DIRECTORY: + if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR: deluge.setTorrentPath({'hash': torrentid}) # Get folder name from Deluge, it's usually the torrent name