From 8f70eb9b03fc82e9ad7953b9b8c9c6b3b1eb1f48 Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 1 Aug 2011 07:35:40 -0500 Subject: [PATCH] Added support for Newzbin. This required a bunch of new functions and etc as Newzbin doesn't allow for GET authentication with a hash. They want you to HTTP auth. I started creating some classes that I copied from SickBeard to extract away things in the future. Added USER_AGENT --- Headphones.py | 176 +++++----- headphones/__init__.py | 658 +++++++++++++++++++------------------ headphones/searcher.py | 702 ++++++++++++++++++++++++---------------- headphones/templates.py | 26 ++ headphones/webserve.py | 8 +- 5 files changed, 879 insertions(+), 691 deletions(-) mode change 100755 => 100644 Headphones.py diff --git a/Headphones.py b/Headphones.py old mode 100755 new mode 100644 index f8053ff4..ccef9ec1 --- a/Headphones.py +++ b/Headphones.py @@ -8,98 +8,98 @@ import headphones from headphones import webstart, logger try: - import argparse + import argparse except ImportError: - import lib.argparse as argparse - + import lib.argparse as argparse + def main(): - # Fixed paths to Headphones - if hasattr(sys, 'frozen'): - headphones.FULL_PATH = os.path.abspath(sys.executable) - else: - headphones.FULL_PATH = os.path.abspath(__file__) - - headphones.PROG_DIR = os.path.dirname(headphones.FULL_PATH) - headphones.ARGS = sys.argv[1:] - - # Set up and gather command line arguments - parser = argparse.ArgumentParser(description='Music add-on for SABnzbd+') + # Fixed paths to Headphones + if hasattr(sys, 'frozen'): + headphones.FULL_PATH = os.path.abspath(sys.executable) + else: + headphones.FULL_PATH = os.path.abspath(__file__) + + headphones.PROG_DIR = os.path.dirname(headphones.FULL_PATH) + headphones.ARGS = sys.argv[1:] + + # Set up and gather command line arguments + parser = argparse.ArgumentParser(description='Music add-on for SABnzbd+') - parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging') - parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon') - parser.add_argument('-p', '--port', type=int, help='Force Headphones to run on a specified port') - parser.add_argument('--datadir', help='Specify a directory where to store your data files') - parser.add_argument('--config', help='Specify a config file to use') - parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup') - - args = parser.parse_args() - - if args.quiet: - headphones.QUIET=True - - if args.daemon: - headphones.DAEMON=True - headphones.QUIET=True - - if args.datadir: - headphones.DATA_DIR = args.datadir - else: - headphones.DATA_DIR = headphones.PROG_DIR - - if args.config: - headphones.CONFIG_FILE = args.config - else: - headphones.CONFIG_FILE = os.path.join(headphones.DATA_DIR, 'config.ini') - - # Try to create the DATA_DIR if it doesn't exist - if not os.path.exists(headphones.DATA_DIR): - try: - os.makedirs(headphones.DATA_DIR) - except OSError: - raise SystemExit('Could not create data directory: ' + headphones.DATA_DIR + '. Exiting....') - - # Make sure the DATA_DIR is writeable - if not os.access(headphones.DATA_DIR, os.W_OK): - raise SystemExit('Cannot write to the data directory: ' + headphones.DATA_DIR + '. Exiting...') - - # Put the database in the DATA_DIR - headphones.DB_FILE = os.path.join(headphones.DATA_DIR, 'headphones.db') - - headphones.CFG = ConfigObj(headphones.CONFIG_FILE) - - # Read config & start logging - headphones.initialize() - - if headphones.DAEMON: - headphones.daemonize() - - # Force the http port if neccessary - if args.port: - http_port = args.port - logger.info('Starting Headphones on foced port: %i' % http_port) - else: - http_port = int(headphones.HTTP_PORT) - - # Try to start the server. - webstart.initialize({ - 'http_port': http_port, - 'http_host': headphones.HTTP_HOST, - 'http_root': headphones.HTTP_ROOT, - 'http_username': headphones.HTTP_USERNAME, - 'http_password': headphones.HTTP_PASSWORD, - }) - - logger.info('Starting Headphones on port: %i' % http_port) - - if headphones.LAUNCH_BROWSER and not args.nolaunch: - headphones.launch_browser(headphones.HTTP_HOST, http_port, headphones.HTTP_ROOT) - - # Start the background threads - headphones.start() - - return + parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging') + parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon') + parser.add_argument('-p', '--port', type=int, help='Force Headphones to run on a specified port') + parser.add_argument('--datadir', help='Specify a directory where to store your data files') + parser.add_argument('--config', help='Specify a config file to use') + parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup') + + args = parser.parse_args() + + if args.quiet: + headphones.QUIET=True + + if args.daemon: + headphones.DAEMON=True + headphones.QUIET=True + + if args.datadir: + headphones.DATA_DIR = args.datadir + else: + headphones.DATA_DIR = headphones.PROG_DIR + + if args.config: + headphones.CONFIG_FILE = args.config + else: + headphones.CONFIG_FILE = os.path.join(headphones.DATA_DIR, 'config.ini') + + # Try to create the DATA_DIR if it doesn't exist + if not os.path.exists(headphones.DATA_DIR): + try: + os.makedirs(headphones.DATA_DIR) + except OSError: + raise SystemExit('Could not create data directory: ' + headphones.DATA_DIR + '. Exiting....') + + # Make sure the DATA_DIR is writeable + if not os.access(headphones.DATA_DIR, os.W_OK): + raise SystemExit('Cannot write to the data directory: ' + headphones.DATA_DIR + '. Exiting...') + + # Put the database in the DATA_DIR + headphones.DB_FILE = os.path.join(headphones.DATA_DIR, 'headphones.db') + + headphones.CFG = ConfigObj(headphones.CONFIG_FILE) + + # Read config & start logging + headphones.initialize() + + if headphones.DAEMON: + headphones.daemonize() + + # Force the http port if neccessary + if args.port: + http_port = args.port + logger.info('Starting Headphones on foced port: %i' % http_port) + else: + http_port = int(headphones.HTTP_PORT) + + # Try to start the server. + webstart.initialize({ + 'http_port': http_port, + 'http_host': headphones.HTTP_HOST, + 'http_root': headphones.HTTP_ROOT, + 'http_username': headphones.HTTP_USERNAME, + 'http_password': headphones.HTTP_PASSWORD, + }) + + logger.info('Starting Headphones on port: %i' % http_port) + + if headphones.LAUNCH_BROWSER and not args.nolaunch: + headphones.launch_browser(headphones.HTTP_HOST, http_port, headphones.HTTP_ROOT) + + # Start the background threads + headphones.start() + + return if __name__ == "__main__": - main() + main() diff --git a/headphones/__init__.py b/headphones/__init__.py index eb8a5dd0..84023254 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -11,7 +11,8 @@ from lib.configobj import ConfigObj import cherrypy -from headphones import updater, searcher, importer, versioncheck, logger, postprocessor +from headphones import updater, searcher, importer, versioncheck, logger, postprocessor, version, sab +from headphones.common import * FULL_PATH = None PROG_DIR = None @@ -92,8 +93,13 @@ NZBSORG = False NZBSORG_UID = None NZBSORG_HASH = None +NEWZBIN = False +NEWZBIN_UID = None +NEWZBIN_PASSWORD = None + LASTFM_USERNAME = None + def CheckSection(sec): """ Check if INI section exists, if not create it """ try: @@ -138,351 +144,361 @@ def check_setting_str(config, cfg_name, item_name, def_val, log=True): else: logger.debug(item_name + " -> ******") return my_val - + def initialize(): - with INIT_LOCK: - - global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \ - HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ - CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \ - CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ - ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ - LIBRARYSCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ - NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, LASTFM_USERNAME - - if __INITIALIZED__: - return False - - # Make sure all the config sections exist - CheckSection('General') - CheckSection('SABnzbd') - CheckSection('NZBMatrix') - CheckSection('Newznab') - CheckSection('NZBsorg') - - # Set global variables based on config file or use defaults - try: - HTTP_PORT = check_setting_int(CFG, 'General', 'http_port', 8181) - except: - HTTP_PORT = 8181 - - if HTTP_PORT < 21 or HTTP_PORT > 65535: - HTTP_PORT = 8181 - - HTTP_HOST = check_setting_str(CFG, 'General', 'http_host', '0.0.0.0') - HTTP_USERNAME = check_setting_str(CFG, 'General', 'http_username', '') - HTTP_PASSWORD = check_setting_str(CFG, 'General', 'http_password', '') - HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/') - LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1)) - GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') - LOG_DIR = check_setting_str(CFG, 'General', 'log_dir', '') - - MUSIC_DIR = check_setting_str(CFG, 'General', 'music_dir', '') - DESTINATION_DIR = check_setting_str(CFG, 'General', 'destination_dir', '') - PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) - PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '') - DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) - CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) - MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) - RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) - FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', 'artist/album [year]') - FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', 'tracknumber artist - album [year]- title') - CLEANUP_FILES = bool(check_setting_int(CFG, 'General', 'cleanup_files', 0)) - ADD_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'add_album_art', 0)) - EMBED_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'embed_album_art', 0)) - DOWNLOAD_DIR = check_setting_str(CFG, 'General', 'download_dir', '') - BLACKHOLE = bool(check_setting_int(CFG, 'General', 'blackhole', 0)) - BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '') - USENET_RETENTION = check_setting_int(CFG, 'General', 'usenet_retention', '') - INCLUDE_EXTRAS = bool(check_setting_int(CFG, 'General', 'include_extras', 0)) - - NZB_SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'nzb_search_interval', 360) - LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 180) - - SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '') - SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '') - SAB_PASSWORD = check_setting_str(CFG, 'SABnzbd', 'sab_password', '') - SAB_APIKEY = check_setting_str(CFG, 'SABnzbd', 'sab_apikey', '') - SAB_CATEGORY = check_setting_str(CFG, 'SABnzbd', 'sab_category', '') - - NZBMATRIX = bool(check_setting_int(CFG, 'NZBMatrix', 'nzbmatrix', 0)) - NZBMATRIX_USERNAME = check_setting_str(CFG, 'NZBMatrix', 'nzbmatrix_username', '') - NZBMATRIX_APIKEY = check_setting_str(CFG, 'NZBMatrix', 'nzbmatrix_apikey', '') - - NEWZNAB = bool(check_setting_int(CFG, 'Newznab', 'newznab', 0)) - NEWZNAB_HOST = check_setting_str(CFG, 'Newznab', 'newznab_host', '') - NEWZNAB_APIKEY = check_setting_str(CFG, 'Newznab', 'newznab_apikey', '') - - NZBSORG = bool(check_setting_int(CFG, 'NZBsorg', 'nzbsorg', 0)) - NZBSORG_UID = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_uid', '') - NZBSORG_HASH = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_hash', '') - - LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '') - - if not LOG_DIR: - LOG_DIR = os.path.join(DATA_DIR, 'logs') - - if not os.path.exists(LOG_DIR): - try: - os.makedirs(LOG_DIR) - except OSError: - if not QUIET: - print 'Unable to create the log directory. Logging to screen only.' - - # Start the logger, silence console logging if we need to - logger.headphones_log.initLogger(quiet=QUIET) - - # Update some old config code: - if FOLDER_FORMAT == '%artist/%album/%track': - FOLDER_FORMAT = 'artist/album [year]' - if FILE_FORMAT == '%tracknumber %artist - %album - %title': - FILE_FORMAT = 'tracknumber artist - album - title' - - # Put the cache dir in the data dir for now - CACHE_DIR = os.path.join(DATA_DIR, 'cache') - if not os.path.exists(CACHE_DIR): - try: - os.makedirs(CACHE_DIR) - except OSError: - logger.error('Could not create cache dir. Check permissions of datadir: ' + DATA_DIR) - - # Initialize the database - logger.info('Checking to see if the database has all tables....') - try: - dbcheck() - except Exception, e: - logger.error("Can't connect to the database: %s" % e) - - # Get the currently installed version - returns None, 'win32' or the git hash - # Also sets INSTALL_TYPE variable to 'win', 'git' or 'source' - CURRENT_VERSION = versioncheck.getVersion() - - # Check for new versions - LATEST_VERSION = versioncheck.checkGithub() + with INIT_LOCK: + + global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \ + HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ + CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \ + CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ + ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ + LIBRARYSCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ + NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ + NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME + + if __INITIALIZED__: + return False + + # Make sure all the config sections exist + CheckSection('General') + CheckSection('SABnzbd') + CheckSection('NZBMatrix') + CheckSection('Newznab') + CheckSection('NZBsorg') + CheckSection('Newzbin') + + # Set global variables based on config file or use defaults + try: + HTTP_PORT = check_setting_int(CFG, 'General', 'http_port', 8181) + except: + HTTP_PORT = 8181 + + if HTTP_PORT < 21 or HTTP_PORT > 65535: + HTTP_PORT = 8181 + + HTTP_HOST = check_setting_str(CFG, 'General', 'http_host', '0.0.0.0') + HTTP_USERNAME = check_setting_str(CFG, 'General', 'http_username', '') + HTTP_PASSWORD = check_setting_str(CFG, 'General', 'http_password', '') + HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/') + LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1)) + GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') + LOG_DIR = check_setting_str(CFG, 'General', 'log_dir', '') + + MUSIC_DIR = check_setting_str(CFG, 'General', 'music_dir', '') + DESTINATION_DIR = check_setting_str(CFG, 'General', 'destination_dir', '') + PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) + PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '') + DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) + CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) + MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) + RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) + FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', 'artist/album [year]') + FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', 'tracknumber artist - album [year]- title') + CLEANUP_FILES = bool(check_setting_int(CFG, 'General', 'cleanup_files', 0)) + ADD_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'add_album_art', 0)) + EMBED_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'embed_album_art', 0)) + DOWNLOAD_DIR = check_setting_str(CFG, 'General', 'download_dir', '') + BLACKHOLE = bool(check_setting_int(CFG, 'General', 'blackhole', 0)) + BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '') + USENET_RETENTION = check_setting_int(CFG, 'General', 'usenet_retention', '') + INCLUDE_EXTRAS = bool(check_setting_int(CFG, 'General', 'include_extras', 0)) + + NZB_SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'nzb_search_interval', 360) + LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 180) + + SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '') + SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '') + SAB_PASSWORD = check_setting_str(CFG, 'SABnzbd', 'sab_password', '') + SAB_APIKEY = check_setting_str(CFG, 'SABnzbd', 'sab_apikey', '') + SAB_CATEGORY = check_setting_str(CFG, 'SABnzbd', 'sab_category', '') + + NZBMATRIX = bool(check_setting_int(CFG, 'NZBMatrix', 'nzbmatrix', 0)) + NZBMATRIX_USERNAME = check_setting_str(CFG, 'NZBMatrix', 'nzbmatrix_username', '') + NZBMATRIX_APIKEY = check_setting_str(CFG, 'NZBMatrix', 'nzbmatrix_apikey', '') + + NEWZNAB = bool(check_setting_int(CFG, 'Newznab', 'newznab', 0)) + NEWZNAB_HOST = check_setting_str(CFG, 'Newznab', 'newznab_host', '') + NEWZNAB_APIKEY = check_setting_str(CFG, 'Newznab', 'newznab_apikey', '') + + NZBSORG = bool(check_setting_int(CFG, 'NZBsorg', 'nzbsorg', 0)) + NZBSORG_UID = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_uid', '') + NZBSORG_HASH = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_hash', '') - __INITIALIZED__ = True - return True - + NEWZBIN = bool(check_setting_int(CFG, 'Newzbin', 'newzbin', 0)) + NEWZBIN_UID = check_setting_str(CFG, 'Newzbin', 'newzbin_uid', '') + NEWZBIN_PASSWORD = check_setting_str(CFG, 'Newzbin', 'newzbin_password', '') + + LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '') + + if not LOG_DIR: + LOG_DIR = os.path.join(DATA_DIR, 'logs') + + if not os.path.exists(LOG_DIR): + try: + os.makedirs(LOG_DIR) + except OSError: + if not QUIET: + print 'Unable to create the log directory. Logging to screen only.' + + # Start the logger, silence console logging if we need to + logger.headphones_log.initLogger(quiet=QUIET) + + # Update some old config code: + if FOLDER_FORMAT == '%artist/%album/%track': + FOLDER_FORMAT = 'artist/album [year]' + if FILE_FORMAT == '%tracknumber %artist - %album - %title': + FILE_FORMAT = 'tracknumber artist - album - title' + + # Put the cache dir in the data dir for now + CACHE_DIR = os.path.join(DATA_DIR, 'cache') + if not os.path.exists(CACHE_DIR): + try: + os.makedirs(CACHE_DIR) + except OSError: + logger.error('Could not create cache dir. Check permissions of datadir: ' + DATA_DIR) + + # Initialize the database + logger.info('Checking to see if the database has all tables....') + try: + dbcheck() + except Exception, e: + logger.error("Can't connect to the database: %s" % e) + + # Get the currently installed version - returns None, 'win32' or the git hash + # Also sets INSTALL_TYPE variable to 'win', 'git' or 'source' + CURRENT_VERSION = versioncheck.getVersion() + + # Check for new versions + LATEST_VERSION = versioncheck.checkGithub() + + __INITIALIZED__ = True + return True + def daemonize(): - if threading.activeCount() != 1: - logger.warn('There are %r active threads. Daemonizing may cause \ - strange behavior.' % threading.enumerate()) - - sys.stdout.flush() - sys.stderr.flush() - - # Do first fork - try: - pid = os.fork() - if pid == 0: - pass - else: - # Exit the parent process - logger.debug('Forking once...') - os._exit(0) - except OSError, e: - sys.exit("1st fork failed: %s [%d]" % (e.strerror, e.errno)) - - os.setsid() + if threading.activeCount() != 1: + logger.warn('There are %r active threads. Daemonizing may cause \ + strange behavior.' % threading.enumerate()) + + sys.stdout.flush() + sys.stderr.flush() + + # Do first fork + try: + pid = os.fork() + if pid == 0: + pass + else: + # Exit the parent process + logger.debug('Forking once...') + os._exit(0) + except OSError, e: + sys.exit("1st fork failed: %s [%d]" % (e.strerror, e.errno)) + + os.setsid() - # Do second fork - try: - pid = os.fork() - if pid > 0: - logger.debug('Forking twice...') - os._exit(0) # Exit second parent process - except OSError, e: - sys.exit("2nd fork failed: %s [%d]" % (e.strerror, e.errno)) + # Do second fork + try: + pid = os.fork() + if pid > 0: + logger.debug('Forking twice...') + os._exit(0) # Exit second parent process + except OSError, e: + sys.exit("2nd fork failed: %s [%d]" % (e.strerror, e.errno)) - os.chdir("/") - os.umask(0) - - si = open('/dev/null', "r") - so = open('/dev/null', "a+") - se = open('/dev/null', "a+") - - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) - - logger.info('Daemonized to PID: %s' % os.getpid()) - + os.chdir("/") + os.umask(0) + + si = open('/dev/null', "r") + so = open('/dev/null', "a+") + se = open('/dev/null', "a+") + + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + logger.info('Daemonized to PID: %s' % os.getpid()) + def launch_browser(host, port, root): - if host == '0.0.0.0': - host = 'localhost' - - try: - webbrowser.open('http://%s:%i%s' % (host, port, root)) - except Exception, e: - logger.error('Could not launch browser: %s' % e) + if host == '0.0.0.0': + host = 'localhost' + + try: + webbrowser.open('http://%s:%i%s' % (host, port, root)) + except Exception, e: + logger.error('Could not launch browser: %s' % e) def config_write(): - new_config = ConfigObj() - new_config.filename = CONFIG_FILE + new_config = ConfigObj() + new_config.filename = CONFIG_FILE - new_config['General'] = {} - new_config['General']['http_port'] = HTTP_PORT - new_config['General']['http_host'] = HTTP_HOST - new_config['General']['http_username'] = HTTP_USERNAME - new_config['General']['http_password'] = HTTP_PASSWORD - new_config['General']['http_root'] = HTTP_ROOT - new_config['General']['launch_browser'] = int(LAUNCH_BROWSER) - new_config['General']['log_dir'] = LOG_DIR - new_config['General']['git_path'] = GIT_PATH + new_config['General'] = {} + new_config['General']['http_port'] = HTTP_PORT + new_config['General']['http_host'] = HTTP_HOST + new_config['General']['http_username'] = HTTP_USERNAME + new_config['General']['http_password'] = HTTP_PASSWORD + new_config['General']['http_root'] = HTTP_ROOT + new_config['General']['launch_browser'] = int(LAUNCH_BROWSER) + new_config['General']['log_dir'] = LOG_DIR + new_config['General']['git_path'] = GIT_PATH - new_config['General']['music_dir'] = MUSIC_DIR - new_config['General']['destination_dir'] = DESTINATION_DIR - new_config['General']['preferred_quality'] = PREFERRED_QUALITY - new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE - new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) - new_config['General']['correct_metadata'] = int(CORRECT_METADATA) - new_config['General']['move_files'] = int(MOVE_FILES) - new_config['General']['rename_files'] = int(RENAME_FILES) - new_config['General']['folder_format'] = FOLDER_FORMAT - new_config['General']['file_format'] = FILE_FORMAT - new_config['General']['cleanup_files'] = int(CLEANUP_FILES) - new_config['General']['add_album_art'] = int(ADD_ALBUM_ART) - new_config['General']['embed_album_art'] = int(EMBED_ALBUM_ART) - new_config['General']['download_dir'] = DOWNLOAD_DIR - new_config['General']['blackhole'] = int(BLACKHOLE) - new_config['General']['blackhole_dir'] = BLACKHOLE_DIR - new_config['General']['usenet_retention'] = USENET_RETENTION - new_config['General']['include_extras'] = int(INCLUDE_EXTRAS) - - new_config['General']['nzb_search_interval'] = NZB_SEARCH_INTERVAL - new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL + new_config['General']['music_dir'] = MUSIC_DIR + new_config['General']['destination_dir'] = DESTINATION_DIR + new_config['General']['preferred_quality'] = PREFERRED_QUALITY + new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE + new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) + new_config['General']['correct_metadata'] = int(CORRECT_METADATA) + new_config['General']['move_files'] = int(MOVE_FILES) + new_config['General']['rename_files'] = int(RENAME_FILES) + new_config['General']['folder_format'] = FOLDER_FORMAT + new_config['General']['file_format'] = FILE_FORMAT + new_config['General']['cleanup_files'] = int(CLEANUP_FILES) + new_config['General']['add_album_art'] = int(ADD_ALBUM_ART) + new_config['General']['embed_album_art'] = int(EMBED_ALBUM_ART) + new_config['General']['download_dir'] = DOWNLOAD_DIR + new_config['General']['blackhole'] = int(BLACKHOLE) + new_config['General']['blackhole_dir'] = BLACKHOLE_DIR + new_config['General']['usenet_retention'] = USENET_RETENTION + new_config['General']['include_extras'] = int(INCLUDE_EXTRAS) + + new_config['General']['nzb_search_interval'] = NZB_SEARCH_INTERVAL + new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL - new_config['SABnzbd'] = {} - new_config['SABnzbd']['sab_host'] = SAB_HOST - new_config['SABnzbd']['sab_username'] = SAB_USERNAME - new_config['SABnzbd']['sab_password'] = SAB_PASSWORD - new_config['SABnzbd']['sab_apikey'] = SAB_APIKEY - new_config['SABnzbd']['sab_category'] = SAB_CATEGORY + new_config['SABnzbd'] = {} + new_config['SABnzbd']['sab_host'] = SAB_HOST + new_config['SABnzbd']['sab_username'] = SAB_USERNAME + new_config['SABnzbd']['sab_password'] = SAB_PASSWORD + new_config['SABnzbd']['sab_apikey'] = SAB_APIKEY + new_config['SABnzbd']['sab_category'] = SAB_CATEGORY - new_config['NZBMatrix'] = {} - new_config['NZBMatrix']['nzbmatrix'] = int(NZBMATRIX) - new_config['NZBMatrix']['nzbmatrix_username'] = NZBMATRIX_USERNAME - new_config['NZBMatrix']['nzbmatrix_apikey'] = NZBMATRIX_APIKEY + new_config['NZBMatrix'] = {} + new_config['NZBMatrix']['nzbmatrix'] = int(NZBMATRIX) + new_config['NZBMatrix']['nzbmatrix_username'] = NZBMATRIX_USERNAME + new_config['NZBMatrix']['nzbmatrix_apikey'] = NZBMATRIX_APIKEY - new_config['Newznab'] = {} - new_config['Newznab']['newznab'] = int(NEWZNAB) - new_config['Newznab']['newznab_host'] = NEWZNAB_HOST - new_config['Newznab']['newznab_apikey'] = NEWZNAB_APIKEY + new_config['Newznab'] = {} + new_config['Newznab']['newznab'] = int(NEWZNAB) + new_config['Newznab']['newznab_host'] = NEWZNAB_HOST + new_config['Newznab']['newznab_apikey'] = NEWZNAB_APIKEY - new_config['NZBsorg'] = {} - new_config['NZBsorg']['nzbsorg'] = int(NZBSORG) - new_config['NZBsorg']['nzbsorg_uid'] = NZBSORG_UID - new_config['NZBsorg']['nzbsorg_hash'] = NZBSORG_HASH - - new_config['General']['lastfm_username'] = LASTFM_USERNAME - - new_config.write() + new_config['NZBsorg'] = {} + new_config['NZBsorg']['nzbsorg'] = int(NZBSORG) + new_config['NZBsorg']['nzbsorg_uid'] = NZBSORG_UID + new_config['NZBsorg']['nzbsorg_hash'] = NZBSORG_HASH + + new_config['Newzbin'] = {} + new_config['Newzbin']['newzbin'] = int(NEWZBIN) + new_config['Newzbin']['newzbin_uid'] = NEWZBIN_UID + new_config['Newzbin']['newzbin_password'] = NEWZBIN_PASSWORD + + new_config['General']['lastfm_username'] = LASTFM_USERNAME + + new_config.write() - + def start(): - - global __INITIALIZED__, started - - if __INITIALIZED__: - - # Start our scheduled background tasks + + global __INITIALIZED__, started + + if __INITIALIZED__: + + # Start our scheduled background tasks - SCHED.add_cron_job(updater.dbUpdate, hour=4, minute=0, second=0) - SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) - SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL) - SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) - SCHED.add_interval_job(postprocessor.checkFolder, minutes=5) + SCHED.add_cron_job(updater.dbUpdate, hour=4, minute=0, second=0) + SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) + SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL) + SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) + SCHED.add_interval_job(postprocessor.checkFolder, minutes=5) - SCHED.start() - - started = True - + SCHED.start() + + started = True + def dbcheck(): - conn=sqlite3.connect(DB_FILE) - c=conn.cursor() - c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') - c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER)') - c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') - - try: - c.execute('SELECT IncludeExtras from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') - - try: - c.execute('SELECT LatestAlbum from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') - - try: - c.execute('SELECT ReleaseDate from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') - - try: - c.execute('SELECT AlbumID from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') - - try: - c.execute('SELECT HaveTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT TotalTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT Type from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') + conn=sqlite3.connect(DB_FILE) + c=conn.cursor() + c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') + + try: + c.execute('SELECT IncludeExtras from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') + + try: + c.execute('SELECT LatestAlbum from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') + + try: + c.execute('SELECT ReleaseDate from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') + + try: + c.execute('SELECT AlbumID from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') + + try: + c.execute('SELECT HaveTracks from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT TotalTracks from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT Type from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') - try: - c.execute('SELECT TrackNumber from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') - - try: - c.execute('SELECT FolderName from snatched') - except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') - - conn.commit() - c.close() + try: + c.execute('SELECT TrackNumber from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') + + try: + c.execute('SELECT FolderName from snatched') + except sqlite3.OperationalError: + c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + + conn.commit() + c.close() - + def shutdown(restart=False, update=False): - - cherrypy.engine.exit() - SCHED.shutdown(wait=False) - - config_write() - - if update: - try: - versioncheck.update() - except Exception, e: - logger.warn('Headphones failed to update: %s. Restarting.' % e) - - if restart: - - popen_list = [sys.executable, FULL_PATH] - popen_list += ARGS - if '--nolaunch' not in popen_list: - popen_list += ['--nolaunch'] - logger.info('Restarting Headphones with ' + str(popen_list)) - subprocess.Popen(popen_list, cwd=os.getcwd()) - - os._exit(0) \ No newline at end of file + + cherrypy.engine.exit() + SCHED.shutdown(wait=False) + + config_write() + + if update: + try: + versioncheck.update() + except Exception, e: + logger.warn('Headphones failed to update: %s. Restarting.' % e) + + if restart: + + popen_list = [sys.executable, FULL_PATH] + popen_list += ARGS + if '--nolaunch' not in popen_list: + popen_list += ['--nolaunch'] + logger.info('Restarting Headphones with ' + str(popen_list)) + subprocess.Popen(popen_list, cwd=os.getcwd()) + + os._exit(0) \ No newline at end of file diff --git a/headphones/searcher.py b/headphones/searcher.py index 970e0ec2..05a5159f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -1,299 +1,439 @@ import urllib import lib.feedparser as feedparser from xml.dom import minidom -import os, re +from xml.parsers.expat import ExpatError +import os, re, time import headphones -from headphones import logger, db, helpers +from headphones import logger, db, helpers, classes, sab + +class NewzbinDownloader(urllib.FancyURLopener): + + def __init__(self): + urllib.FancyURLopener.__init__(self) + + def http_error_default(self, url, fp, errcode, errmsg, headers): + + # if newzbin is throttling us, wait seconds and try again + if errcode == 400: + + newzbinErrCode = int(headers.getheader('X-DNZB-RCode')) + + if newzbinErrCode == 450: + rtext = str(headers.getheader('X-DNZB-RText')) + result = re.search("wait (\d+) seconds", rtext) + + elif newzbinErrCode == 401: + logger.info("Newzbin error 401") + #raise exceptions.AuthException("Newzbin username or password incorrect") + + elif newzbinErrCode == 402: + #raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs") + logger.info("Newzbin error 402") + + logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + "seconds") + + time.sleep(int(result.group(1))) + + #raise exceptions.NewzbinAPIThrottled() + +#this should be in a class somewhere +def getNewzbinURL(url): + + myOpener = classes.AuthURLOpener(headphones.NEWZBIN_UID, headphones.NEWZBIN_PASSWORD) + try: + f = myOpener.openit(url) + except (urllib.ContentTooShortError, IOError), e: + logger.info("Error loading search results: ContentTooShortError ") + return None + + data = f.read() + f.close() + + return data def searchNZB(albumid=None, new=False): - myDB = db.DBConnection() - - if albumid: - results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" AND AlbumID=?', [albumid]) - else: - results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted"') - new = True - - for albums in results: - - albumid = albums[2] - reldate = albums[3] - - try: - year = reldate[:4] - except TypeError: - year = '' - - dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':''} + myDB = db.DBConnection() + + if albumid: + results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" AND AlbumID=?', [albumid]) + else: + results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted"') + new = True + + for albums in results: + + albumid = albums[2] + reldate = albums[3] + + try: + year = reldate[:4] + except TypeError: + year = '' + + dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':''} - cleanartistalbum = helpers.latinToAscii(helpers.replace_all(albums[0]+' '+albums[1], dic)) + cleanartistalbum = helpers.latinToAscii(helpers.replace_all(albums[0]+' '+albums[1], dic)) - # FLAC usually doesn't have a year for some reason so I'll leave it out: - term = re.sub('[\.\-]', ' ', '%s' % (cleanartistalbum)).encode('utf-8') - altterm = re.sub('[\.\-]', ' ', '%s %s' % (cleanartistalbum, year)).encode('utf-8') - - # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums - if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4: - term = altterm - - logger.info("Searching for %s since it was marked as wanted" % term) - - resultlist = [] - - if headphones.NZBMATRIX: + # FLAC usually doesn't have a year for some reason so I'll leave it out: + term = re.sub('[\.\-]', ' ', '%s' % (cleanartistalbum)).encode('utf-8') + altterm = re.sub('[\.\-]', ' ', '%s %s' % (cleanartistalbum, year)).encode('utf-8') + + # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums + if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4: + term = altterm + + logger.info("Searching for %s since it was marked as wanted" % term) + + resultlist = [] + + if headphones.NZBMATRIX: + provider = "nzbmatrix" + if headphones.PREFERRED_QUALITY == 3: + categories = "23" + maxsize = 10000000000 + elif headphones.PREFERRED_QUALITY: + categories = "23,22" + maxsize = 2000000000 + else: + categories = "22" + maxsize = 300000000 + + + params = { "page": "download", + "username": headphones.NZBMATRIX_USERNAME, + "apikey": headphones.NZBMATRIX_APIKEY, + "subcat": categories, + "age": headphones.USENET_RETENTION, + "english": 1, + "ssl": 1, + "scenename": 1, + "term": term + } + + searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params) + logger.info(u"Parsing results from "+searchURL) + d = feedparser.parse(searchURL) + + for item in d.entries: + try: + url = item.link + title = item.title + size = int(item.links[1]['length']) + if size < maxsize: + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + else: + logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + + except AttributeError, e: + logger.info(u"No results found from NZBMatrix for %s" % term) + + if headphones.NEWZNAB: + provider = "newznab" + if headphones.PREFERRED_QUALITY == 3: + categories = "3040" + maxsize = 10000000000 + elif headphones.PREFERRED_QUALITY: + categories = "3040,3010" + maxsize = 2000000000 + else: + categories = "3010" + maxsize = 300000000 - if headphones.PREFERRED_QUALITY == 3: - categories = "23" - maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY: - categories = "23,22" - maxsize = 2000000000 - else: - categories = "22" - maxsize = 300000000 - - - params = { "page": "download", - "username": headphones.NZBMATRIX_USERNAME, - "apikey": headphones.NZBMATRIX_APIKEY, - "subcat": categories, - "age": headphones.USENET_RETENTION, - "english": 1, - "ssl": 1, - "scenename": 1, - "term": term - } - - searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params) - logger.info(u"Parsing results from "+searchURL) - d = feedparser.parse(searchURL) - - for item in d.entries: - try: - url = item.link - title = item.title - size = int(item.links[1]['length']) - if size < maxsize: - resultlist.append((title, size, url)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) - - except AttributeError, e: - logger.info(u"No results found from NZBMatrix for %s" % term) - - if headphones.NEWZNAB: - - if headphones.PREFERRED_QUALITY == 3: - categories = "3040" - maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY: - categories = "3040,3010" - maxsize = 2000000000 - else: - categories = "3010" - maxsize = 300000000 + params = { "t": "search", + "apikey": headphones.NEWZNAB_APIKEY, + "cat": categories, + "maxage": headphones.USENET_RETENTION, + "q": term + } + + searchURL = headphones.NEWZNAB_HOST + '/api?' + urllib.urlencode(params) + logger.info(u"Parsing results from "+searchURL) + + d = feedparser.parse(searchURL) + + if not len(d.entries): + logger.info(u"No results found from %s for %s" % (headphones.NEWZNAB_HOST, term)) + pass + + else: + for item in d.entries: + try: + url = item.link + title = item.title + size = int(item.links[1]['length']) + if size < maxsize: + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + else: + logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + + except Exception, e: + logger.error(u"An unknown error occured trying to parse the feed: %s" % e) + + if headphones.NZBSORG: + provider = "nzbsorg" + if headphones.PREFERRED_QUALITY == 3: + categories = "5" + maxsize = 10000000000 + term = term + ' flac' + elif headphones.PREFERRED_QUALITY: + categories = "5" + maxsize = 2000000000 + else: + categories = "5" + maxsize = 300000000 - params = { "t": "search", - "apikey": headphones.NEWZNAB_APIKEY, - "cat": categories, - "maxage": headphones.USENET_RETENTION, - "q": term - } - - searchURL = headphones.NEWZNAB_HOST + '/api?' + urllib.urlencode(params) - logger.info(u"Parsing results from "+searchURL) - - d = feedparser.parse(searchURL) - - if not len(d.entries): - logger.info(u"No results found from %s for %s" % (headphones.NEWZNAB_HOST, term)) - pass - - else: - for item in d.entries: - try: - url = item.link - title = item.title - size = int(item.links[1]['length']) - if size < maxsize: - resultlist.append((title, size, url)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) - - except Exception, e: - logger.error(u"An unknown error occured trying to parse the feed: %s" % e) - - if headphones.NZBSORG: - - if headphones.PREFERRED_QUALITY == 3: - categories = "5" - maxsize = 10000000000 - term = term + ' flac' - elif headphones.PREFERRED_QUALITY: - categories = "5" - maxsize = 2000000000 - else: - categories = "5" - maxsize = 300000000 + params = { "action": "search", + "dl": 1, + "catid": categories, + "i": headphones.NZBSORG_UID, + "h": headphones.NZBSORG_HASH, + "age": headphones.USENET_RETENTION, + "q": term + } + + searchURL = 'https://secure.nzbs.org/rss.php?' + urllib.urlencode(params) + + #data = urllib.urlopen(searchURL).read() + data = urllib.urlopen(searchURL).read() + + logger.info(u"Parsing results from "+searchURL) + + try: + d = minidom.parseString(data) + node = d.documentElement + items = d.getElementsByTagName("item") + except ExpatError: + logger.error('Unable to get the NZBs.org feed. Check that your settings are correct - post a bug if they are') + items = None + + if len(items): + + for item in items: + + sizenode = item.getElementsByTagName("report:size")[0].childNodes + titlenode = item.getElementsByTagName("title")[0].childNodes + linknode = item.getElementsByTagName("link")[0].childNodes + + for node in sizenode: + size = int(node.data) + for node in titlenode: + title = node.data + for node in linknode: + url = node.data + + if size < maxsize: + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + else: + logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + + else: + + logger.info('No results found from NZBs.org for %s' % term) - params = { "action": "search", - "dl": 1, - "catid": categories, - "i": headphones.NZBSORG_UID, - "h": headphones.NZBSORG_HASH, - "age": headphones.USENET_RETENTION, - "q": term - } - - searchURL = 'https://secure.nzbs.org/rss.php?' + urllib.urlencode(params) - - data = urllib.urlopen(searchURL).read() - - logger.info(u"Parsing results from "+searchURL) - - try: - d = minidom.parseString(data) - node = d.documentElement - items = d.getElementsByTagName("item") - except ExpatError: - logger.error('Unable to get the NZBs.org feed. Check that your settings are correct - post a bug if they are') - items = None - - if len(items): - - for item in items: - - sizenode = item.getElementsByTagName("report:size")[0].childNodes - titlenode = item.getElementsByTagName("title")[0].childNodes - linknode = item.getElementsByTagName("link")[0].childNodes - - for node in sizenode: - size = int(node.data) - for node in titlenode: - title = node.data - for node in linknode: - url = node.data - - if size < maxsize: - resultlist.append((title, size, url)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) - - else: - - logger.info('No results found from NZBs.org for %s' % term) - - if len(resultlist): - - if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE: + if headphones.NEWZBIN: + provider = "newzbin" + providerurl = "https:/www.newzbin.com/" + if headphones.PREFERRED_QUALITY == 3: + categories = "7" #music + format = "2" #flac + maxsize = 10000000000 + elif headphones.PREFERRED_QUALITY: + categories = "7" #music + format = "10" #mp3+flac + maxsize = 2000000000 + else: + categories = "7" #music + format = "8" #mp3 + maxsize = 300000000 - logger.debug('Target bitrate: %s kbps' % headphones.PREFERRED_BITRATE) + params = { + "fpn": "p", + 'u_nfo_posts_only': 0, + 'u_url_posts_only': 0, + 'u_comment_posts_only': 0, + 'u_show_passworded': 0, + "searchaction": "Search", + #"dl": 1, + "category": categories, + "retention": headphones.USENET_RETENTION, + "ps_rb_audio_format": format, + "feed": "rss", + "u_post_results_amt": 50, #this can default to a high number per user + "hauth": 1, + "q": term + } + searchURL = providerurl + "search/?%s" % urllib.urlencode(params) + data = getNewzbinURL(searchURL) + if data: + logger.info(u"Parsing results from "+searchURL) + + try: + d = minidom.parseString(data) + node = d.documentElement + items = d.getElementsByTagName("item") + except ExpatError: + logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are') + items = None + + if len(items): + + for item in items: + + sizenode = item.getElementsByTagName("report:size")[0].childNodes + titlenode = item.getElementsByTagName("title")[0].childNodes + linknode = item.getElementsByTagName("link")[0].childNodes + + for node in sizenode: + size = int(node.data) + for node in titlenode: + title = node.data + for node in linknode: + url = node.data + + #exract the reportid from the link nodes + id_regex = re.escape(providerurl) + 'browse/post/(\d+)/' + id_match = re.match(id_regex, url) + if not id_match: + logger.info("Didn't find a valid Newzbin reportid in linknode") + else: + url = id_match.group(1) #we have to make a post request later, need the id + if size < maxsize and url: + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + else: + logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + + else: + logger.info('No results found from NEWZBIN for %s' % term) + + if len(resultlist): + + if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE: - tracks = myDB.select('SELECT TrackDuration from tracks WHERE AlbumID=?', [albumid]) + logger.debug('Target bitrate: %s kbps' % headphones.PREFERRED_BITRATE) - try: - albumlength = sum([pair[0] for pair in tracks]) - logger.debug('Album length = %s' % albumlength) - targetsize = albumlength/1000 * int(headphones.PREFERRED_BITRATE) * 128 - logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) - - newlist = [] + tracks = myDB.select('SELECT TrackDuration from tracks WHERE AlbumID=?', [albumid]) - for result in resultlist: - delta = abs(targetsize - result[1]) - newlist.append((result[0], result[1], result[2], delta)) - - nzblist = sorted(newlist, key=lambda title: title[3]) - - except Exception, e: - - logger.debug('Error: %s' % str(e)) - logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - - nzblist = sorted(resultlist, key=lambda title: title[1], reverse=True) - - else: - - nzblist = sorted(resultlist, key=lambda title: title[1], reverse=True) - - - if new: - # Checks to see if it's already downloaded - i = 0 - - while i < len(nzblist): - alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [nzblist[i][2]]) - - if len(alreadydownloaded) >= 1: - logger.info('%s has already been downloaded. Skipping.' % nzblist[i][0]) - i += 1 - - else: - bestqual = nzblist[i] - break - - try: - x = bestqual[0] - except UnboundLocalError: - logger.info('No more matches for %s' % term) - return - - else: - bestqual = nzblist[0] - - - logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1]))) - - downloadurl = bestqual[2] - nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8'), helpers.latinToAscii(albums[1]).encode('UTF-8'), year) + try: + albumlength = sum([pair[0] for pair in tracks]) + logger.debug('Album length = %s' % albumlength) + targetsize = albumlength/1000 * int(headphones.PREFERRED_BITRATE) * 128 + logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) + + newlist = [] - if headphones.SAB_HOST and not headphones.BLACKHOLE: - linkparams = {} - - linkparams["mode"] = "addurl" - - if headphones.SAB_APIKEY: - linkparams["apikey"] = headphones.SAB_APIKEY - if headphones.SAB_USERNAME: - linkparams["ma_username"] = headphones.SAB_USERNAME - if headphones.SAB_PASSWORD: - linkparams["ma_password"] = headphones.SAB_PASSWORD - if headphones.SAB_CATEGORY: - linkparams["cat"] = headphones.SAB_CATEGORY - - linkparams["name"] = downloadurl - - linkparams["nzbname"] = nzb_folder_name - - saburl = 'http://' + headphones.SAB_HOST + '/sabnzbd/api?' + urllib.urlencode(linkparams) - logger.info(u"Sending link to SABNZBD: " + saburl) - - try: - urllib.urlopen(saburl) - - except: - logger.error(u"Unable to send link. Are you sure the host address is correct?") - break - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) - - - elif headphones.BLACKHOLE: - - nzb_name = nzb_folder_name + '.nzb' - download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) - - try: - urllib.urlretrieve(downloadurl, download_path) - except Exception, e: - logger.error('Couldn\'t retrieve NZB: %s' % e) - break - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) \ No newline at end of file + for result in resultlist: + delta = abs(targetsize - result[1]) + newlist.append((result[0], result[1], result[2], delta)) + + nzblist = sorted(newlist, key=lambda title: title[3]) + + except Exception, e: + + logger.debug('Error: %s' % str(e)) + logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) + + nzblist = sorted(resultlist, key=lambda title: title[1], reverse=True) + + else: + + nzblist = sorted(resultlist, key=lambda title: title[1], reverse=True) + + + if new: + # Checks to see if it's already downloaded + i = 0 + + while i < len(nzblist): + alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [nzblist[i][2]]) + + if len(alreadydownloaded) >= 1: + logger.info('%s has already been downloaded. Skipping.' % nzblist[i][0]) + i += 1 + + else: + bestqual = nzblist[i] + break + + try: + x = bestqual[0] + except UnboundLocalError: + logger.info('No more matches for %s' % term) + return + + else: + bestqual = nzblist[0] + + + logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1]))) + + if bestqual[3] == "newzbin": + #logger.info("Found a newzbin result") + reportid = bestqual[2] + params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": reportid}) + url = providerurl + "/api/dnzb/" + urllib._urlopener = NewzbinDownloader() + data = urllib.urlopen(url, data=params).read() + nzb = classes.NZBDataSearchResult() + nzb.extraInfo.append(data) + nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8'), helpers.latinToAscii(albums[1]).encode('UTF-8'), year) + nzb.name = nzb_folder_name + logger.info(u"Sending FILE to SABNZBD: " + nzb.name) + sab.sendNZB(nzb) + + myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) + else: + downloadurl = bestqual[2] + nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8'), helpers.latinToAscii(albums[1]).encode('UTF-8'), year) + + if headphones.SAB_HOST and not headphones.BLACKHOLE: + linkparams = {} + + linkparams["mode"] = "addurl" + + if headphones.SAB_APIKEY: + linkparams["apikey"] = headphones.SAB_APIKEY + if headphones.SAB_USERNAME: + linkparams["ma_username"] = headphones.SAB_USERNAME + if headphones.SAB_PASSWORD: + linkparams["ma_password"] = headphones.SAB_PASSWORD + if headphones.SAB_CATEGORY: + linkparams["cat"] = headphones.SAB_CATEGORY + + linkparams["name"] = downloadurl + + linkparams["nzbname"] = nzb_folder_name + + saburl = 'http://' + headphones.SAB_HOST + '/sabnzbd/api?' + urllib.urlencode(linkparams) + logger.info(u"Sending link to SABNZBD: " + saburl) + + try: + urllib.urlopen(saburl) + + except: + logger.error(u"Unable to send link. Are you sure the host address is correct?") + break + + myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) + + + elif headphones.BLACKHOLE: + + nzb_name = nzb_folder_name + '.nzb' + download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) + + try: + urllib.urlretrieve(downloadurl, download_path) + except Exception, e: + logger.error('Couldn\'t retrieve NZB: %s' % e) + break + + myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) \ No newline at end of file diff --git a/headphones/templates.py b/headphones/templates.py index 40671190..a5efb9cf 100644 --- a/headphones/templates.py +++ b/headphones/templates.py @@ -258,6 +258,32 @@ configform = form = '''

+ + + + +
+ +

Newzbin:

+ + + +
+ +

+ Newzbin UID:
+ +

+ + + +
+ +

+ Newzbin Password:
+ +

+ diff --git a/headphones/webserve.py b/headphones/webserve.py index 31f22821..f20f5a97 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -565,6 +565,9 @@ class WebInterface(object): checked(headphones.NZBSORG), headphones.NZBSORG_UID, headphones.NZBSORG_HASH, + checked(headphones.NEWZBIN), + headphones.NEWZBIN_UID, + headphones.NEWZBIN_PASSWORD, radio(headphones.PREFERRED_QUALITY, 0), radio(headphones.PREFERRED_QUALITY, 1), radio(headphones.PREFERRED_QUALITY, 3), @@ -592,7 +595,7 @@ class WebInterface(object): def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, download_dir=None, blackhole=0, blackhole_dir=None, usenet_retention=None, nzbmatrix=0, nzbmatrix_username=None, nzbmatrix_apikey=None, newznab=0, newznab_host=None, newznab_apikey=None, - nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, + nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, newzbin=0, newzbin_uid=None, newzbin_password=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, embed_album_art=0, destination_dir=None, folder_format=None, file_format=None, include_extras=0, log_dir=None): headphones.HTTP_HOST = http_host @@ -618,6 +621,9 @@ class WebInterface(object): headphones.NZBSORG = nzbsorg headphones.NZBSORG_UID = nzbsorg_uid headphones.NZBSORG_HASH = nzbsorg_hash + headphones.NEWZBIN = newzbin + headphones.NEWZBIN_UID = newzbin_uid + headphones.NEWZBIN_PASSWORD = newzbin_password headphones.PREFERRED_QUALITY = int(preferred_quality) headphones.PREFERRED_BITRATE = preferred_bitrate headphones.DETECT_BITRATE = detect_bitrate