From d2ae29d2d6d8c9c6328488e6e5943aafa8e561b5 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Sat, 28 Jul 2012 23:56:28 +0530 Subject: [PATCH] Updated cherrypy to 3.2.2 --- cherrypy/__init__.py | 16 +- cherrypy/_cpcompat.py | 45 +- cherrypy/_cpdispatch.py | 28 +- cherrypy/_cperror.py | 53 +- cherrypy/_cplogging.py | 85 +- cherrypy/_cpmodpy.py | 15 +- cherrypy/_cpreqbody.py | 50 +- cherrypy/_cprequest.py | 48 +- cherrypy/_cpserver.py | 24 +- cherrypy/_cptools.py | 4 +- cherrypy/_cptree.py | 35 +- cherrypy/_cpwsgi.py | 95 +- cherrypy/_cpwsgi_server.py | 13 +- cherrypy/lib/__init__.py | 2 +- cherrypy/lib/cpstats.py | 7 +- cherrypy/lib/cptools.py | 14 +- cherrypy/lib/gctools.py | 214 ++ cherrypy/lib/httputil.py | 51 +- cherrypy/lib/jsontools.py | 2 +- cherrypy/lib/reprconf.py | 168 +- cherrypy/lib/sessions.py | 73 +- cherrypy/lib/static.py | 13 +- cherrypy/lib/{xmlrpc.py => xmlrpcutil.py} | 26 +- cherrypy/process/plugins.py | 12 +- cherrypy/process/servers.py | 21 +- cherrypy/process/wspbus.py | 57 +- cherrypy/test/__init__.py | 25 - cherrypy/test/_test_decorators.py | 41 - cherrypy/test/_test_states_demo.py | 66 - cherrypy/test/benchmark.py | 409 ---- cherrypy/test/checkerdemo.py | 47 - cherrypy/test/fastcgi.conf | 18 - cherrypy/test/fcgi.conf | 14 - cherrypy/test/helper.py | 476 ---- cherrypy/test/logtest.py | 181 -- cherrypy/test/modfastcgi.py | 135 -- cherrypy/test/modfcgid.py | 125 -- cherrypy/test/modpy.py | 163 -- cherrypy/test/modwsgi.py | 148 -- cherrypy/test/native-server.ini | 9 - cherrypy/test/sessiondemo.py | 153 -- cherrypy/test/static/dirback.jpg | Bin 18238 -> 0 bytes cherrypy/test/static/index.html | 1 - cherrypy/test/style.css | 1 - cherrypy/test/test.pem | 38 - cherrypy/test/test_auth_basic.py | 79 - cherrypy/test/test_auth_digest.py | 115 - cherrypy/test/test_bus.py | 263 --- cherrypy/test/test_caching.py | 329 --- cherrypy/test/test_config.py | 249 --- cherrypy/test/test_config_server.py | 121 - cherrypy/test/test_conn.py | 734 ------- cherrypy/test/test_core.py | 617 ------ cherrypy/test/test_dynamicobjectmapping.py | 403 ---- cherrypy/test/test_encoding.py | 363 --- cherrypy/test/test_etags.py | 81 - cherrypy/test/test_http.py | 168 -- cherrypy/test/test_httpauth.py | 151 -- cherrypy/test/test_httplib.py | 29 - cherrypy/test/test_json.py | 79 - cherrypy/test/test_logging.py | 149 -- cherrypy/test/test_mime.py | 128 -- cherrypy/test/test_misc_tools.py | 202 -- cherrypy/test/test_objectmapping.py | 403 ---- cherrypy/test/test_proxy.py | 129 -- cherrypy/test/test_refleaks.py | 119 - cherrypy/test/test_request_obj.py | 722 ------ cherrypy/test/test_routes.py | 69 - cherrypy/test/test_session.py | 464 ---- cherrypy/test/test_sessionauthenticate.py | 62 - cherrypy/test/test_states.py | 436 ---- cherrypy/test/test_static.py | 300 --- cherrypy/test/test_tools.py | 393 ---- cherrypy/test/test_tutorials.py | 201 -- cherrypy/test/test_virtualhost.py | 107 - cherrypy/test/test_wsgi_ns.py | 80 - cherrypy/test/test_wsgi_vhost.py | 36 - cherrypy/test/test_wsgiapps.py | 111 - cherrypy/test/test_xmlrpc.py | 172 -- cherrypy/test/webtest.py | 535 ----- cherrypy/wsgiserver/__init__.py | 2227 +------------------ cherrypy/wsgiserver/ssl_builtin.py | 25 +- cherrypy/wsgiserver/wsgiserver2.py | 2322 ++++++++++++++++++++ cherrypy/wsgiserver/wsgiserver3.py | 2040 +++++++++++++++++ 84 files changed, 5359 insertions(+), 13075 deletions(-) create mode 100644 cherrypy/lib/gctools.py rename cherrypy/lib/{xmlrpc.py => xmlrpcutil.py} (61%) delete mode 100644 cherrypy/test/__init__.py delete mode 100644 cherrypy/test/_test_decorators.py delete mode 100644 cherrypy/test/_test_states_demo.py delete mode 100644 cherrypy/test/benchmark.py delete mode 100644 cherrypy/test/checkerdemo.py delete mode 100644 cherrypy/test/fastcgi.conf delete mode 100644 cherrypy/test/fcgi.conf delete mode 100644 cherrypy/test/helper.py delete mode 100644 cherrypy/test/logtest.py delete mode 100644 cherrypy/test/modfastcgi.py delete mode 100644 cherrypy/test/modfcgid.py delete mode 100644 cherrypy/test/modpy.py delete mode 100644 cherrypy/test/modwsgi.py delete mode 100644 cherrypy/test/native-server.ini delete mode 100755 cherrypy/test/sessiondemo.py delete mode 100644 cherrypy/test/static/dirback.jpg delete mode 100644 cherrypy/test/static/index.html delete mode 100644 cherrypy/test/style.css delete mode 100644 cherrypy/test/test.pem delete mode 100644 cherrypy/test/test_auth_basic.py delete mode 100644 cherrypy/test/test_auth_digest.py delete mode 100644 cherrypy/test/test_bus.py delete mode 100644 cherrypy/test/test_caching.py delete mode 100644 cherrypy/test/test_config.py delete mode 100644 cherrypy/test/test_config_server.py delete mode 100644 cherrypy/test/test_conn.py delete mode 100644 cherrypy/test/test_core.py delete mode 100644 cherrypy/test/test_dynamicobjectmapping.py delete mode 100644 cherrypy/test/test_encoding.py delete mode 100644 cherrypy/test/test_etags.py delete mode 100644 cherrypy/test/test_http.py delete mode 100644 cherrypy/test/test_httpauth.py delete mode 100644 cherrypy/test/test_httplib.py delete mode 100644 cherrypy/test/test_json.py delete mode 100644 cherrypy/test/test_logging.py delete mode 100644 cherrypy/test/test_mime.py delete mode 100644 cherrypy/test/test_misc_tools.py delete mode 100644 cherrypy/test/test_objectmapping.py delete mode 100644 cherrypy/test/test_proxy.py delete mode 100644 cherrypy/test/test_refleaks.py delete mode 100644 cherrypy/test/test_request_obj.py delete mode 100644 cherrypy/test/test_routes.py delete mode 100755 cherrypy/test/test_session.py delete mode 100644 cherrypy/test/test_sessionauthenticate.py delete mode 100644 cherrypy/test/test_states.py delete mode 100644 cherrypy/test/test_static.py delete mode 100644 cherrypy/test/test_tools.py delete mode 100644 cherrypy/test/test_tutorials.py delete mode 100644 cherrypy/test/test_virtualhost.py delete mode 100644 cherrypy/test/test_wsgi_ns.py delete mode 100644 cherrypy/test/test_wsgi_vhost.py delete mode 100644 cherrypy/test/test_wsgiapps.py delete mode 100644 cherrypy/test/test_xmlrpc.py delete mode 100644 cherrypy/test/webtest.py create mode 100644 cherrypy/wsgiserver/wsgiserver2.py create mode 100644 cherrypy/wsgiserver/wsgiserver3.py diff --git a/cherrypy/__init__.py b/cherrypy/__init__.py index eb7cabf6..41e3898b 100644 --- a/cherrypy/__init__.py +++ b/cherrypy/__init__.py @@ -57,10 +57,10 @@ These API's are described in the CherryPy specification: http://www.cherrypy.org/wiki/CherryPySpec """ -__version__ = "3.2.0" +__version__ = "3.2.2" from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode -from cherrypy._cpcompat import basestring, unicodestr +from cherrypy._cpcompat import basestring, unicodestr, set from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect from cherrypy._cperror import NotFound, CherryPyException, TimeoutError @@ -89,17 +89,21 @@ except ImportError: engine = process.bus -# Timeout monitor +# Timeout monitor. We add two channels to the engine +# to which cherrypy.Application will publish. +engine.listeners['before_request'] = set() +engine.listeners['after_request'] = set() + class _TimeoutMonitor(process.plugins.Monitor): def __init__(self, bus): self.servings = [] process.plugins.Monitor.__init__(self, bus, self.run) - def acquire(self): + def before_request(self): self.servings.append((serving.request, serving.response)) - def release(self): + def after_request(self): try: self.servings.remove((serving.request, serving.response)) except ValueError: @@ -585,7 +589,7 @@ def url(path="", qs="", script_name=None, base=None, relative=None): elif relative: # "A relative reference that does not begin with a scheme name # or a slash character is termed a relative-path reference." - old = url().split('/')[:-1] + old = url(relative=False).split('/')[:-1] new = newurl.split('/') while old and new: a, b = old[0], new[0] diff --git a/cherrypy/_cpcompat.py b/cherrypy/_cpcompat.py index 216ddddc..ed24c1ab 100644 --- a/cherrypy/_cpcompat.py +++ b/cherrypy/_cpcompat.py @@ -16,9 +16,11 @@ It also provides a 'base64_decode' function with native strings as input and output. """ import os +import re import sys if sys.version_info >= (3, 0): + py3k = True bytestr = bytes unicodestr = str nativestr = unicodestr @@ -31,12 +33,19 @@ if sys.version_info >= (3, 0): """Return the given native string as a unicode string with the given encoding.""" # In Python 3, the native string type is unicode return n + def tonative(n, encoding='ISO-8859-1'): + """Return the given string as a native string in the given encoding.""" + # In Python 3, the native string type is unicode + if isinstance(n, bytes): + return n.decode(encoding) + return n # type("") from io import StringIO # bytes: from io import BytesIO as BytesIO else: # Python 2 + py3k = False bytestr = str unicodestr = unicode nativestr = bytestr @@ -49,10 +58,25 @@ else: return n def ntou(n, encoding='ISO-8859-1'): """Return the given native string as a unicode string with the given encoding.""" - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. + # In Python 2, the native string type is bytes. + # First, check for the special encoding 'escape'. The test suite uses this + # to signal that it wants to pass a string with embedded \uXXXX escapes, + # but without having to prefix it with u'' for Python 2, but no prefix + # for Python 3. + if encoding == 'escape': + return unicode( + re.sub(r'\\u([0-9a-zA-Z]{4})', + lambda m: unichr(int(m.group(1), 16)), + n.decode('ISO-8859-1'))) + # Assume it's already in the given encoding, which for ISO-8859-1 is almost + # always what was intended. return n.decode(encoding) + def tonative(n, encoding='ISO-8859-1'): + """Return the given string as a native string in the given encoding.""" + # In Python 2, the native string type is bytes. + if isinstance(n, unicode): + return n.encode(encoding) + return n try: # type("") from cStringIO import StringIO @@ -185,6 +209,18 @@ except ImportError: from http.client import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected from http.server import BaseHTTPRequestHandler +try: + # Python 2. We have to do it in this order so Python 2 builds + # don't try to import the 'http' module from cherrypy.lib + from httplib import HTTPSConnection +except ImportError: + try: + # Python 3 + from http.client import HTTPSConnection + except ImportError: + # Some platforms which don't have SSL don't expose HTTPSConnection + HTTPSConnection = None + try: # Python 2 xrange = xrange @@ -229,7 +265,7 @@ try: json_decode = json.JSONDecoder().decode json_encode = json.JSONEncoder().iterencode except ImportError: - if sys.version_info >= (3, 0): + if py3k: # Python 3.0: json is part of the standard library, # but outputs unicode. We need bytes. import json @@ -280,4 +316,3 @@ except NameError: # Python 2 def next(i): return i.next() - diff --git a/cherrypy/_cpdispatch.py b/cherrypy/_cpdispatch.py index 7250ac92..d614e086 100644 --- a/cherrypy/_cpdispatch.py +++ b/cherrypy/_cpdispatch.py @@ -12,8 +12,13 @@ to a hierarchical arrangement of objects, starting at request.app.root. import string import sys import types +try: + classtype = (type, types.ClassType) +except AttributeError: + classtype = type import cherrypy +from cherrypy._cpcompat import set class PageHandler(object): @@ -197,8 +202,18 @@ class LateParamPageHandler(PageHandler): 'cherrypy.request.params copied in)') -punctuation_to_underscores = string.maketrans( - string.punctuation, '_' * len(string.punctuation)) +if sys.version_info < (3, 0): + punctuation_to_underscores = string.maketrans( + string.punctuation, '_' * len(string.punctuation)) + def validate_translator(t): + if not isinstance(t, str) or len(t) != 256: + raise ValueError("The translate argument must be a str of len 256.") +else: + punctuation_to_underscores = str.maketrans( + string.punctuation, '_' * len(string.punctuation)) + def validate_translator(t): + if not isinstance(t, dict): + raise ValueError("The translate argument must be a dict.") class Dispatcher(object): """CherryPy Dispatcher which walks a tree of objects to find a handler. @@ -222,8 +237,7 @@ class Dispatcher(object): def __init__(self, dispatch_method_name=None, translate=punctuation_to_underscores): - if not isinstance(translate, str) or len(translate) != 256: - raise ValueError("The translate argument must be a str of len 256.") + validate_translator(translate) self.translate = translate if dispatch_method_name: self.dispatch_method_name = dispatch_method_name @@ -524,7 +538,7 @@ class RoutesDispatcher(object): controller = result.get('controller') controller = self.controllers.get(controller, controller) if controller: - if isinstance(controller, (type, types.ClassType)): + if isinstance(controller, classtype): controller = controller() # Get config from the controller. if hasattr(controller, "_cp_config"): @@ -550,9 +564,9 @@ class RoutesDispatcher(object): def XMLRPCDispatcher(next_dispatcher=Dispatcher()): - from cherrypy.lib import xmlrpc + from cherrypy.lib import xmlrpcutil def xmlrpc_dispatch(path_info): - path_info = xmlrpc.patched_path(path_info) + path_info = xmlrpcutil.patched_path(path_info) return next_dispatcher(path_info) return xmlrpc_dispatch diff --git a/cherrypy/_cperror.py b/cherrypy/_cperror.py index 00e5b532..76a409ff 100644 --- a/cherrypy/_cperror.py +++ b/cherrypy/_cperror.py @@ -107,7 +107,7 @@ and not simply return an error message as a result. from cgi import escape as _escape from sys import exc_info as _exc_info from traceback import format_exception as _format_exception -from cherrypy._cpcompat import basestring, iteritems, urljoin as _urljoin +from cherrypy._cpcompat import basestring, bytestr, iteritems, ntob, tonative, urljoin as _urljoin from cherrypy.lib import httputil as _httputil @@ -183,7 +183,7 @@ class HTTPRedirect(CherryPyException): """The list of URL's to emit.""" encoding = 'utf-8' - """The encoding when passed urls are unicode objects""" + """The encoding when passed urls are not native strings""" def __init__(self, urls, status=None, encoding=None): import cherrypy @@ -194,8 +194,7 @@ class HTTPRedirect(CherryPyException): abs_urls = [] for url in urls: - if isinstance(url, unicode): - url = url.encode(encoding or self.encoding) + url = tonative(url, encoding or self.encoding) # Note that urljoin will "do the right thing" whether url is: # 1. a complete URL with host (e.g. "http://www.example.com/test") @@ -248,7 +247,7 @@ class HTTPRedirect(CherryPyException): 307: "This resource has moved temporarily to %s.", }[status] msgs = [msg % (u, u) for u in self.urls] - response.body = "
\n".join(msgs) + response.body = ntob("
\n".join(msgs), 'utf-8') # Previous code may have set C-L, so we have to reset it # (allow finalize to set it). response.headers.pop('Content-Length', None) @@ -341,8 +340,8 @@ class HTTPError(CherryPyException): self.status = status try: self.code, self.reason, defaultmsg = _httputil.valid_status(status) - except ValueError, x: - raise self.__class__(500, x.args[0]) + except ValueError: + raise self.__class__(500, _exc_info()[1].args[0]) if self.code < 400 or self.code > 599: raise ValueError("status must be between 400 and 599.") @@ -373,8 +372,8 @@ class HTTPError(CherryPyException): response.headers['Content-Type'] = "text/html;charset=utf-8" response.headers.pop('Content-Length', None) - content = self.get_error_page(self.status, traceback=tb, - message=self._message) + content = ntob(self.get_error_page(self.status, traceback=tb, + message=self._message), 'utf-8') response.body = content _be_ie_unfriendly(self.code) @@ -442,8 +441,8 @@ def get_error_page(status, **kwargs): try: code, reason, message = _httputil.valid_status(status) - except ValueError, x: - raise cherrypy.HTTPError(500, x.args[0]) + except ValueError: + raise cherrypy.HTTPError(500, _exc_info()[1].args[0]) # We can't use setdefault here, because some # callers send None for kwarg values. @@ -470,7 +469,8 @@ def get_error_page(status, **kwargs): if hasattr(error_page, '__call__'): return error_page(**kwargs) else: - return open(error_page, 'rb').read() % kwargs + data = open(error_page, 'rb').read() + return tonative(data) % kwargs except: e = _format_exception(*_exc_info())[-1] m = kwargs['message'] @@ -508,19 +508,22 @@ def _be_ie_unfriendly(status): if l and l < s: # IN ADDITION: the response must be written to IE # in one chunk or it will still get replaced! Bah. - content = content + (" " * (s - l)) + content = content + (ntob(" ") * (s - l)) response.body = content response.headers['Content-Length'] = str(len(content)) def format_exc(exc=None): """Return exc (or sys.exc_info if None), formatted.""" - if exc is None: - exc = _exc_info() - if exc == (None, None, None): - return "" - import traceback - return "".join(traceback.format_exception(*exc)) + try: + if exc is None: + exc = _exc_info() + if exc == (None, None, None): + return "" + import traceback + return "".join(traceback.format_exception(*exc)) + finally: + del exc def bare_error(extrabody=None): """Produce status, headers, body for a critical error. @@ -539,15 +542,15 @@ def bare_error(extrabody=None): # it cannot be allowed to fail. Therefore, don't add to it! # In particular, don't call any other CP functions. - body = "Unrecoverable error in the server." + body = ntob("Unrecoverable error in the server.") if extrabody is not None: - if not isinstance(extrabody, str): + if not isinstance(extrabody, bytestr): extrabody = extrabody.encode('utf-8') - body += "\n" + extrabody + body += ntob("\n") + extrabody - return ("500 Internal Server Error", - [('Content-Type', 'text/plain'), - ('Content-Length', str(len(body)))], + return (ntob("500 Internal Server Error"), + [(ntob('Content-Type'), ntob('text/plain')), + (ntob('Content-Length'), ntob(str(len(body)),'ISO-8859-1'))], [body]) diff --git a/cherrypy/_cplogging.py b/cherrypy/_cplogging.py index d6ca979e..e10c9420 100644 --- a/cherrypy/_cplogging.py +++ b/cherrypy/_cplogging.py @@ -109,6 +109,20 @@ import sys import cherrypy from cherrypy import _cperror +from cherrypy._cpcompat import ntob, py3k + + +class NullHandler(logging.Handler): + """A no-op logging handler to silence the logging.lastResort handler.""" + + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None class LogManager(object): @@ -127,8 +141,12 @@ class LogManager(object): access_log = None """The actual :class:`logging.Logger` instance for access messages.""" - access_log_format = \ - '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + if py3k: + access_log_format = \ + '{h} {l} {u} {t} "{r}" {s} {b} "{f}" "{a}"' + else: + access_log_format = \ + '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' logger_root = None """The "top-level" logger name. @@ -152,8 +170,13 @@ class LogManager(object): self.access_log = logging.getLogger("%s.access.%s" % (logger_root, appid)) self.error_log.setLevel(logging.INFO) self.access_log.setLevel(logging.INFO) + + # Silence the no-handlers "warning" (stderr write!) in stdlib logging + self.error_log.addHandler(NullHandler()) + self.access_log.addHandler(NullHandler()) + cherrypy.engine.subscribe('graceful', self.reopen_files) - + def reopen_files(self): """Close and reopen all file handlers.""" for log in (self.error_log, self.access_log): @@ -206,7 +229,9 @@ class LogManager(object): if response.output_status is None: status = "-" else: - status = response.output_status.split(" ", 1)[0] + status = response.output_status.split(ntob(" "), 1)[0] + if py3k: + status = status.decode('ISO-8859-1') atoms = {'h': remote.name or remote.ip, 'l': '-', @@ -218,21 +243,43 @@ class LogManager(object): 'f': dict.get(inheaders, 'Referer', ''), 'a': dict.get(inheaders, 'User-Agent', ''), } - for k, v in atoms.items(): - if isinstance(v, unicode): - v = v.encode('utf8') - elif not isinstance(v, str): - v = str(v) - # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc - # and backslash for us. All we have to do is strip the quotes. - v = repr(v)[1:-1] - # Escape double-quote. - atoms[k] = v.replace('"', '\\"') - - try: - self.access_log.log(logging.INFO, self.access_log_format % atoms) - except: - self(traceback=True) + if py3k: + for k, v in atoms.items(): + if not isinstance(v, str): + v = str(v) + v = v.replace('"', '\\"').encode('utf8') + # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc + # and backslash for us. All we have to do is strip the quotes. + v = repr(v)[2:-1] + + # in python 3.0 the repr of bytes (as returned by encode) + # uses double \'s. But then the logger escapes them yet, again + # resulting in quadruple slashes. Remove the extra one here. + v = v.replace('\\\\', '\\') + + # Escape double-quote. + atoms[k] = v + + try: + self.access_log.log(logging.INFO, self.access_log_format.format(**atoms)) + except: + self(traceback=True) + else: + for k, v in atoms.items(): + if isinstance(v, unicode): + v = v.encode('utf8') + elif not isinstance(v, str): + v = str(v) + # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc + # and backslash for us. All we have to do is strip the quotes. + v = repr(v)[1:-1] + # Escape double-quote. + atoms[k] = v.replace('"', '\\"') + + try: + self.access_log.log(logging.INFO, self.access_log_format % atoms) + except: + self(traceback=True) def time(self): """Return now() in Apache Common Log Format (no timezone).""" diff --git a/cherrypy/_cpmodpy.py b/cherrypy/_cpmodpy.py index ba2ab22f..76ef6ead 100644 --- a/cherrypy/_cpmodpy.py +++ b/cherrypy/_cpmodpy.py @@ -224,7 +224,7 @@ def handler(req): qs = ir.query_string rfile = BytesIO() - send_response(req, response.status, response.header_list, + send_response(req, response.output_status, response.header_list, response.body, response.stream) finally: app.release_serving() @@ -266,11 +266,22 @@ def send_response(req, status, headers, body, stream=False): import os import re +try: + import subprocess + def popen(fullcmd): + p = subprocess.Popen(fullcmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + return p.stdout +except ImportError: + def popen(fullcmd): + pipein, pipeout = os.popen4(fullcmd) + return pipeout def read_process(cmd, args=""): fullcmd = "%s %s" % (cmd, args) - pipein, pipeout = os.popen4(fullcmd) + pipeout = popen(fullcmd) try: firstline = pipeout.readline() if (re.search(ntob("(not recognized|No such file|not found)"), firstline, diff --git a/cherrypy/_cpreqbody.py b/cherrypy/_cpreqbody.py index 1b0496e3..5d72c854 100644 --- a/cherrypy/_cpreqbody.py +++ b/cherrypy/_cpreqbody.py @@ -101,10 +101,28 @@ If we were defining a custom processor, we can do so without making a ``Tool``. Note that you can only replace the ``processors`` dict wholesale this way, not update the existing one. """ +try: + from io import DEFAULT_BUFFER_SIZE +except ImportError: + DEFAULT_BUFFER_SIZE = 8192 import re import sys import tempfile -from urllib import unquote_plus +try: + from urllib import unquote_plus +except ImportError: + def unquote_plus(bs): + """Bytes version of urllib.parse.unquote_plus.""" + bs = bs.replace(ntob('+'), ntob(' ')) + atoms = bs.split(ntob('%')) + for i in range(1, len(atoms)): + item = atoms[i] + try: + pct = int(item[:2], 16) + atoms[i] = bytes([pct]) + item[2:] + except ValueError: + pass + return ntob('').join(atoms) import cherrypy from cherrypy._cpcompat import basestring, ntob, ntou @@ -399,7 +417,6 @@ class Entity(object): # Copy the class 'attempt_charsets', prepending any Content-Type charset dec = self.content_type.params.get("charset", None) if dec: - #dec = dec.decode('ISO-8859-1') self.attempt_charsets = [dec] + [c for c in self.attempt_charsets if c != dec] else: @@ -446,11 +463,14 @@ class Entity(object): def __iter__(self): return self - def next(self): + def __next__(self): line = self.readline() if not line: raise StopIteration return line + + def next(self): + return self.__next__() def read_into_file(self, fp_out=None): """Read the request body into fp_out (or make_file() if None). Return fp_out.""" @@ -671,13 +691,16 @@ class Part(Entity): Entity.part_class = Part - -class Infinity(object): - def __cmp__(self, other): - return 1 - def __sub__(self, other): - return self -inf = Infinity() +try: + inf = float('inf') +except ValueError: + # Python 2.4 and lower + class Infinity(object): + def __cmp__(self, other): + return 1 + def __sub__(self, other): + return self + inf = Infinity() comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', @@ -689,7 +712,7 @@ comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', class SizedReader: - def __init__(self, fp, length, maxbytes, bufsize=8192, has_trailers=False): + def __init__(self, fp, length, maxbytes, bufsize=DEFAULT_BUFFER_SIZE, has_trailers=False): # Wrap our fp in a buffer so peek() works self.fp = fp self.length = length @@ -930,8 +953,9 @@ class RequestBody(Entity): request_params = self.request_params for key, value in self.params.items(): # Python 2 only: keyword arguments must be byte strings (type 'str'). - if isinstance(key, unicode): - key = key.encode('ISO-8859-1') + if sys.version_info < (3, 0): + if isinstance(key, unicode): + key = key.encode('ISO-8859-1') if key in request_params: if not isinstance(request_params[key], list): diff --git a/cherrypy/_cprequest.py b/cherrypy/_cprequest.py index ae5e8971..5890c728 100644 --- a/cherrypy/_cprequest.py +++ b/cherrypy/_cprequest.py @@ -6,7 +6,7 @@ import warnings import cherrypy from cherrypy._cpcompat import basestring, copykeys, ntob, unicodestr -from cherrypy._cpcompat import SimpleCookie, CookieError +from cherrypy._cpcompat import SimpleCookie, CookieError, py3k from cherrypy import _cpreqbody, _cpconfig from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil, file_generator @@ -49,7 +49,12 @@ class Hook(object): self.kwargs = kwargs + def __lt__(self, other): + # Python 3 + return self.priority < other.priority + def __cmp__(self, other): + # Python 2 return cmp(self.priority, other.priority) def __call__(self): @@ -104,7 +109,7 @@ class HookMap(dict): exc = sys.exc_info()[1] cherrypy.log(traceback=True, severity=40) if exc: - raise + raise exc def __copy__(self): newmap = self.__class__() @@ -488,14 +493,20 @@ class Request(object): self.stage = 'close' def run(self, method, path, query_string, req_protocol, headers, rfile): - """Process the Request. (Core) + r"""Process the Request. (Core) method, path, query_string, and req_protocol should be pulled directly from the Request-Line (e.g. "GET /path?key=val HTTP/1.0"). path This should be %XX-unquoted, but query_string should not be. - They both MUST be byte strings, not unicode strings. + + When using Python 2, they both MUST be byte strings, + not unicode strings. + + When using Python 3, they both MUST be unicode strings, + not byte strings, and preferably not bytes \x00-\xFF + disguised as unicode. headers A list of (name, value) tuples. @@ -676,10 +687,11 @@ class Request(object): self.query_string_encoding) # Python 2 only: keyword arguments must be byte strings (type 'str'). - for key, value in p.items(): - if isinstance(key, unicode): - del p[key] - p[key.encode(self.query_string_encoding)] = value + if not py3k: + for key, value in p.items(): + if isinstance(key, unicode): + del p[key] + p[key.encode(self.query_string_encoding)] = value self.params.update(p) def process_headers(self): @@ -770,6 +782,10 @@ class Request(object): class ResponseBody(object): """The body of the HTTP response (the response entity).""" + if py3k: + unicode_err = ("Page handlers MUST return bytes. Use tools.encode " + "if you wish to return unicode.") + def __get__(self, obj, objclass=None): if obj is None: # When calling on the class instead of an instance... @@ -779,6 +795,9 @@ class ResponseBody(object): def __set__(self, obj, value): # Convert the given value to an iterable object. + if py3k and isinstance(value, str): + raise ValueError(self.unicode_err) + if isinstance(value, basestring): # strings get wrapped in a list because iterating over a single # item list is much faster than iterating over every character @@ -788,6 +807,11 @@ class ResponseBody(object): else: # [''] doesn't evaluate to False, so replace it with []. value = [] + elif py3k and isinstance(value, list): + # every item in a list must be bytes... + for i, item in enumerate(value): + if isinstance(item, str): + raise ValueError(self.unicode_err) # Don't use isinstance here; io.IOBase which has an ABC takes # 1000 times as long as, say, isinstance(value, str) elif hasattr(value, 'read'): @@ -862,7 +886,12 @@ class Response(object): if isinstance(self.body, basestring): return self.body - newbody = ''.join([chunk for chunk in self.body]) + newbody = [] + for chunk in self.body: + if py3k and not isinstance(chunk, bytes): + raise TypeError("Chunk %s is not of type 'bytes'." % repr(chunk)) + newbody.append(chunk) + newbody = ntob('').join(newbody) self.body = newbody return newbody @@ -876,6 +905,7 @@ class Response(object): headers = self.headers + self.status = "%s %s" % (code, reason) self.output_status = ntob(str(code), 'ascii') + ntob(" ") + headers.encode(reason) if self.stream: diff --git a/cherrypy/_cpserver.py b/cherrypy/_cpserver.py index c1695a66..2eecd6ec 100644 --- a/cherrypy/_cpserver.py +++ b/cherrypy/_cpserver.py @@ -4,7 +4,7 @@ import warnings import cherrypy from cherrypy.lib import attributes -from cherrypy._cpcompat import basestring +from cherrypy._cpcompat import basestring, py3k # We import * because we want to export check_port # et al as attributes of this module. @@ -98,12 +98,22 @@ class Server(ServerAdapter): ssl_private_key = None """The filename of the private key to use with SSL.""" - ssl_module = 'pyopenssl' - """The name of a registered SSL adaptation module to use with the builtin - WSGI server. Builtin options are 'builtin' (to use the SSL library built - into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL - project, which you must install separately). You may also register your - own classes in the wsgiserver.ssl_adapters dict.""" + if py3k: + ssl_module = 'builtin' + """The name of a registered SSL adaptation module to use with the builtin + WSGI server. Builtin options are: 'builtin' (to use the SSL library built + into recent versions of Python). You may also register your + own classes in the wsgiserver.ssl_adapters dict.""" + else: + ssl_module = 'pyopenssl' + """The name of a registered SSL adaptation module to use with the builtin + WSGI server. Builtin options are 'builtin' (to use the SSL library built + into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL + project, which you must install separately). You may also register your + own classes in the wsgiserver.ssl_adapters dict.""" + + statistics = False + """Turns statistics-gathering on or off for aware HTTP servers.""" nodelay = True """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" diff --git a/cherrypy/_cptools.py b/cherrypy/_cptools.py index d3eab059..22316b31 100644 --- a/cherrypy/_cptools.py +++ b/cherrypy/_cptools.py @@ -243,7 +243,7 @@ class ErrorTool(Tool): # Builtin tools # from cherrypy.lib import cptools, encoding, auth, static, jsontools -from cherrypy.lib import sessions as _sessions, xmlrpc as _xmlrpc +from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc from cherrypy.lib import caching as _caching from cherrypy.lib import auth_basic, auth_digest @@ -367,7 +367,7 @@ class XMLRPCController(object): # http://www.cherrypy.org/ticket/533 # if a method is not found, an xmlrpclib.Fault should be returned # raising an exception here will do that; see - # cherrypy.lib.xmlrpc.on_error + # cherrypy.lib.xmlrpcutil.on_error raise Exception('method "%s" is not supported' % attr) conf = cherrypy.serving.request.toolmaps['tools'].get("xmlrpc", {}) diff --git a/cherrypy/_cptree.py b/cherrypy/_cptree.py index 67ce5465..3aa4b9e1 100644 --- a/cherrypy/_cptree.py +++ b/cherrypy/_cptree.py @@ -1,8 +1,10 @@ """CherryPy Application and Tree objects.""" import os +import sys + import cherrypy -from cherrypy._cpcompat import ntou +from cherrypy._cpcompat import ntou, py3k from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools from cherrypy.lib import httputil @@ -123,8 +125,8 @@ class Application(object): resp = self.response_class() cherrypy.serving.load(req, resp) - cherrypy.engine.timeout_monitor.acquire() cherrypy.engine.publish('acquire_thread') + cherrypy.engine.publish('before_request') return req, resp @@ -132,7 +134,7 @@ class Application(object): """Release the current serving (request and response).""" req = cherrypy.serving.request - cherrypy.engine.timeout_monitor.release() + cherrypy.engine.publish('after_request') try: req.close() @@ -266,14 +268,23 @@ class Tree(object): # Correct the SCRIPT_NAME and PATH_INFO environ entries. environ = environ.copy() - if environ.get(u'wsgi.version') == (u'u', 0): - # Python 2/WSGI u.0: all strings MUST be of type unicode - enc = environ[u'wsgi.url_encoding'] - environ[u'SCRIPT_NAME'] = sn.decode(enc) - environ[u'PATH_INFO'] = path[len(sn.rstrip("/")):].decode(enc) + if not py3k: + if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + # Python 2/WSGI u.0: all strings MUST be of type unicode + enc = environ[ntou('wsgi.url_encoding')] + environ[ntou('SCRIPT_NAME')] = sn.decode(enc) + environ[ntou('PATH_INFO')] = path[len(sn.rstrip("/")):].decode(enc) + else: + # Python 2/WSGI 1.x: all strings MUST be of type str + environ['SCRIPT_NAME'] = sn + environ['PATH_INFO'] = path[len(sn.rstrip("/")):] else: - # Python 2/WSGI 1.x: all strings MUST be of type str - environ['SCRIPT_NAME'] = sn - environ['PATH_INFO'] = path[len(sn.rstrip("/")):] + if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + # Python 3/WSGI u.0: all strings MUST be full unicode + environ['SCRIPT_NAME'] = sn + environ['PATH_INFO'] = path[len(sn.rstrip("/")):] + else: + # Python 3/WSGI 1.x: all strings MUST be ISO-8859-1 str + environ['SCRIPT_NAME'] = sn.encode('utf-8').decode('ISO-8859-1') + environ['PATH_INFO'] = path[len(sn.rstrip("/")):].encode('utf-8').decode('ISO-8859-1') return app(environ, start_response) - diff --git a/cherrypy/_cpwsgi.py b/cherrypy/_cpwsgi.py index aa4b7631..91cd044e 100644 --- a/cherrypy/_cpwsgi.py +++ b/cherrypy/_cpwsgi.py @@ -10,7 +10,7 @@ still be translatable to bytes via the Latin-1 encoding!" import sys as _sys import cherrypy as _cherrypy -from cherrypy._cpcompat import BytesIO +from cherrypy._cpcompat import BytesIO, bytestr, ntob, ntou, py3k, unicodestr from cherrypy import _cperror from cherrypy.lib import httputil @@ -19,11 +19,11 @@ def downgrade_wsgi_ux_to_1x(environ): """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ.""" env1x = {} - url_encoding = environ[u'wsgi.url_encoding'] - for k, v in environ.items(): - if k in [u'PATH_INFO', u'SCRIPT_NAME', u'QUERY_STRING']: + url_encoding = environ[ntou('wsgi.url_encoding')] + for k, v in list(environ.items()): + if k in [ntou('PATH_INFO'), ntou('SCRIPT_NAME'), ntou('QUERY_STRING')]: v = v.encode(url_encoding) - elif isinstance(v, unicode): + elif isinstance(v, unicodestr): v = v.encode('ISO-8859-1') env1x[k.encode('ISO-8859-1')] = v @@ -94,7 +94,8 @@ class InternalRedirector(object): environ = environ.copy() try: return self.nextapp(environ, start_response) - except _cherrypy.InternalRedirect, ir: + except _cherrypy.InternalRedirect: + ir = _sys.exc_info()[1] sn = environ.get('SCRIPT_NAME', '') path = environ.get('PATH_INFO', '') qs = environ.get('QUERY_STRING', '') @@ -152,8 +153,12 @@ class _TrappedResponse(object): self.started_response = True return self - def next(self): - return self.trap(self.iter_response.next) + if py3k: + def __next__(self): + return self.trap(next, self.iter_response) + else: + def next(self): + return self.trap(self.iter_response.next) def close(self): if hasattr(self.response, 'close'): @@ -173,6 +178,11 @@ class _TrappedResponse(object): if not _cherrypy.request.show_tracebacks: tb = "" s, h, b = _cperror.bare_error(tb) + if py3k: + # What fun. + s = s.decode('ISO-8859-1') + h = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in h] if self.started_response: # Empty our iterable (so future calls raise StopIteration) self.iter_response = iter([]) @@ -191,7 +201,7 @@ class _TrappedResponse(object): raise if self.started_response: - return "".join(b) + return ntob("").join(b) else: return b @@ -203,24 +213,52 @@ class AppResponse(object): """WSGI response iterable for CherryPy applications.""" def __init__(self, environ, start_response, cpapp): - if environ.get(u'wsgi.version') == (u'u', 0): - environ = downgrade_wsgi_ux_to_1x(environ) - self.environ = environ self.cpapp = cpapp try: + if not py3k: + if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + environ = downgrade_wsgi_ux_to_1x(environ) + self.environ = environ self.run() + + r = _cherrypy.serving.response + + outstatus = r.output_status + if not isinstance(outstatus, bytestr): + raise TypeError("response.output_status is not a byte string.") + + outheaders = [] + for k, v in r.header_list: + if not isinstance(k, bytestr): + raise TypeError("response.header_list key %r is not a byte string." % k) + if not isinstance(v, bytestr): + raise TypeError("response.header_list value %r is not a byte string." % v) + outheaders.append((k, v)) + + if py3k: + # According to PEP 3333, when using Python 3, the response status + # and headers must be bytes masquerading as unicode; that is, they + # must be of type "str" but are restricted to code points in the + # "latin-1" set. + outstatus = outstatus.decode('ISO-8859-1') + outheaders = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in outheaders] + + self.iter_response = iter(r.body) + self.write = start_response(outstatus, outheaders) except: self.close() raise - r = _cherrypy.serving.response - self.iter_response = iter(r.body) - self.write = start_response(r.output_status, r.header_list) def __iter__(self): return self - def next(self): - return self.iter_response.next() + if py3k: + def __next__(self): + return next(self.iter_response) + else: + def next(self): + return self.iter_response.next() def close(self): """Close and de-reference the current request and response. (Core)""" @@ -253,6 +291,29 @@ class AppResponse(object): path = httputil.urljoin(self.environ.get('SCRIPT_NAME', ''), self.environ.get('PATH_INFO', '')) qs = self.environ.get('QUERY_STRING', '') + + if py3k: + # This isn't perfect; if the given PATH_INFO is in the wrong encoding, + # it may fail to match the appropriate config section URI. But meh. + old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') + new_enc = self.cpapp.find_config(self.environ.get('PATH_INFO', ''), + "request.uri_encoding", 'utf-8') + if new_enc.lower() != old_enc.lower(): + # Even though the path and qs are unicode, the WSGI server is + # required by PEP 3333 to coerce them to ISO-8859-1 masquerading + # as unicode. So we have to encode back to bytes and then decode + # again using the "correct" encoding. + try: + u_path = path.encode(old_enc).decode(new_enc) + u_qs = qs.encode(old_enc).decode(new_enc) + except (UnicodeEncodeError, UnicodeDecodeError): + # Just pass them through without transcoding and hope. + pass + else: + # Only set transcoded values if they both succeed. + path = u_path + qs = u_qs + rproto = self.environ.get('SERVER_PROTOCOL') headers = self.translate_headers(self.environ) rfile = self.environ['wsgi.input'] diff --git a/cherrypy/_cpwsgi_server.py b/cherrypy/_cpwsgi_server.py index 49fd5a19..21af5134 100644 --- a/cherrypy/_cpwsgi_server.py +++ b/cherrypy/_cpwsgi_server.py @@ -37,8 +37,11 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): ) self.protocol = self.server_adapter.protocol_version self.nodelay = self.server_adapter.nodelay - - ssl_module = self.server_adapter.ssl_module or 'pyopenssl' + + if sys.version_info >= (3, 0): + ssl_module = self.server_adapter.ssl_module or 'builtin' + else: + ssl_module = self.server_adapter.ssl_module or 'pyopenssl' if self.server_adapter.ssl_context: adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) self.ssl_adapter = adapter_class( @@ -52,3 +55,9 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, self.server_adapter.ssl_certificate_chain) + + self.stats['Enabled'] = getattr(self.server_adapter, 'statistics', False) + + def error_log(self, msg="", level=20, traceback=False): + cherrypy.engine.log(msg, level, traceback) + diff --git a/cherrypy/lib/__init__.py b/cherrypy/lib/__init__.py index 611350c9..3fc0ec58 100644 --- a/cherrypy/lib/__init__.py +++ b/cherrypy/lib/__init__.py @@ -1,7 +1,7 @@ """CherryPy Library""" # Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 -from cherrypy.lib.reprconf import _Builder, unrepr, modules, attributes +from cherrypy.lib.reprconf import unrepr, modules, attributes class file_generator(object): """Yield the given input (a file object) in chunks (default 64k). (Core)""" diff --git a/cherrypy/lib/cpstats.py b/cherrypy/lib/cpstats.py index 79d5c3a9..9be947f2 100644 --- a/cherrypy/lib/cpstats.py +++ b/cherrypy/lib/cpstats.py @@ -320,20 +320,21 @@ class StatsTool(cherrypy.Tool): def record_stop(self, uriset=None, slow_queries=1.0, slow_queries_count=100, debug=False, **kwargs): """Record the end of a request.""" + resp = cherrypy.serving.response w = appstats['Requests'][threading._get_ident()] r = cherrypy.request.rfile.bytes_read w['Bytes Read'] = r appstats['Total Bytes Read'] += r - if cherrypy.response.stream: + if resp.stream: w['Bytes Written'] = 'chunked' else: - cl = int(cherrypy.response.headers.get('Content-Length', 0)) + cl = int(resp.headers.get('Content-Length', 0)) w['Bytes Written'] = cl appstats['Total Bytes Written'] += cl - w['Response Status'] = cherrypy.response.status + w['Response Status'] = getattr(resp, 'output_status', None) or resp.status w['End Time'] = time.time() p = w['End Time'] - w['Start Time'] diff --git a/cherrypy/lib/cptools.py b/cherrypy/lib/cptools.py index 3eedf97a..b426a3e7 100644 --- a/cherrypy/lib/cptools.py +++ b/cherrypy/lib/cptools.py @@ -116,7 +116,7 @@ def validate_since(): # Tool code # def allow(methods=None, debug=False): - """Raise 405 if request.method not in methods (default GET/HEAD). + """Raise 405 if request.method not in methods (default ['GET', 'HEAD']). The given methods are case-insensitive, and may be in any order. If only one method is allowed, you may supply a single string; @@ -151,6 +151,10 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', For running a CP server behind Apache, lighttpd, or other HTTP server. + For Apache and lighttpd, you should leave the 'local' argument at the + default value of 'X-Forwarded-Host'. For Squid, you probably want to set + tools.proxy.local = 'Origin'. + If you want the new request.base to include path info (not just the host), you must explicitly set base to the full base path, and ALSO set 'local' to '', so that the X-Forwarded-Host request header (which never includes @@ -581,9 +585,11 @@ class MonitoredHeaderMap(_httputil.HeaderMap): self.accessed_headers.add(key) return _httputil.HeaderMap.get(self, key, default=default) - def has_key(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.has_key(self, key) + if hasattr({}, 'has_key'): + # Python 2 + def has_key(self, key): + self.accessed_headers.add(key) + return _httputil.HeaderMap.has_key(self, key) def autovary(ignore=None, debug=False): diff --git a/cherrypy/lib/gctools.py b/cherrypy/lib/gctools.py new file mode 100644 index 00000000..183148b2 --- /dev/null +++ b/cherrypy/lib/gctools.py @@ -0,0 +1,214 @@ +import gc +import inspect +import os +import sys +import time + +try: + import objgraph +except ImportError: + objgraph = None + +import cherrypy +from cherrypy import _cprequest, _cpwsgi +from cherrypy.process.plugins import SimplePlugin + + +class ReferrerTree(object): + """An object which gathers all referrers of an object to a given depth.""" + + peek_length = 40 + + def __init__(self, ignore=None, maxdepth=2, maxparents=10): + self.ignore = ignore or [] + self.ignore.append(inspect.currentframe().f_back) + self.maxdepth = maxdepth + self.maxparents = maxparents + + def ascend(self, obj, depth=1): + """Return a nested list containing referrers of the given object.""" + depth += 1 + parents = [] + + # Gather all referrers in one step to minimize + # cascading references due to repr() logic. + refs = gc.get_referrers(obj) + self.ignore.append(refs) + if len(refs) > self.maxparents: + return [("[%s referrers]" % len(refs), [])] + + try: + ascendcode = self.ascend.__code__ + except AttributeError: + ascendcode = self.ascend.im_func.func_code + for parent in refs: + if inspect.isframe(parent) and parent.f_code is ascendcode: + continue + if parent in self.ignore: + continue + if depth <= self.maxdepth: + parents.append((parent, self.ascend(parent, depth))) + else: + parents.append((parent, [])) + + return parents + + def peek(self, s): + """Return s, restricted to a sane length.""" + if len(s) > (self.peek_length + 3): + half = self.peek_length // 2 + return s[:half] + '...' + s[-half:] + else: + return s + + def _format(self, obj, descend=True): + """Return a string representation of a single object.""" + if inspect.isframe(obj): + filename, lineno, func, context, index = inspect.getframeinfo(obj) + return "" % func + + if not descend: + return self.peek(repr(obj)) + + if isinstance(obj, dict): + return "{" + ", ".join(["%s: %s" % (self._format(k, descend=False), + self._format(v, descend=False)) + for k, v in obj.items()]) + "}" + elif isinstance(obj, list): + return "[" + ", ".join([self._format(item, descend=False) + for item in obj]) + "]" + elif isinstance(obj, tuple): + return "(" + ", ".join([self._format(item, descend=False) + for item in obj]) + ")" + + r = self.peek(repr(obj)) + if isinstance(obj, (str, int, float)): + return r + return "%s: %s" % (type(obj), r) + + def format(self, tree): + """Return a list of string reprs from a nested list of referrers.""" + output = [] + def ascend(branch, depth=1): + for parent, grandparents in branch: + output.append((" " * depth) + self._format(parent)) + if grandparents: + ascend(grandparents, depth + 1) + ascend(tree) + return output + + +def get_instances(cls): + return [x for x in gc.get_objects() if isinstance(x, cls)] + + +class RequestCounter(SimplePlugin): + + def start(self): + self.count = 0 + + def before_request(self): + self.count += 1 + + def after_request(self): + self.count -=1 +request_counter = RequestCounter(cherrypy.engine) +request_counter.subscribe() + + +def get_context(obj): + if isinstance(obj, _cprequest.Request): + return "path=%s;stage=%s" % (obj.path_info, obj.stage) + elif isinstance(obj, _cprequest.Response): + return "status=%s" % obj.status + elif isinstance(obj, _cpwsgi.AppResponse): + return "PATH_INFO=%s" % obj.environ.get('PATH_INFO', '') + elif hasattr(obj, "tb_lineno"): + return "tb_lineno=%s" % obj.tb_lineno + return "" + + +class GCRoot(object): + """A CherryPy page handler for testing reference leaks.""" + + classes = [(_cprequest.Request, 2, 2, + "Should be 1 in this request thread and 1 in the main thread."), + (_cprequest.Response, 2, 2, + "Should be 1 in this request thread and 1 in the main thread."), + (_cpwsgi.AppResponse, 1, 1, + "Should be 1 in this request thread only."), + ] + + def index(self): + return "Hello, world!" + index.exposed = True + + def stats(self): + output = ["Statistics:"] + + for trial in range(10): + if request_counter.count > 0: + break + time.sleep(0.5) + else: + output.append("\nNot all requests closed properly.") + + # gc_collect isn't perfectly synchronous, because it may + # break reference cycles that then take time to fully + # finalize. Call it thrice and hope for the best. + gc.collect() + gc.collect() + unreachable = gc.collect() + if unreachable: + if objgraph is not None: + final = objgraph.by_type('Nondestructible') + if final: + objgraph.show_backrefs(final, filename='finalizers.png') + + trash = {} + for x in gc.garbage: + trash[type(x)] = trash.get(type(x), 0) + 1 + if trash: + output.insert(0, "\n%s unreachable objects:" % unreachable) + trash = [(v, k) for k, v in trash.items()] + trash.sort() + for pair in trash: + output.append(" " + repr(pair)) + + # Check declared classes to verify uncollected instances. + # These don't have to be part of a cycle; they can be + # any objects that have unanticipated referrers that keep + # them from being collected. + allobjs = {} + for cls, minobj, maxobj, msg in self.classes: + allobjs[cls] = get_instances(cls) + + for cls, minobj, maxobj, msg in self.classes: + objs = allobjs[cls] + lenobj = len(objs) + if lenobj < minobj or lenobj > maxobj: + if minobj == maxobj: + output.append( + "\nExpected %s %r references, got %s." % + (minobj, cls, lenobj)) + else: + output.append( + "\nExpected %s to %s %r references, got %s." % + (minobj, maxobj, cls, lenobj)) + + for obj in objs: + if objgraph is not None: + ig = [id(objs), id(inspect.currentframe())] + fname = "graph_%s_%s.png" % (cls.__name__, id(obj)) + objgraph.show_backrefs( + obj, extra_ignore=ig, max_depth=4, too_many=20, + filename=fname, extra_info=get_context) + output.append("\nReferrers for %s (refcount=%s):" % + (repr(obj), sys.getrefcount(obj))) + t = ReferrerTree(ignore=[objs], maxdepth=3) + tree = t.ascend(obj) + output.extend(t.format(tree)) + + return "\n".join(output) + stats.exposed = True + diff --git a/cherrypy/lib/httputil.py b/cherrypy/lib/httputil.py index e0058751..5f77d547 100644 --- a/cherrypy/lib/httputil.py +++ b/cherrypy/lib/httputil.py @@ -9,7 +9,7 @@ to a public caning. from binascii import b2a_base64 from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou, reversed, sorted -from cherrypy._cpcompat import basestring, iteritems, unicodestr, unquote_qs +from cherrypy._cpcompat import basestring, bytestr, iteritems, nativestr, unicodestr, unquote_qs response_codes = BaseHTTPRequestHandler.responses.copy() # From http://www.cherrypy.org/ticket/361 @@ -38,6 +38,18 @@ def urljoin(*atoms): # Special-case the final url of "", and return "/" instead. return url or "/" +def urljoin_bytes(*atoms): + """Return the given path *atoms, joined into a single URL. + + This will correctly join a SCRIPT_NAME and PATH_INFO into the + original URL, even if either atom is blank. + """ + url = ntob("/").join([x for x in atoms if x]) + while ntob("//") in url: + url = url.replace(ntob("//"), ntob("/")) + # Special-case the final url of "", and return "/" instead. + return url or ntob("/") + def protocol_from_http(protocol_str): """Return a protocol tuple from the given 'HTTP/x.y' string.""" return int(protocol_str[5]), int(protocol_str[7]) @@ -105,9 +117,15 @@ class HeaderElement(object): def __cmp__(self, other): return cmp(self.value, other.value) + def __lt__(self, other): + return self.value < other.value + def __str__(self): p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] return "%s%s" % (self.value, "".join(p)) + + def __bytes__(self): + return ntob(self.__str__()) def __unicode__(self): return ntou(self.__str__()) @@ -181,6 +199,12 @@ class AcceptElement(HeaderElement): if diff == 0: diff = cmp(str(self), str(other)) return diff + + def __lt__(self, other): + if self.qvalue == other.qvalue: + return str(self) < str(other) + else: + return self.qvalue < other.qvalue def header_elements(fieldname, fieldvalue): @@ -199,8 +223,12 @@ def header_elements(fieldname, fieldvalue): return list(reversed(sorted(result))) def decode_TEXT(value): - r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> u"f\xfcr").""" - from email.Header import decode_header + r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" + try: + # Python 3 + from email.header import decode_header + except ImportError: + from email.Header import decode_header atoms = decode_header(value) decodedvalue = "" for atom, charset in atoms: @@ -253,6 +281,10 @@ def valid_status(status): return code, reason, message +# NOTE: the parse_qs functions that follow are modified version of those +# in the python3.0 source - we need to pass through an encoding to the unquote +# method, but the default parse_qs function doesn't allow us to. These do. + def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): """Parse a query given as a string argument. @@ -338,8 +370,9 @@ class CaseInsensitiveDict(dict): def get(self, key, default=None): return dict.get(self, str(key).title(), default) - def has_key(self, key): - return dict.has_key(self, str(key).title()) + if hasattr({}, 'has_key'): + def has_key(self, key): + return dict.has_key(self, str(key).title()) def update(self, E): for k in E.keys(): @@ -369,8 +402,12 @@ class CaseInsensitiveDict(dict): # A CRLF is allowed in the definition of TEXT only as part of a header # field continuation. It is expected that the folding LWS will be # replaced with a single SP before interpretation of the TEXT value." -header_translate_table = ''.join([chr(i) for i in xrange(256)]) -header_translate_deletechars = ''.join([chr(i) for i in xrange(32)]) + chr(127) +if nativestr == bytestr: + header_translate_table = ''.join([chr(i) for i in xrange(256)]) + header_translate_deletechars = ''.join([chr(i) for i in xrange(32)]) + chr(127) +else: + header_translate_table = None + header_translate_deletechars = bytes(range(32)) + bytes([127]) class HeaderMap(CaseInsensitiveDict): diff --git a/cherrypy/lib/jsontools.py b/cherrypy/lib/jsontools.py index 09042e45..20925791 100644 --- a/cherrypy/lib/jsontools.py +++ b/cherrypy/lib/jsontools.py @@ -82,6 +82,6 @@ def json_out(content_type='application/json', debug=False, handler=json_handler) request.handler = handler if content_type is not None: if debug: - cherrypy.log('Setting Content-Type to %s' % ct, 'TOOLS.JSON_OUT') + cherrypy.log('Setting Content-Type to %s' % content_type, 'TOOLS.JSON_OUT') cherrypy.serving.response.headers['Content-Type'] = content_type diff --git a/cherrypy/lib/reprconf.py b/cherrypy/lib/reprconf.py index e18949ee..ba8ff51e 100644 --- a/cherrypy/lib/reprconf.py +++ b/cherrypy/lib/reprconf.py @@ -28,6 +28,20 @@ try: set except NameError: from sets import Set as set + +try: + basestring +except NameError: + basestring = str + +try: + # Python 3 + import builtins +except ImportError: + # Python 2 + import __builtin__ as builtins + +import operator as _operator import sys def as_dict(config): @@ -195,10 +209,11 @@ class Parser(ConfigParser): if section not in result: result[section] = {} for option in self.options(section): - value = self.get(section, option, raw, vars) + value = self.get(section, option, raw=raw, vars=vars) try: value = unrepr(value) - except Exception, x: + except Exception: + x = sys.exc_info()[1] msg = ("Config error in section: %r, option: %r, " "value: %r. Config values must be valid Python." % (section, option, value)) @@ -216,7 +231,8 @@ class Parser(ConfigParser): # public domain "unrepr" implementation, found on the web and then improved. -class _Builder: + +class _Builder2: def build(self, o): m = getattr(self, 'build_' + o.__class__.__name__, None) @@ -225,6 +241,18 @@ class _Builder: repr(o.__class__.__name__)) return m(o) + def astnode(self, s): + """Return a Python2 ast Node compiled from a string.""" + try: + import compiler + except ImportError: + # Fallback to eval when compiler package is not available, + # e.g. IronPython 1.0. + return eval(s) + + p = compiler.parse("__tempvalue__ = " + s) + return p.getChildren()[1].getChildren()[0].getChildren()[1] + def build_Subscript(self, o): expr, flags, subs = o.getChildren() expr = self.build(expr) @@ -272,8 +300,7 @@ class _Builder: # See if the Name is in builtins. try: - import __builtin__ - return getattr(__builtin__, name) + return getattr(builtins, name) except AttributeError: pass @@ -282,6 +309,10 @@ class _Builder: def build_Add(self, o): left, right = map(self.build, o.getChildren()) return left + right + + def build_Mul(self, o): + left, right = map(self.build, o.getChildren()) + return left * right def build_Getattr(self, o): parent = self.build(o.expr) @@ -297,25 +328,128 @@ class _Builder: return self.build(o.getChildren()[0]) -def _astnode(s): - """Return a Python ast Node compiled from a string.""" - try: - import compiler - except ImportError: - # Fallback to eval when compiler package is not available, - # e.g. IronPython 1.0. - return eval(s) +class _Builder3: - p = compiler.parse("__tempvalue__ = " + s) - return p.getChildren()[1].getChildren()[0].getChildren()[1] + def build(self, o): + m = getattr(self, 'build_' + o.__class__.__name__, None) + if m is None: + raise TypeError("unrepr does not recognize %s" % + repr(o.__class__.__name__)) + return m(o) + def astnode(self, s): + """Return a Python3 ast Node compiled from a string.""" + try: + import ast + except ImportError: + # Fallback to eval when ast package is not available, + # e.g. IronPython 1.0. + return eval(s) + + p = ast.parse("__tempvalue__ = " + s) + return p.body[0].value + + def build_Subscript(self, o): + return self.build(o.value)[self.build(o.slice)] + + def build_Index(self, o): + return self.build(o.value) + + def build_Call(self, o): + callee = self.build(o.func) + + if o.args is None: + args = () + else: + args = tuple([self.build(a) for a in o.args]) + + if o.starargs is None: + starargs = () + else: + starargs = self.build(o.starargs) + + if o.kwargs is None: + kwargs = {} + else: + kwargs = self.build(o.kwargs) + + return callee(*(args + starargs), **kwargs) + + def build_List(self, o): + return list(map(self.build, o.elts)) + + def build_Str(self, o): + return o.s + + def build_Num(self, o): + return o.n + + def build_Dict(self, o): + return dict([(self.build(k), self.build(v)) + for k, v in zip(o.keys, o.values)]) + + def build_Tuple(self, o): + return tuple(self.build_List(o)) + + def build_Name(self, o): + name = o.id + if name == 'None': + return None + if name == 'True': + return True + if name == 'False': + return False + + # See if the Name is a package or module. If it is, import it. + try: + return modules(name) + except ImportError: + pass + + # See if the Name is in builtins. + try: + import builtins + return getattr(builtins, name) + except AttributeError: + pass + + raise TypeError("unrepr could not resolve the name %s" % repr(name)) + + def build_UnaryOp(self, o): + op, operand = map(self.build, [o.op, o.operand]) + return op(operand) + + def build_BinOp(self, o): + left, op, right = map(self.build, [o.left, o.op, o.right]) + return op(left, right) + + def build_Add(self, o): + return _operator.add + + def build_Mult(self, o): + return _operator.mul + + def build_USub(self, o): + return _operator.neg + + def build_Attribute(self, o): + parent = self.build(o.value) + return getattr(parent, o.attr) + + def build_NoneType(self, o): + return None + def unrepr(s): """Return a Python object compiled from a string.""" if not s: return s - obj = _astnode(s) - return _Builder().build(obj) + if sys.version_info < (3, 0): + b = _Builder2() + else: + b = _Builder3() + obj = b.astnode(s) + return b.build(obj) def modules(modulePath): diff --git a/cherrypy/lib/sessions.py b/cherrypy/lib/sessions.py index 42c28009..9763f120 100644 --- a/cherrypy/lib/sessions.py +++ b/cherrypy/lib/sessions.py @@ -93,7 +93,7 @@ import types from warnings import warn import cherrypy -from cherrypy._cpcompat import copyitems, pickle, random20 +from cherrypy._cpcompat import copyitems, pickle, random20, unicodestr from cherrypy.lib import httputil @@ -171,7 +171,15 @@ class Session(object): self.id = None self.missing = True self._regenerate() - + + def now(self): + """Generate the session specific concept of 'now'. + + Other session providers can override this to use alternative, + possibly timezone aware, versions of 'now'. + """ + return datetime.datetime.now() + def regenerate(self): """Replace the current session (with a new id).""" self.regenerated = True @@ -210,7 +218,7 @@ class Session(object): # accessed: no need to save it if self.loaded: t = datetime.timedelta(seconds = self.timeout * 60) - expiration_time = datetime.datetime.now() + t + expiration_time = self.now() + t if self.debug: cherrypy.log('Saving with expiry %s' % expiration_time, 'TOOLS.SESSIONS') @@ -225,7 +233,7 @@ class Session(object): """Copy stored session data into this session instance.""" data = self._load() # data is either None or a tuple (session_data, expiration_time) - if data is None or data[1] < datetime.datetime.now(): + if data is None or data[1] < self.now(): if self.debug: cherrypy.log('Expired session, flushing data', 'TOOLS.SESSIONS') self._data = {} @@ -277,10 +285,11 @@ class Session(object): if not self.loaded: self.load() return key in self._data - def has_key(self, key): - """D.has_key(k) -> True if D has a key k, else False.""" - if not self.loaded: self.load() - return key in self._data + if hasattr({}, 'has_key'): + def has_key(self, key): + """D.has_key(k) -> True if D has a key k, else False.""" + if not self.loaded: self.load() + return key in self._data def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" @@ -326,7 +335,7 @@ class RamSession(Session): def clean_up(self): """Clean up expired sessions.""" - now = datetime.datetime.now() + now = self.now() for id, (data, expiration_time) in copyitems(self.cache): if expiration_time <= now: try: @@ -337,6 +346,11 @@ class RamSession(Session): del self.locks[id] except KeyError: pass + + # added to remove obsolete lock objects + for id in list(self.locks): + if id not in self.cache: + self.locks.pop(id, None) def _exists(self): return self.id in self.cache @@ -467,7 +481,7 @@ class FileSession(Session): def clean_up(self): """Clean up expired sessions.""" - now = datetime.datetime.now() + now = self.now() # Iterate over all session files in self.storage_path for fname in os.listdir(self.storage_path): if (fname.startswith(self.SESSION_PREFIX) @@ -575,7 +589,7 @@ class PostgresqlSession(Session): def clean_up(self): """Clean up expired sessions.""" self.cursor.execute('delete from session where expiration_time < %s', - (datetime.datetime.now(),)) + (self.now(),)) class MemcachedSession(Session): @@ -602,6 +616,19 @@ class MemcachedSession(Session): cls.cache = memcache.Client(cls.servers) setup = classmethod(setup) + def _get_id(self): + return self._id + def _set_id(self, value): + # This encode() call is where we differ from the superclass. + # Memcache keys MUST be byte strings, not unicode. + if isinstance(value, unicodestr): + value = value.encode('utf-8') + + self._id = value + for o in self.id_observers: + o(value) + id = property(_get_id, _set_id, doc="The current session ID.") + def _exists(self): self.mc_lock.acquire() try: @@ -683,12 +710,12 @@ close.priority = 90 def init(storage_type='ram', path=None, path_header=None, name='session_id', timeout=60, domain=None, secure=False, clean_freq=5, - persistent=True, debug=False, **kwargs): + persistent=True, httponly=False, debug=False, **kwargs): """Initialize session object (using cookies). storage_type - One of 'ram', 'file', 'postgresql'. This will be used - to look up the corresponding class in cherrypy.lib.sessions + One of 'ram', 'file', 'postgresql', 'memcached'. This will be + used to look up the corresponding class in cherrypy.lib.sessions globals. For example, 'file' will use the FileSession class. path @@ -722,6 +749,10 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', and the cookie will be a "session cookie" which expires when the browser is closed. + httponly + If False (the default) the cookie 'httponly' value will not be set. + If True, the cookie 'httponly' value will be set (to 1). + Any additional kwargs will be bound to the new Session instance, and may be specific to the storage type. See the subclass of Session you're using for more information. @@ -772,11 +803,12 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', # and http://support.mozilla.com/en-US/kb/Cookies cookie_timeout = None set_response_cookie(path=path, path_header=path_header, name=name, - timeout=cookie_timeout, domain=domain, secure=secure) + timeout=cookie_timeout, domain=domain, secure=secure, + httponly=httponly) def set_response_cookie(path=None, path_header=None, name='session_id', - timeout=60, domain=None, secure=False): + timeout=60, domain=None, secure=False, httponly=False): """Set a response cookie for the client. path @@ -801,6 +833,10 @@ def set_response_cookie(path=None, path_header=None, name='session_id', if False (the default) the cookie 'secure' value will not be set. If True, the cookie 'secure' value will be set (to 1). + httponly + If False (the default) the cookie 'httponly' value will not be set. + If True, the cookie 'httponly' value will be set (to 1). + """ # Set response cookie cookie = cherrypy.serving.response.cookie @@ -820,7 +856,10 @@ def set_response_cookie(path=None, path_header=None, name='session_id', cookie[name]['domain'] = domain if secure: cookie[name]['secure'] = 1 - + if httponly: + if not cookie[name].isReservedKey('httponly'): + raise ValueError("The httponly cookie token is not supported.") + cookie[name]['httponly'] = 1 def expire(): """Expire the current session cookie.""" diff --git a/cherrypy/lib/static.py b/cherrypy/lib/static.py index cb9a68cb..2d142307 100644 --- a/cherrypy/lib/static.py +++ b/cherrypy/lib/static.py @@ -1,3 +1,7 @@ +try: + from io import UnsupportedOperation +except ImportError: + UnsupportedOperation = object() import logging import mimetypes mimetypes.init() @@ -115,6 +119,8 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, if debug: cherrypy.log('os has no fstat attribute', 'TOOLS.STATIC') content_length = None + except UnsupportedOperation: + content_length = None else: # Set the Last-Modified response header, so that # modified-since validation code can work. @@ -174,7 +180,12 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): else: # Return a multipart/byteranges response. response.status = "206 Partial Content" - from mimetools import choose_boundary + try: + # Python 3 + from email.generator import _make_boundary as choose_boundary + except ImportError: + # Python 2 + from mimetools import choose_boundary boundary = choose_boundary() ct = "multipart/byteranges; boundary=%s" % boundary response.headers['Content-Type'] = ct diff --git a/cherrypy/lib/xmlrpc.py b/cherrypy/lib/xmlrpcutil.py similarity index 61% rename from cherrypy/lib/xmlrpc.py rename to cherrypy/lib/xmlrpcutil.py index 8a5ef546..9a44464b 100644 --- a/cherrypy/lib/xmlrpc.py +++ b/cherrypy/lib/xmlrpcutil.py @@ -1,13 +1,19 @@ import sys import cherrypy +from cherrypy._cpcompat import ntob +def get_xmlrpclib(): + try: + import xmlrpc.client as x + except ImportError: + import xmlrpclib as x + return x def process_body(): """Return (params, method) from request body.""" try: - import xmlrpclib - return xmlrpclib.loads(cherrypy.request.body.read()) + return get_xmlrpclib().loads(cherrypy.request.body.read()) except Exception: return ('ERROR PARAMS', ), 'ERRORMETHOD' @@ -29,21 +35,21 @@ def _set_response(body): # as a "Protocol Error", we'll just return 200 every time. response = cherrypy.response response.status = '200 OK' - response.body = body + response.body = ntob(body, 'utf-8') response.headers['Content-Type'] = 'text/xml' response.headers['Content-Length'] = len(body) def respond(body, encoding='utf-8', allow_none=0): - from xmlrpclib import Fault, dumps - if not isinstance(body, Fault): + xmlrpclib = get_xmlrpclib() + if not isinstance(body, xmlrpclib.Fault): body = (body,) - _set_response(dumps(body, methodresponse=1, - encoding=encoding, - allow_none=allow_none)) + _set_response(xmlrpclib.dumps(body, methodresponse=1, + encoding=encoding, + allow_none=allow_none)) def on_error(*args, **kwargs): body = str(sys.exc_info()[1]) - from xmlrpclib import Fault, dumps - _set_response(dumps(Fault(1, body))) + xmlrpclib = get_xmlrpclib() + _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) diff --git a/cherrypy/process/plugins.py b/cherrypy/process/plugins.py index 488958eb..ba618a0b 100644 --- a/cherrypy/process/plugins.py +++ b/cherrypy/process/plugins.py @@ -453,13 +453,14 @@ class BackgroundTask(threading.Thread): it won't delay stopping the whole process. """ - def __init__(self, interval, function, args=[], kwargs={}): + def __init__(self, interval, function, args=[], kwargs={}, bus=None): threading.Thread.__init__(self) self.interval = interval self.function = function self.args = args self.kwargs = kwargs self.running = False + self.bus = bus def cancel(self): self.running = False @@ -473,8 +474,9 @@ class BackgroundTask(threading.Thread): try: self.function(*self.args, **self.kwargs) except Exception: - self.bus.log("Error in background task thread function %r." % - self.function, level=40, traceback=True) + if self.bus: + self.bus.log("Error in background task thread function %r." + % self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise @@ -506,8 +508,8 @@ class Monitor(SimplePlugin): if self.frequency > 0: threadname = self.name or self.__class__.__name__ if self.thread is None: - self.thread = BackgroundTask(self.frequency, self.callback) - self.thread.bus = self.bus + self.thread = BackgroundTask(self.frequency, self.callback, + bus = self.bus) self.thread.setName(threadname) self.thread.start() self.bus.log("Started monitor thread %r." % threadname) diff --git a/cherrypy/process/servers.py b/cherrypy/process/servers.py index 272e8436..fa714d65 100644 --- a/cherrypy/process/servers.py +++ b/cherrypy/process/servers.py @@ -385,34 +385,43 @@ def check_port(host, port, timeout=1.0): if s: s.close() -def wait_for_free_port(host, port): + +# Feel free to increase these defaults on slow systems: +free_port_timeout = 0.1 +occupied_port_timeout = 1.0 + +def wait_for_free_port(host, port, timeout=None): """Wait for the specified port to become free (drop requests).""" if not host: raise ValueError("Host values of '' or None are not allowed.") + if timeout is None: + timeout = free_port_timeout for trial in range(50): try: # we are expecting a free port, so reduce the timeout - check_port(host, port, timeout=0.1) + check_port(host, port, timeout=timeout) except IOError: # Give the old server thread time to free the port. - time.sleep(0.1) + time.sleep(timeout) else: return raise IOError("Port %r not free on %r" % (port, host)) -def wait_for_occupied_port(host, port): +def wait_for_occupied_port(host, port, timeout=None): """Wait for the specified port to become active (receive requests).""" if not host: raise ValueError("Host values of '' or None are not allowed.") + if timeout is None: + timeout = occupied_port_timeout for trial in range(50): try: - check_port(host, port) + check_port(host, port, timeout=timeout) except IOError: return else: - time.sleep(.1) + time.sleep(timeout) raise IOError("Port %r not bound on %r" % (port, host)) diff --git a/cherrypy/process/wspbus.py b/cherrypy/process/wspbus.py index 46cd75a2..6ef768dc 100644 --- a/cherrypy/process/wspbus.py +++ b/cherrypy/process/wspbus.py @@ -90,11 +90,11 @@ class ChannelFailures(Exception): def handle_exception(self): """Append the current exception to self.""" - self._exceptions.append(sys.exc_info()) + self._exceptions.append(sys.exc_info()[1]) def get_instances(self): """Return a list of seen exception instances.""" - return [instance for cls, instance, traceback in self._exceptions] + return self._exceptions[:] def __str__(self): exception_strings = map(repr, self.get_instances()) @@ -102,8 +102,9 @@ class ChannelFailures(Exception): __repr__ = __str__ - def __nonzero__(self): + def __bool__(self): return bool(self._exceptions) + __nonzero__ = __bool__ # Use a flag to indicate the state of the bus. class _StateEnum(object): @@ -124,6 +125,17 @@ states.STOPPING = states.State() states.EXITING = states.State() +try: + import fcntl +except ImportError: + max_files = 0 +else: + try: + max_files = os.sysconf('SC_OPEN_MAX') + except AttributeError: + max_files = 1024 + + class Bus(object): """Process state-machine and messenger for HTTP site deployment. @@ -137,6 +149,7 @@ class Bus(object): states = states state = states.STOPPED execv = False + max_cloexec_files = max_files def __init__(self): self.execv = False @@ -173,13 +186,19 @@ class Bus(object): items = [(self._priorities[(channel, listener)], listener) for listener in self.listeners[channel]] - items.sort() + try: + items.sort(key=lambda item: item[0]) + except TypeError: + # Python 2.3 had no 'key' arg, but that doesn't matter + # since it could sort dissimilar types just fine. + items.sort() for priority, listener in items: try: output.append(listener(*args, **kwargs)) except KeyboardInterrupt: raise - except SystemExit, e: + except SystemExit: + e = sys.exc_info()[1] # If we have previous errors ensure the exit code is non-zero if exc and e.code == 0: e.code = 1 @@ -221,13 +240,14 @@ class Bus(object): except: self.log("Shutting down due to error in start listener:", level=40, traceback=True) - e_info = sys.exc_info() + e_info = sys.exc_info()[1] try: self.exit() except: # Any stop/exit errors will be logged inside publish(). pass - raise e_info[0], e_info[1], e_info[2] + # Re-raise the original error + raise e_info def exit(self): """Stop all services and prepare to exit the process.""" @@ -354,8 +374,28 @@ class Bus(object): args = ['"%s"' % arg for arg in args] os.chdir(_startup_cwd) + if self.max_cloexec_files: + self._set_cloexec() os.execv(sys.executable, args) + def _set_cloexec(self): + """Set the CLOEXEC flag on all open files (except stdin/out/err). + + If self.max_cloexec_files is an integer (the default), then on + platforms which support it, it represents the max open files setting + for the operating system. This function will be called just before + the process is restarted via os.execv() to prevent open files + from persisting into the new process. + + Set self.max_cloexec_files to 0 to disable this behavior. + """ + for fd in range(3, self.max_cloexec_files): # skip stdin/out/err + try: + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + except IOError: + continue + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + def stop(self): """Stop all services.""" self.state = states.STOPPING @@ -386,8 +426,7 @@ class Bus(object): def log(self, msg="", level=20, traceback=False): """Log the given message. Append the last traceback if requested.""" if traceback: - exc = sys.exc_info() - msg += "\n" + "".join(_traceback.format_exception(*exc)) + msg += "\n" + "".join(_traceback.format_exception(*sys.exc_info())) self.publish('log', msg, level) bus = Bus() diff --git a/cherrypy/test/__init__.py b/cherrypy/test/__init__.py deleted file mode 100644 index e4c400d6..00000000 --- a/cherrypy/test/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Regression test suite for CherryPy. - -Run 'nosetests -s test/' to exercise all tests. - -The '-s' flag instructs nose to output stdout messages, wihch is crucial to -the 'interactive' mode of webtest.py. If you run these tests without the '-s' -flag, don't be surprised if the test seems to hang: it's waiting for your -interactive input. -""" - -import sys -def newexit(): - raise SystemExit('Exit called') - -def setup(): - # We want to monkey patch sys.exit so that we can get some - # information about where exit is being called. - newexit._old = sys.exit - sys.exit = newexit - -def teardown(): - try: - sys.exit = sys.exit._old - except AttributeError: - sys.exit = sys._exit diff --git a/cherrypy/test/_test_decorators.py b/cherrypy/test/_test_decorators.py deleted file mode 100644 index 5bcbc1e6..00000000 --- a/cherrypy/test/_test_decorators.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Test module for the @-decorator syntax, which is version-specific""" - -from cherrypy import expose, tools -from cherrypy._cpcompat import ntob - - -class ExposeExamples(object): - - @expose - def no_call(self): - return "Mr E. R. Bradshaw" - - @expose() - def call_empty(self): - return "Mrs. B.J. Smegma" - - @expose("call_alias") - def nesbitt(self): - return "Mr Nesbitt" - - @expose(["alias1", "alias2"]) - def andrews(self): - return "Mr Ken Andrews" - - @expose(alias="alias3") - def watson(self): - return "Mr. and Mrs. Watson" - - -class ToolExamples(object): - - @expose - @tools.response_headers(headers=[('Content-Type', 'application/data')]) - def blah(self): - yield ntob("blah") - # This is here to demonstrate that _cp_config = {...} overwrites - # the _cp_config attribute added by the Tool decorator. You have - # to write _cp_config[k] = v or _cp_config.update(...) instead. - blah._cp_config['response.stream'] = True - - diff --git a/cherrypy/test/_test_states_demo.py b/cherrypy/test/_test_states_demo.py deleted file mode 100644 index 3f8f196c..00000000 --- a/cherrypy/test/_test_states_demo.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import sys -import time -starttime = time.time() - -import cherrypy - - -class Root: - - def index(self): - return "Hello World" - index.exposed = True - - def mtimes(self): - return repr(cherrypy.engine.publish("Autoreloader", "mtimes")) - mtimes.exposed = True - - def pid(self): - return str(os.getpid()) - pid.exposed = True - - def start(self): - return repr(starttime) - start.exposed = True - - def exit(self): - # This handler might be called before the engine is STARTED if an - # HTTP worker thread handles it before the HTTP server returns - # control to engine.start. We avoid that race condition here - # by waiting for the Bus to be STARTED. - cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) - cherrypy.engine.exit() - exit.exposed = True - - -def unsub_sig(): - cherrypy.log("unsubsig: %s" % cherrypy.config.get('unsubsig', False)) - if cherrypy.config.get('unsubsig', False): - cherrypy.log("Unsubscribing the default cherrypy signal handler") - cherrypy.engine.signal_handler.unsubscribe() - try: - from signal import signal, SIGTERM - except ImportError: - pass - else: - def old_term_handler(signum=None, frame=None): - cherrypy.log("I am an old SIGTERM handler.") - sys.exit(0) - cherrypy.log("Subscribing the new one.") - signal(SIGTERM, old_term_handler) -cherrypy.engine.subscribe('start', unsub_sig, priority=100) - - -def starterror(): - if cherrypy.config.get('starterror', False): - zerodiv = 1 / 0 -cherrypy.engine.subscribe('start', starterror, priority=6) - -def log_test_case_name(): - if cherrypy.config.get('test_case_name', False): - cherrypy.log("STARTED FROM: %s" % cherrypy.config.get('test_case_name')) -cherrypy.engine.subscribe('start', log_test_case_name, priority=6) - - -cherrypy.tree.mount(Root(), '/', {'/': {}}) diff --git a/cherrypy/test/benchmark.py b/cherrypy/test/benchmark.py deleted file mode 100644 index 90536a56..00000000 --- a/cherrypy/test/benchmark.py +++ /dev/null @@ -1,409 +0,0 @@ -"""CherryPy Benchmark Tool - - Usage: - benchmark.py --null --notests --help --cpmodpy --modpython --ab=path --apache=path - - --null: use a null Request object (to bench the HTTP server only) - --notests: start the server but do not run the tests; this allows - you to check the tested pages with a browser - --help: show this help message - --cpmodpy: run tests via apache on 8080 (with the builtin _cpmodpy) - --modpython: run tests via apache on 8080 (with modpython_gateway) - --ab=path: Use the ab script/executable at 'path' (see below) - --apache=path: Use the apache script/exe at 'path' (see below) - - To run the benchmarks, the Apache Benchmark tool "ab" must either be on - your system path, or specified via the --ab=path option. - - To run the modpython tests, the "apache" executable or script must be - on your system path, or provided via the --apache=path option. On some - platforms, "apache" may be called "apachectl" or "apache2ctl"--create - a symlink to them if needed. -""" - -import getopt -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - -import re -import sys -import time -import traceback - -import cherrypy -from cherrypy._cpcompat import ntob -from cherrypy import _cperror, _cpmodpy -from cherrypy.lib import httputil - - -AB_PATH = "" -APACHE_PATH = "apache" -SCRIPT_NAME = "/cpbench/users/rdelon/apps/blog" - -__all__ = ['ABSession', 'Root', 'print_report', - 'run_standard_benchmarks', 'safe_threads', - 'size_report', 'startup', 'thread_report', - ] - -size_cache = {} - -class Root: - - def index(self): - return """ - - CherryPy Benchmark - - - - -""" - index.exposed = True - - def hello(self): - return "Hello, world\r\n" - hello.exposed = True - - def sizer(self, size): - resp = size_cache.get(size, None) - if resp is None: - size_cache[size] = resp = "X" * int(size) - return resp - sizer.exposed = True - - -cherrypy.config.update({ - 'log.error.file': '', - 'environment': 'production', - 'server.socket_host': '127.0.0.1', - 'server.socket_port': 8080, - 'server.max_request_header_size': 0, - 'server.max_request_body_size': 0, - 'engine.deadlock_poll_freq': 0, - }) - -# Cheat mode on ;) -del cherrypy.config['tools.log_tracebacks.on'] -del cherrypy.config['tools.log_headers.on'] -del cherrypy.config['tools.trailing_slash.on'] - -appconf = { - '/static': { - 'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.root': curdir, - }, - } -app = cherrypy.tree.mount(Root(), SCRIPT_NAME, appconf) - - -class NullRequest: - """A null HTTP request class, returning 200 and an empty body.""" - - def __init__(self, local, remote, scheme="http"): - pass - - def close(self): - pass - - def run(self, method, path, query_string, protocol, headers, rfile): - cherrypy.response.status = "200 OK" - cherrypy.response.header_list = [("Content-Type", 'text/html'), - ("Server", "Null CherryPy"), - ("Date", httputil.HTTPDate()), - ("Content-Length", "0"), - ] - cherrypy.response.body = [""] - return cherrypy.response - - -class NullResponse: - pass - - -class ABSession: - """A session of 'ab', the Apache HTTP server benchmarking tool. - -Example output from ab: - -This is ApacheBench, Version 2.0.40-dev <$Revision: 1.121.2.1 $> apache-2.0 -Copyright (c) 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ -Copyright (c) 1998-2002 The Apache Software Foundation, http://www.apache.org/ - -Benchmarking 127.0.0.1 (be patient) -Completed 100 requests -Completed 200 requests -Completed 300 requests -Completed 400 requests -Completed 500 requests -Completed 600 requests -Completed 700 requests -Completed 800 requests -Completed 900 requests - - -Server Software: CherryPy/3.1beta -Server Hostname: 127.0.0.1 -Server Port: 8080 - -Document Path: /static/index.html -Document Length: 14 bytes - -Concurrency Level: 10 -Time taken for tests: 9.643867 seconds -Complete requests: 1000 -Failed requests: 0 -Write errors: 0 -Total transferred: 189000 bytes -HTML transferred: 14000 bytes -Requests per second: 103.69 [#/sec] (mean) -Time per request: 96.439 [ms] (mean) -Time per request: 9.644 [ms] (mean, across all concurrent requests) -Transfer rate: 19.08 [Kbytes/sec] received - -Connection Times (ms) - min mean[+/-sd] median max -Connect: 0 0 2.9 0 10 -Processing: 20 94 7.3 90 130 -Waiting: 0 43 28.1 40 100 -Total: 20 95 7.3 100 130 - -Percentage of the requests served within a certain time (ms) - 50% 100 - 66% 100 - 75% 100 - 80% 100 - 90% 100 - 95% 100 - 98% 100 - 99% 110 - 100% 130 (longest request) -Finished 1000 requests -""" - - parse_patterns = [('complete_requests', 'Completed', - ntob(r'^Complete requests:\s*(\d+)')), - ('failed_requests', 'Failed', - ntob(r'^Failed requests:\s*(\d+)')), - ('requests_per_second', 'req/sec', - ntob(r'^Requests per second:\s*([0-9.]+)')), - ('time_per_request_concurrent', 'msec/req', - ntob(r'^Time per request:\s*([0-9.]+).*concurrent requests\)$')), - ('transfer_rate', 'KB/sec', - ntob(r'^Transfer rate:\s*([0-9.]+)')), - ] - - def __init__(self, path=SCRIPT_NAME + "/hello", requests=1000, concurrency=10): - self.path = path - self.requests = requests - self.concurrency = concurrency - - def args(self): - port = cherrypy.server.socket_port - assert self.concurrency > 0 - assert self.requests > 0 - # Don't use "localhost". - # Cf http://mail.python.org/pipermail/python-win32/2008-March/007050.html - return ("-k -n %s -c %s http://127.0.0.1:%s%s" % - (self.requests, self.concurrency, port, self.path)) - - def run(self): - # Parse output of ab, setting attributes on self - try: - self.output = _cpmodpy.read_process(AB_PATH or "ab", self.args()) - except: - print(_cperror.format_exc()) - raise - - for attr, name, pattern in self.parse_patterns: - val = re.search(pattern, self.output, re.MULTILINE) - if val: - val = val.group(1) - setattr(self, attr, val) - else: - setattr(self, attr, None) - - -safe_threads = (25, 50, 100, 200, 400) -if sys.platform in ("win32",): - # For some reason, ab crashes with > 50 threads on my Win2k laptop. - safe_threads = (10, 20, 30, 40, 50) - - -def thread_report(path=SCRIPT_NAME + "/hello", concurrency=safe_threads): - sess = ABSession(path) - attrs, names, patterns = list(zip(*sess.parse_patterns)) - avg = dict.fromkeys(attrs, 0.0) - - yield ('threads',) + names - for c in concurrency: - sess.concurrency = c - sess.run() - row = [c] - for attr in attrs: - val = getattr(sess, attr) - if val is None: - print(sess.output) - row = None - break - val = float(val) - avg[attr] += float(val) - row.append(val) - if row: - yield row - - # Add a row of averages. - yield ["Average"] + [str(avg[attr] / len(concurrency)) for attr in attrs] - -def size_report(sizes=(10, 100, 1000, 10000, 100000, 100000000), - concurrency=50): - sess = ABSession(concurrency=concurrency) - attrs, names, patterns = list(zip(*sess.parse_patterns)) - yield ('bytes',) + names - for sz in sizes: - sess.path = "%s/sizer?size=%s" % (SCRIPT_NAME, sz) - sess.run() - yield [sz] + [getattr(sess, attr) for attr in attrs] - -def print_report(rows): - for row in rows: - print("") - for i, val in enumerate(row): - sys.stdout.write(str(val).rjust(10) + " | ") - print("") - - -def run_standard_benchmarks(): - print("") - print("Client Thread Report (1000 requests, 14 byte response body, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(thread_report()) - - print("") - print("Client Thread Report (1000 requests, 14 bytes via staticdir, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(thread_report("%s/static/index.html" % SCRIPT_NAME)) - - print("") - print("Size Report (1000 requests, 50 client threads, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(size_report()) - - -# modpython and other WSGI # - -def startup_modpython(req=None): - """Start the CherryPy app server in 'serverless' mode (for modpython/WSGI).""" - if cherrypy.engine.state == cherrypy._cpengine.STOPPED: - if req: - if "nullreq" in req.get_options(): - cherrypy.engine.request_class = NullRequest - cherrypy.engine.response_class = NullResponse - ab_opt = req.get_options().get("ab", "") - if ab_opt: - global AB_PATH - AB_PATH = ab_opt - cherrypy.engine.start() - if cherrypy.engine.state == cherrypy._cpengine.STARTING: - cherrypy.engine.wait() - return 0 # apache.OK - - -def run_modpython(use_wsgi=False): - print("Starting mod_python...") - pyopts = [] - - # Pass the null and ab=path options through Apache - if "--null" in opts: - pyopts.append(("nullreq", "")) - - if "--ab" in opts: - pyopts.append(("ab", opts["--ab"])) - - s = _cpmodpy.ModPythonServer - if use_wsgi: - pyopts.append(("wsgi.application", "cherrypy::tree")) - pyopts.append(("wsgi.startup", "cherrypy.test.benchmark::startup_modpython")) - handler = "modpython_gateway::handler" - s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH, handler=handler) - else: - pyopts.append(("cherrypy.setup", "cherrypy.test.benchmark::startup_modpython")) - s = s(port=8080, opts=pyopts, apache_path=APACHE_PATH) - - try: - s.start() - run() - finally: - s.stop() - - - -if __name__ == '__main__': - longopts = ['cpmodpy', 'modpython', 'null', 'notests', - 'help', 'ab=', 'apache='] - try: - switches, args = getopt.getopt(sys.argv[1:], "", longopts) - opts = dict(switches) - except getopt.GetoptError: - print(__doc__) - sys.exit(2) - - if "--help" in opts: - print(__doc__) - sys.exit(0) - - if "--ab" in opts: - AB_PATH = opts['--ab'] - - if "--notests" in opts: - # Return without stopping the server, so that the pages - # can be tested from a standard web browser. - def run(): - port = cherrypy.server.socket_port - print("You may now open http://127.0.0.1:%s%s/" % - (port, SCRIPT_NAME)) - - if "--null" in opts: - print("Using null Request object") - else: - def run(): - end = time.time() - start - print("Started in %s seconds" % end) - if "--null" in opts: - print("\nUsing null Request object") - try: - try: - run_standard_benchmarks() - except: - print(_cperror.format_exc()) - raise - finally: - cherrypy.engine.exit() - - print("Starting CherryPy app server...") - - class NullWriter(object): - """Suppresses the printing of socket errors.""" - def write(self, data): - pass - sys.stderr = NullWriter() - - start = time.time() - - if "--cpmodpy" in opts: - run_modpython() - elif "--modpython" in opts: - run_modpython(use_wsgi=True) - else: - if "--null" in opts: - cherrypy.server.request_class = NullRequest - cherrypy.server.response_class = NullResponse - - cherrypy.engine.start_with_callback(run) - cherrypy.engine.block() diff --git a/cherrypy/test/checkerdemo.py b/cherrypy/test/checkerdemo.py deleted file mode 100644 index 32a7dee2..00000000 --- a/cherrypy/test/checkerdemo.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Demonstration app for cherrypy.checker. - -This application is intentionally broken and badly designed. -To demonstrate the output of the CherryPy Checker, simply execute -this module. -""" - -import os -import cherrypy -thisdir = os.path.dirname(os.path.abspath(__file__)) - -class Root: - pass - -if __name__ == '__main__': - conf = {'/base': {'tools.staticdir.root': thisdir, - # Obsolete key. - 'throw_errors': True, - }, - # This entry should be OK. - '/base/static': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static'}, - # Warn on missing folder. - '/base/js': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'js'}, - # Warn on dir with an abs path even though we provide root. - '/base/static2': {'tools.staticdir.on': True, - 'tools.staticdir.dir': '/static'}, - # Warn on dir with a relative path with no root. - '/static3': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static'}, - # Warn on unknown namespace - '/unknown': {'toobles.gzip.on': True}, - # Warn special on cherrypy..* - '/cpknown': {'cherrypy.tools.encode.on': True}, - # Warn on mismatched types - '/conftype': {'request.show_tracebacks': 14}, - # Warn on unknown tool. - '/web': {'tools.unknown.on': True}, - # Warn on server.* in app config. - '/app1': {'server.socket_host': '0.0.0.0'}, - # Warn on 'localhost' - 'global': {'server.socket_host': 'localhost'}, - # Warn on '[name]' - '[/extra_brackets]': {}, - } - cherrypy.quickstart(Root(), config=conf) diff --git a/cherrypy/test/fastcgi.conf b/cherrypy/test/fastcgi.conf deleted file mode 100644 index e5c5163c..00000000 --- a/cherrypy/test/fastcgi.conf +++ /dev/null @@ -1,18 +0,0 @@ - -# Apache2 server conf file for testing CherryPy with mod_fastcgi. -# fumanchu: I had to hard-code paths due to crazy Debian layouts :( -ServerRoot /usr/lib/apache2 -User #1000 -ErrorLog /usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test/mod_fastcgi.error.log - -DocumentRoot "/usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test" -ServerName 127.0.0.1 -Listen 8080 -LoadModule fastcgi_module modules/mod_fastcgi.so -LoadModule rewrite_module modules/mod_rewrite.so - -Options +ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "/usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test/fastcgi.pyc" -host 127.0.0.1:4000 diff --git a/cherrypy/test/fcgi.conf b/cherrypy/test/fcgi.conf deleted file mode 100644 index 8cf24b64..00000000 --- a/cherrypy/test/fcgi.conf +++ /dev/null @@ -1,14 +0,0 @@ - -# Apache2 server conf file for testing CherryPy with mod_fcgid. - -DocumentRoot "C:\Python25\Lib\site-packages\cherrypy\test" -ServerName 127.0.0.1 -Listen 8080 -LoadModule fastcgi_module modules/mod_fastcgi.dll -LoadModule rewrite_module modules/mod_rewrite.so - -Options ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "C:\\Python25\\Lib\\site-packages\\cherrypy\\test\\fastcgi.pyc" -host 127.0.0.1:4000 diff --git a/cherrypy/test/helper.py b/cherrypy/test/helper.py deleted file mode 100644 index ff9e06cf..00000000 --- a/cherrypy/test/helper.py +++ /dev/null @@ -1,476 +0,0 @@ -"""A library of helper functions for the CherryPy test suite.""" - -import datetime -import logging -log = logging.getLogger(__name__) -import os -thisdir = os.path.abspath(os.path.dirname(__file__)) -serverpem = os.path.join(os.getcwd(), thisdir, 'test.pem') - -import re -import sys -import time -import warnings - -import cherrypy -from cherrypy._cpcompat import basestring, copyitems, HTTPSConnection, ntob -from cherrypy.lib import httputil -from cherrypy.lib.reprconf import unrepr -from cherrypy.test import webtest - -import nose - -_testconfig = None - -def get_tst_config(overconf = {}): - global _testconfig - if _testconfig is None: - conf = { - 'scheme': 'http', - 'protocol': "HTTP/1.1", - 'port': 8080, - 'host': '127.0.0.1', - 'validate': False, - 'conquer': False, - 'server': 'wsgi', - } - try: - import testconfig - _conf = testconfig.config.get('supervisor', None) - if _conf is not None: - for k, v in _conf.items(): - if isinstance(v, basestring): - _conf[k] = unrepr(v) - conf.update(_conf) - except ImportError: - pass - _testconfig = conf - conf = _testconfig.copy() - conf.update(overconf) - - return conf - -class Supervisor(object): - """Base class for modeling and controlling servers during testing.""" - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - if k == 'port': - setattr(self, k, int(v)) - setattr(self, k, v) - - -log_to_stderr = lambda msg, level: sys.stderr.write(msg + os.linesep) - -class LocalSupervisor(Supervisor): - """Base class for modeling/controlling servers which run in the same process. - - When the server side runs in a different process, start/stop can dump all - state between each test module easily. When the server side runs in the - same process as the client, however, we have to do a bit more work to ensure - config and mounted apps are reset between tests. - """ - - using_apache = False - using_wsgi = False - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - - cherrypy.server.httpserver = self.httpserver_class - - engine = cherrypy.engine - if hasattr(engine, "signal_handler"): - engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.subscribe() - #engine.subscribe('log', log_to_stderr) - - def start(self, modulename=None): - """Load and start the HTTP server.""" - if modulename: - # Unhook httpserver so cherrypy.server.start() creates a new - # one (with config from setup_server, if declared). - cherrypy.server.httpserver = None - - cherrypy.engine.start() - - self.sync_apps() - - def sync_apps(self): - """Tell the server about any apps which the setup functions mounted.""" - pass - - def stop(self): - td = getattr(self, 'teardown', None) - if td: - td() - - cherrypy.engine.exit() - - for name, server in copyitems(getattr(cherrypy, 'servers', {})): - server.unsubscribe() - del cherrypy.servers[name] - - -class NativeServerSupervisor(LocalSupervisor): - """Server supervisor for the builtin HTTP server.""" - - httpserver_class = "cherrypy._cpnative_server.CPHTTPServer" - using_apache = False - using_wsgi = False - - def __str__(self): - return "Builtin HTTP Server on %s:%s" % (self.host, self.port) - - -class LocalWSGISupervisor(LocalSupervisor): - """Server supervisor for the builtin WSGI server.""" - - httpserver_class = "cherrypy._cpwsgi_server.CPWSGIServer" - using_apache = False - using_wsgi = True - - def __str__(self): - return "Builtin WSGI Server on %s:%s" % (self.host, self.port) - - def sync_apps(self): - """Hook a new WSGI app into the origin server.""" - cherrypy.server.httpserver.wsgi_app = self.get_app() - - def get_app(self, app=None): - """Obtain a new (decorated) WSGI app to hook into the origin server.""" - if app is None: - app = cherrypy.tree - - if self.conquer: - try: - import wsgiconq - except ImportError: - warnings.warn("Error importing wsgiconq. pyconquer will not run.") - else: - app = wsgiconq.WSGILogger(app, c_calls=True) - - if self.validate: - try: - from wsgiref import validate - except ImportError: - warnings.warn("Error importing wsgiref. The validator will not run.") - else: - #wraps the app in the validator - app = validate.validator(app) - - return app - - -def get_cpmodpy_supervisor(**options): - from cherrypy.test import modpy - sup = modpy.ModPythonSupervisor(**options) - sup.template = modpy.conf_cpmodpy - return sup - -def get_modpygw_supervisor(**options): - from cherrypy.test import modpy - sup = modpy.ModPythonSupervisor(**options) - sup.template = modpy.conf_modpython_gateway - sup.using_wsgi = True - return sup - -def get_modwsgi_supervisor(**options): - from cherrypy.test import modwsgi - return modwsgi.ModWSGISupervisor(**options) - -def get_modfcgid_supervisor(**options): - from cherrypy.test import modfcgid - return modfcgid.ModFCGISupervisor(**options) - -def get_modfastcgi_supervisor(**options): - from cherrypy.test import modfastcgi - return modfastcgi.ModFCGISupervisor(**options) - -def get_wsgi_u_supervisor(**options): - cherrypy.server.wsgi_version = ('u', 0) - return LocalWSGISupervisor(**options) - - -class CPWebCase(webtest.WebCase): - - script_name = "" - scheme = "http" - - available_servers = {'wsgi': LocalWSGISupervisor, - 'wsgi_u': get_wsgi_u_supervisor, - 'native': NativeServerSupervisor, - 'cpmodpy': get_cpmodpy_supervisor, - 'modpygw': get_modpygw_supervisor, - 'modwsgi': get_modwsgi_supervisor, - 'modfcgid': get_modfcgid_supervisor, - 'modfastcgi': get_modfastcgi_supervisor, - } - default_server = "wsgi" - - def _setup_server(cls, supervisor, conf): - v = sys.version.split()[0] - log.info("Python version used to run this test script: %s" % v) - log.info("CherryPy version: %s" % cherrypy.__version__) - if supervisor.scheme == "https": - ssl = " (ssl)" - else: - ssl = "" - log.info("HTTP server version: %s%s" % (supervisor.protocol, ssl)) - log.info("PID: %s" % os.getpid()) - - cherrypy.server.using_apache = supervisor.using_apache - cherrypy.server.using_wsgi = supervisor.using_wsgi - - if sys.platform[:4] == 'java': - cherrypy.config.update({'server.nodelay': False}) - - if isinstance(conf, basestring): - parser = cherrypy.lib.reprconf.Parser() - conf = parser.dict_from_file(conf).get('global', {}) - else: - conf = conf or {} - baseconf = conf.copy() - baseconf.update({'server.socket_host': supervisor.host, - 'server.socket_port': supervisor.port, - 'server.protocol_version': supervisor.protocol, - 'environment': "test_suite", - }) - if supervisor.scheme == "https": - #baseconf['server.ssl_module'] = 'builtin' - baseconf['server.ssl_certificate'] = serverpem - baseconf['server.ssl_private_key'] = serverpem - - # helper must be imported lazily so the coverage tool - # can run against module-level statements within cherrypy. - # Also, we have to do "from cherrypy.test import helper", - # exactly like each test module does, because a relative import - # would stick a second instance of webtest in sys.modules, - # and we wouldn't be able to globally override the port anymore. - if supervisor.scheme == "https": - webtest.WebCase.HTTP_CONN = HTTPSConnection - return baseconf - _setup_server = classmethod(_setup_server) - - def setup_class(cls): - '' - #Creates a server - conf = get_tst_config() - supervisor_factory = cls.available_servers.get(conf.get('server', 'wsgi')) - if supervisor_factory is None: - raise RuntimeError('Unknown server in config: %s' % conf['server']) - supervisor = supervisor_factory(**conf) - - #Copied from "run_test_suite" - cherrypy.config.reset() - baseconf = cls._setup_server(supervisor, conf) - cherrypy.config.update(baseconf) - setup_client() - - if hasattr(cls, 'setup_server'): - # Clear the cherrypy tree and clear the wsgi server so that - # it can be updated with the new root - cherrypy.tree = cherrypy._cptree.Tree() - cherrypy.server.httpserver = None - cls.setup_server() - supervisor.start(cls.__module__) - - cls.supervisor = supervisor - setup_class = classmethod(setup_class) - - def teardown_class(cls): - '' - if hasattr(cls, 'setup_server'): - cls.supervisor.stop() - teardown_class = classmethod(teardown_class) - - def prefix(self): - return self.script_name.rstrip("/") - - def base(self): - if ((self.scheme == "http" and self.PORT == 80) or - (self.scheme == "https" and self.PORT == 443)): - port = "" - else: - port = ":%s" % self.PORT - - return "%s://%s%s%s" % (self.scheme, self.HOST, port, - self.script_name.rstrip("/")) - - def exit(self): - sys.exit() - - def getPage(self, url, headers=None, method="GET", body=None, protocol=None): - """Open the url. Return status, headers, body.""" - if self.script_name: - url = httputil.urljoin(self.script_name, url) - return webtest.WebCase.getPage(self, url, headers, method, body, protocol) - - def skip(self, msg='skipped '): - raise nose.SkipTest(msg) - - def assertErrorPage(self, status, message=None, pattern=''): - """Compare the response body with a built in error page. - - The function will optionally look for the regexp pattern, - within the exception embedded in the error page.""" - - # This will never contain a traceback - page = cherrypy._cperror.get_error_page(status, message=message) - - # First, test the response body without checking the traceback. - # Stick a match-all group (.*) in to grab the traceback. - esc = re.escape - epage = esc(page) - epage = epage.replace(esc('
'),
-                              esc('
') + '(.*)' + esc('
')) - m = re.match(ntob(epage, self.encoding), self.body, re.DOTALL) - if not m: - self._handlewebError('Error page does not match; expected:\n' + page) - return - - # Now test the pattern against the traceback - if pattern is None: - # Special-case None to mean that there should be *no* traceback. - if m and m.group(1): - self._handlewebError('Error page contains traceback') - else: - if (m is None) or ( - not re.search(ntob(re.escape(pattern), self.encoding), - m.group(1))): - msg = 'Error page does not contain %s in traceback' - self._handlewebError(msg % repr(pattern)) - - date_tolerance = 2 - - def assertEqualDates(self, dt1, dt2, seconds=None): - """Assert abs(dt1 - dt2) is within Y seconds.""" - if seconds is None: - seconds = self.date_tolerance - - if dt1 > dt2: - diff = dt1 - dt2 - else: - diff = dt2 - dt1 - if not diff < datetime.timedelta(seconds=seconds): - raise AssertionError('%r and %r are not within %r seconds.' % - (dt1, dt2, seconds)) - - -def setup_client(): - """Set up the WebCase classes to match the server's socket settings.""" - webtest.WebCase.PORT = cherrypy.server.socket_port - webtest.WebCase.HOST = cherrypy.server.socket_host - if cherrypy.server.ssl_certificate: - CPWebCase.scheme = 'https' - -# --------------------------- Spawning helpers --------------------------- # - - -class CPProcess(object): - - pid_file = os.path.join(thisdir, 'test.pid') - config_file = os.path.join(thisdir, 'test.conf') - config_template = """[global] -server.socket_host: '%(host)s' -server.socket_port: %(port)s -checker.on: False -log.screen: False -log.error_file: r'%(error_log)s' -log.access_file: r'%(access_log)s' -%(ssl)s -%(extra)s -""" - error_log = os.path.join(thisdir, 'test.error.log') - access_log = os.path.join(thisdir, 'test.access.log') - - def __init__(self, wait=False, daemonize=False, ssl=False, socket_host=None, socket_port=None): - self.wait = wait - self.daemonize = daemonize - self.ssl = ssl - self.host = socket_host or cherrypy.server.socket_host - self.port = socket_port or cherrypy.server.socket_port - - def write_conf(self, extra=""): - if self.ssl: - serverpem = os.path.join(thisdir, 'test.pem') - ssl = """ -server.ssl_certificate: r'%s' -server.ssl_private_key: r'%s' -""" % (serverpem, serverpem) - else: - ssl = "" - - conf = self.config_template % { - 'host': self.host, - 'port': self.port, - 'error_log': self.error_log, - 'access_log': self.access_log, - 'ssl': ssl, - 'extra': extra, - } - f = open(self.config_file, 'wb') - f.write(ntob(conf, 'utf-8')) - f.close() - - def start(self, imports=None): - """Start cherryd in a subprocess.""" - cherrypy._cpserver.wait_for_free_port(self.host, self.port) - - args = [sys.executable, os.path.join(thisdir, '..', 'cherryd'), - '-c', self.config_file, '-p', self.pid_file] - - if not isinstance(imports, (list, tuple)): - imports = [imports] - for i in imports: - if i: - args.append('-i') - args.append(i) - - if self.daemonize: - args.append('-d') - - env = os.environ.copy() - # Make sure we import the cherrypy package in which this module is defined. - grandparentdir = os.path.abspath(os.path.join(thisdir, '..', '..')) - if env.get('PYTHONPATH', ''): - env['PYTHONPATH'] = os.pathsep.join((grandparentdir, env['PYTHONPATH'])) - else: - env['PYTHONPATH'] = grandparentdir - if self.wait: - self.exit_code = os.spawnve(os.P_WAIT, sys.executable, args, env) - else: - os.spawnve(os.P_NOWAIT, sys.executable, args, env) - cherrypy._cpserver.wait_for_occupied_port(self.host, self.port) - - # Give the engine a wee bit more time to finish STARTING - if self.daemonize: - time.sleep(2) - else: - time.sleep(1) - - def get_pid(self): - return int(open(self.pid_file, 'rb').read()) - - def join(self): - """Wait for the process to exit.""" - try: - try: - # Mac, UNIX - os.wait() - except AttributeError: - # Windows - try: - pid = self.get_pid() - except IOError: - # Assume the subprocess deleted the pidfile on shutdown. - pass - else: - os.waitpid(pid, 0) - except OSError: - x = sys.exc_info()[1] - if x.args != (10, 'No child processes'): - raise - diff --git a/cherrypy/test/logtest.py b/cherrypy/test/logtest.py deleted file mode 100644 index c093da2c..00000000 --- a/cherrypy/test/logtest.py +++ /dev/null @@ -1,181 +0,0 @@ -"""logtest, a unittest.TestCase helper for testing log output.""" - -import sys -import time - -import cherrypy - - -try: - # On Windows, msvcrt.getch reads a single char without output. - import msvcrt - def getchar(): - return msvcrt.getch() -except ImportError: - # Unix getchr - import tty, termios - def getchar(): - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - -class LogCase(object): - """unittest.TestCase mixin for testing log messages. - - logfile: a filename for the desired log. Yes, I know modes are evil, - but it makes the test functions so much cleaner to set this once. - - lastmarker: the last marker in the log. This can be used to search for - messages since the last marker. - - markerPrefix: a string with which to prefix log markers. This should be - unique enough from normal log output to use for marker identification. - """ - - logfile = None - lastmarker = None - markerPrefix = "test suite marker: " - - def _handleLogError(self, msg, data, marker, pattern): - print("") - print(" ERROR: %s" % msg) - - if not self.interactive: - raise self.failureException(msg) - - p = " Show: [L]og [M]arker [P]attern; [I]gnore, [R]aise, or sys.e[X]it >> " - print p, - # ARGH - sys.stdout.flush() - while True: - i = getchar().upper() - if i not in "MPLIRX": - continue - print(i.upper()) # Also prints new line - if i == "L": - for x, line in enumerate(data): - if (x + 1) % self.console_height == 0: - # The \r and comma should make the next line overwrite - print "<-- More -->\r", - m = getchar().lower() - # Erase our "More" prompt - print " \r", - if m == "q": - break - print(line.rstrip()) - elif i == "M": - print(repr(marker or self.lastmarker)) - elif i == "P": - print(repr(pattern)) - elif i == "I": - # return without raising the normal exception - return - elif i == "R": - raise self.failureException(msg) - elif i == "X": - self.exit() - print p, - - def exit(self): - sys.exit() - - def emptyLog(self): - """Overwrite self.logfile with 0 bytes.""" - open(self.logfile, 'wb').write("") - - def markLog(self, key=None): - """Insert a marker line into the log and set self.lastmarker.""" - if key is None: - key = str(time.time()) - self.lastmarker = key - - open(self.logfile, 'ab+').write("%s%s\n" % (self.markerPrefix, key)) - - def _read_marked_region(self, marker=None): - """Return lines from self.logfile in the marked region. - - If marker is None, self.lastmarker is used. If the log hasn't - been marked (using self.markLog), the entire log will be returned. - """ -## # Give the logger time to finish writing? -## time.sleep(0.5) - - logfile = self.logfile - marker = marker or self.lastmarker - if marker is None: - return open(logfile, 'rb').readlines() - - data = [] - in_region = False - for line in open(logfile, 'rb'): - if in_region: - if (line.startswith(self.markerPrefix) and not marker in line): - break - else: - data.append(line) - elif marker in line: - in_region = True - return data - - def assertInLog(self, line, marker=None): - """Fail if the given (partial) line is not in the log. - - The log will be searched from the given marker to the next marker. - If marker is None, self.lastmarker is used. If the log hasn't - been marked (using self.markLog), the entire log will be searched. - """ - data = self._read_marked_region(marker) - for logline in data: - if line in logline: - return - msg = "%r not found in log" % line - self._handleLogError(msg, data, marker, line) - - def assertNotInLog(self, line, marker=None): - """Fail if the given (partial) line is in the log. - - The log will be searched from the given marker to the next marker. - If marker is None, self.lastmarker is used. If the log hasn't - been marked (using self.markLog), the entire log will be searched. - """ - data = self._read_marked_region(marker) - for logline in data: - if line in logline: - msg = "%r found in log" % line - self._handleLogError(msg, data, marker, line) - - def assertLog(self, sliceargs, lines, marker=None): - """Fail if log.readlines()[sliceargs] is not contained in 'lines'. - - The log will be searched from the given marker to the next marker. - If marker is None, self.lastmarker is used. If the log hasn't - been marked (using self.markLog), the entire log will be searched. - """ - data = self._read_marked_region(marker) - if isinstance(sliceargs, int): - # Single arg. Use __getitem__ and allow lines to be str or list. - if isinstance(lines, (tuple, list)): - lines = lines[0] - if lines not in data[sliceargs]: - msg = "%r not found on log line %r" % (lines, sliceargs) - self._handleLogError(msg, [data[sliceargs]], marker, lines) - else: - # Multiple args. Use __getslice__ and require lines to be list. - if isinstance(lines, tuple): - lines = list(lines) - elif isinstance(lines, basestring): - raise TypeError("The 'lines' arg must be a list when " - "'sliceargs' is a tuple.") - - start, stop = sliceargs - for line, logline in zip(lines, data[start:stop]): - if line not in logline: - msg = "%r not found in log" % line - self._handleLogError(msg, data[start:stop], marker, line) - diff --git a/cherrypy/test/modfastcgi.py b/cherrypy/test/modfastcgi.py deleted file mode 100644 index 95acf141..00000000 --- a/cherrypy/test/modfastcgi.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Wrapper for mod_fastcgi, for use as a CherryPy HTTP server when testing. - -To autostart fastcgi, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl", "apache2ctl", -or "httpd"--create a symlink to them if needed. - -You'll also need the WSGIServer from flup.servers. -See http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import sys -import time - -import cherrypy -from cherrypy.process import plugins, servers -from cherrypy.test import helper - - -def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "apache2ctl" -CONF_PATH = "fastcgi.conf" - -conf_fastcgi = """ -# Apache2 server conf file for testing CherryPy with mod_fastcgi. -# fumanchu: I had to hard-code paths due to crazy Debian layouts :( -ServerRoot /usr/lib/apache2 -User #1000 -ErrorLog %(root)s/mod_fastcgi.error.log - -DocumentRoot "%(root)s" -ServerName 127.0.0.1 -Listen %(port)s -LoadModule fastcgi_module modules/mod_fastcgi.so -LoadModule rewrite_module modules/mod_rewrite.so - -Options +ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 -""" - -def erase_script_name(environ, start_response): - environ['SCRIPT_NAME'] = '' - return cherrypy.tree(environ, start_response) - -class ModFCGISupervisor(helper.LocalWSGISupervisor): - - httpserver_class = "cherrypy.process.servers.FlupFCGIServer" - using_apache = True - using_wsgi = True - template = conf_fastcgi - - def __str__(self): - return "FCGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - cherrypy.server.httpserver = servers.FlupFCGIServer( - application=erase_script_name, bindAddress=('127.0.0.1', 4000)) - cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) - cherrypy.server.socket_port = 4000 - # For FCGI, we both start apache... - self.start_apache() - # ...and our local server - cherrypy.engine.start() - self.sync_apps() - - def start_apache(self): - fcgiconf = CONF_PATH - if not os.path.isabs(fcgiconf): - fcgiconf = os.path.join(curdir, fcgiconf) - - # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: - server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] - output = self.template % {'port': self.port, 'root': curdir, - 'server': server} - output = output.replace('\r\n', '\n') - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - helper.LocalWSGISupervisor.stop(self) - - def sync_apps(self): - cherrypy.server.httpserver.fcgiserver.application = self.get_app(erase_script_name) - diff --git a/cherrypy/test/modfcgid.py b/cherrypy/test/modfcgid.py deleted file mode 100644 index 736aa4c8..00000000 --- a/cherrypy/test/modfcgid.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Wrapper for mod_fcgid, for use as a CherryPy HTTP server when testing. - -To autostart fcgid, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl", "apache2ctl", -or "httpd"--create a symlink to them if needed. - -You'll also need the WSGIServer from flup.servers. -See http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import sys -import time - -import cherrypy -from cherrypy._cpcompat import ntob -from cherrypy.process import plugins, servers -from cherrypy.test import helper - - -def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "httpd" -CONF_PATH = "fcgi.conf" - -conf_fcgid = """ -# Apache2 server conf file for testing CherryPy with mod_fcgid. - -DocumentRoot "%(root)s" -ServerName 127.0.0.1 -Listen %(port)s -LoadModule fastcgi_module modules/mod_fastcgi.dll -LoadModule rewrite_module modules/mod_rewrite.so - -Options ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 -""" - -class ModFCGISupervisor(helper.LocalSupervisor): - - using_apache = True - using_wsgi = True - template = conf_fcgid - - def __str__(self): - return "FCGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - cherrypy.server.httpserver = servers.FlupFCGIServer( - application=cherrypy.tree, bindAddress=('127.0.0.1', 4000)) - cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) - # For FCGI, we both start apache... - self.start_apache() - # ...and our local server - helper.LocalServer.start(self, modulename) - - def start_apache(self): - fcgiconf = CONF_PATH - if not os.path.isabs(fcgiconf): - fcgiconf = os.path.join(curdir, fcgiconf) - - # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: - server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] - output = self.template % {'port': self.port, 'root': curdir, - 'server': server} - output = ntob(output.replace('\r\n', '\n')) - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - helper.LocalServer.stop(self) - - def sync_apps(self): - cherrypy.server.httpserver.fcgiserver.application = self.get_app() - diff --git a/cherrypy/test/modpy.py b/cherrypy/test/modpy.py deleted file mode 100644 index 519571fc..00000000 --- a/cherrypy/test/modpy.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Wrapper for mod_python, for use as a CherryPy HTTP server when testing. - -To autostart modpython, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- -create a symlink to them if needed. - -If you wish to test the WSGI interface instead of our _cpmodpy interface, -you also need the 'modpython_gateway' module at: -http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import time - -from cherrypy.test import helper - - -def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "httpd" -CONF_PATH = "test_mp.conf" - -conf_modpython_gateway = """ -# Apache2 server conf file for testing CherryPy with modpython_gateway. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - -SetHandler python-program -PythonFixupHandler cherrypy.test.modpy::wsgisetup -PythonOption testmod %(modulename)s -PythonHandler modpython_gateway::handler -PythonOption wsgi.application cherrypy::tree -PythonOption socket_host %(host)s -PythonDebug On -""" - -conf_cpmodpy = """ -# Apache2 server conf file for testing CherryPy with _cpmodpy. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - -SetHandler python-program -PythonFixupHandler cherrypy.test.modpy::cpmodpysetup -PythonHandler cherrypy._cpmodpy::handler -PythonOption cherrypy.setup cherrypy.test.%(modulename)s::setup_server -PythonOption socket_host %(host)s -PythonDebug On -""" - -class ModPythonSupervisor(helper.Supervisor): - - using_apache = True - using_wsgi = False - template = None - - def __str__(self): - return "ModPython Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - mpconf = CONF_PATH - if not os.path.isabs(mpconf): - mpconf = os.path.join(curdir, mpconf) - - f = open(mpconf, 'wb') - try: - f.write(self.template % - {'port': self.port, 'modulename': modulename, - 'host': self.host}) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - - -loaded = False -def wsgisetup(req): - global loaded - if not loaded: - loaded = True - options = req.get_options() - - import cherrypy - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.log"), - "environment": "test_suite", - "server.socket_host": options['socket_host'], - }) - - modname = options['testmod'] - mod = __import__(modname, globals(), locals(), ['']) - mod.setup_server() - - cherrypy.server.unsubscribe() - cherrypy.engine.start() - from mod_python import apache - return apache.OK - - -def cpmodpysetup(req): - global loaded - if not loaded: - loaded = True - options = req.get_options() - - import cherrypy - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.log"), - "environment": "test_suite", - "server.socket_host": options['socket_host'], - }) - from mod_python import apache - return apache.OK - diff --git a/cherrypy/test/modwsgi.py b/cherrypy/test/modwsgi.py deleted file mode 100644 index 309a541c..00000000 --- a/cherrypy/test/modwsgi.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Wrapper for mod_wsgi, for use as a CherryPy HTTP server. - -To autostart modwsgi, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- -create a symlink to them if needed. - - -KNOWN BUGS -========== - -##1. Apache processes Range headers automatically; CherryPy's truncated -## output is then truncated again by Apache. See test_core.testRanges. -## This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -##4. Apache replaces status "reason phrases" automatically. For example, -## CherryPy may set "304 Not modified" but Apache will write out -## "304 Not Modified" (capital "M"). -##5. Apache does not allow custom error codes as per the spec. -##6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the -## Request-URI too early. -7. mod_wsgi will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. When responding with 204 No Content, mod_wsgi adds a Content-Length - header for you. -9. When an error is raised, mod_wsgi has no facility for printing a - traceback as the response content (it's sent to the Apache log instead). -10. Startup and shutdown of Apache when running mod_wsgi seems slow. -""" - -import os -curdir = os.path.abspath(os.path.dirname(__file__)) -import re -import sys -import time - -import cherrypy -from cherrypy.test import helper, webtest - - -def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -if sys.platform == 'win32': - APACHE_PATH = "httpd" -else: - APACHE_PATH = "apache" - -CONF_PATH = "test_mw.conf" - -conf_modwsgi = r""" -# Apache2 server conf file for testing CherryPy with modpython_gateway. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s - -AllowEncodedSlashes On -LoadModule rewrite_module modules/mod_rewrite.so -RewriteEngine on -RewriteMap escaping int:escape - -LoadModule log_config_module modules/mod_log_config.so -LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-agent}i\"" combined -CustomLog "%(curdir)s/apache.access.log" combined -ErrorLog "%(curdir)s/apache.error.log" -LogLevel debug - -LoadModule wsgi_module modules/mod_wsgi.so -LoadModule env_module modules/mod_env.so - -WSGIScriptAlias / "%(curdir)s/modwsgi.py" -SetEnv testmod %(testmod)s -""" - - -class ModWSGISupervisor(helper.Supervisor): - """Server Controller for ModWSGI and CherryPy.""" - - using_apache = True - using_wsgi = True - template=conf_modwsgi - - def __str__(self): - return "ModWSGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - mpconf = CONF_PATH - if not os.path.isabs(mpconf): - mpconf = os.path.join(curdir, mpconf) - - f = open(mpconf, 'wb') - try: - output = (self.template % - {'port': self.port, 'testmod': modulename, - 'curdir': curdir}) - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) - if result: - print(result) - - # Make a request so mod_wsgi starts up our app. - # If we don't, concurrent initial requests will 404. - cherrypy._cpserver.wait_for_occupied_port("127.0.0.1", self.port) - webtest.openURL('/ihopetheresnodefault', port=self.port) - time.sleep(1) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - - -loaded = False -def application(environ, start_response): - import cherrypy - global loaded - if not loaded: - loaded = True - modname = "cherrypy.test." + environ['testmod'] - mod = __import__(modname, globals(), locals(), ['']) - mod.setup_server() - - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.error.log"), - "log.access_file": os.path.join(curdir, "test.access.log"), - "environment": "test_suite", - "engine.SIGHUP": None, - "engine.SIGTERM": None, - }) - return cherrypy.tree(environ, start_response) - diff --git a/cherrypy/test/native-server.ini b/cherrypy/test/native-server.ini deleted file mode 100644 index b32d98dd..00000000 --- a/cherrypy/test/native-server.ini +++ /dev/null @@ -1,9 +0,0 @@ -[supervisor] -scheme="http" -protocol="HTTP/1.1" -port= 8080 -host= "127.0.0.1" -profile= False -validate= False -conquer= False -server="wsgi" diff --git a/cherrypy/test/sessiondemo.py b/cherrypy/test/sessiondemo.py deleted file mode 100755 index 342e5b59..00000000 --- a/cherrypy/test/sessiondemo.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/python -"""A session demonstration app.""" - -import calendar -from datetime import datetime -import sys -import cherrypy -from cherrypy.lib import sessions -from cherrypy._cpcompat import copyitems - - -page = """ - - - - - - - -

Session Demo

-

Reload this page. The session ID should not change from one reload to the next

-

Index | Expire | Regenerate

- - - - - - - - - -
Session ID:%(sessionid)s

%(changemsg)s

Request Cookie%(reqcookie)s
Response Cookie%(respcookie)s

Session Data%(sessiondata)s
Server Time%(servertime)s (Unix time: %(serverunixtime)s)
Browser Time 
Cherrypy Version:%(cpversion)s
Python Version:%(pyversion)s
- -""" - -class Root(object): - - def page(self): - changemsg = [] - if cherrypy.session.id != cherrypy.session.originalid: - if cherrypy.session.originalid is None: - changemsg.append('Created new session because no session id was given.') - if cherrypy.session.missing: - changemsg.append('Created new session due to missing (expired or malicious) session.') - if cherrypy.session.regenerated: - changemsg.append('Application generated a new session.') - - try: - expires = cherrypy.response.cookie['session_id']['expires'] - except KeyError: - expires = '' - - return page % { - 'sessionid': cherrypy.session.id, - 'changemsg': '
'.join(changemsg), - 'respcookie': cherrypy.response.cookie.output(), - 'reqcookie': cherrypy.request.cookie.output(), - 'sessiondata': copyitems(cherrypy.session), - 'servertime': datetime.utcnow().strftime("%Y/%m/%d %H:%M") + " UTC", - 'serverunixtime': calendar.timegm(datetime.utcnow().timetuple()), - 'cpversion': cherrypy.__version__, - 'pyversion': sys.version, - 'expires': expires, - } - - def index(self): - # Must modify data or the session will not be saved. - cherrypy.session['color'] = 'green' - return self.page() - index.exposed = True - - def expire(self): - sessions.expire() - return self.page() - expire.exposed = True - - def regen(self): - cherrypy.session.regenerate() - # Must modify data or the session will not be saved. - cherrypy.session['color'] = 'yellow' - return self.page() - regen.exposed = True - -if __name__ == '__main__': - cherrypy.config.update({ - #'environment': 'production', - 'log.screen': True, - 'tools.sessions.on': True, - }) - cherrypy.quickstart(Root()) - diff --git a/cherrypy/test/static/dirback.jpg b/cherrypy/test/static/dirback.jpg deleted file mode 100644 index 530e6d6a386fc097f3a1dbabbde2d80fec1175ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18238 zcmb5VRajfk7cQI-+=Dy8ixwxi1qsEiMSnO1cPq5GyESOh;1mg3+T!jWq{RyqC|W9% z9{%UzeAnOF&)yfapS5P)tasMT`_8|$f7<|ZEp@m$00;yEG#?+pzYTyY00)GPjSa$i z{NUi=;NlVE<2@P~5fK3~n2dq~Oa=y1(lF6dQZZ12!E|hN49v`|tgMu@?40Z@oJ=gN zEdL7ve00Ub#UsVXCuN}mQ?dO2wtsy9Fg}n7%K-#r2VjALATaRX5P$&y06iuP1pGe( zVu7#$IQWnM6vzQU5EcmcF>P!tAT|gH06sc`*eFDl*+mU(eKN5rBg;6%R1EF1TKac< zqvkJEji@-q%P;Mt2NoXv>HlwF(Ep1J_@6%r8|Q!1g8w(?W5oZ@fM7NeWvqYe0OH5t z$7#R-MZj-6JTiY9Z~0Z+2Qq&pR9Op8Meoo~Tq;rKtSEkl_E3tu zCM=B{oR-z5Uw`_JYl%ol9qWg4hL^5pGP6&Xaz}Gvz%!&qN_iet7@9s1g4FTs1rcyp^&2O1O0JGrrK!FN<9$ zFaN_y?{ufh+PM3#bOK!}lwQ%FTrC@6@S*hL$gzbk@vk&_6TzV$=9zSvcPx`)g%Z`E zw9nPZ=kj`vMTrfHVN!y#1WYKUF zj4(lda${I45pXf3jR)GbdcD}^(3KblfRCV(EVb~COmu5+WNPHI7FXXir>o9V{x%&h(ax_sg_y%QzJ%}NlS&ks%Lf4jKsQ;u@iEV@l;JHO;12~U$u zS8INEphU1F>2A?L_7C%!<@oa5kGCwly%>COoL_!2jz`37iim*OwT)OV>Sz4}G;A2-3kO>J>M0ck4hhyxomHl^Z!F`Ql|O5-8UaRe zh~M(bA+I*cI3s87UzM2vUV604T~t5ye3GYzh96Rv2)q+C9$^-?Ngt>+NEQ3h;-~g( z7fU=U+6^~rHJ>A^#y=6X<*J!rZnL3d8o1z3jyR@o^x_u1GhhjsC7e!TJ5cq|v*tsa zn=^TX2flrjqj%F;=ItpfveDqu%>OTOM$%6!)LpFiwB5>Bc3M^_6;g`99t(UXJ*kCT z6#duMU-tVaSgK#-R)}&F=il(`T5ay2(`aR^a~uUcnEemS71L!hnq@~DLF{SyElQY}xo`%f_Ce-h{r1KsFmBDo`OeTCuq)R}Kp$Xyk$p%c zj6Zh+moU?da$7i|)BbMeOAYq7gJUlPtKDMwB-=S4A}iSYSfK%Qt~&$Dg_PeOsFL?s z%q@Z>mAfeWb&FpNotm)5SZYYEDN~Aa?z;imra+p#hk@>;Pe>=>LSLAP6ec9? znF4Xe3{5glCE?9@wwet@^s>6O@7%R#TIv755ey5OJXj+?3b=9%baafY771!*R z4A>7Xw{5y~cPm=3DaIZsME0rEw51{LUlcT8++weVQ2Ra>Ttm$V5zuzuPwbPw2NqpYX%ogL^;4(E1nQAGG5_@~6MmV_F zRp2+(gViWH57=?pVG=V-EKB1iD%y<8FJBD4_LGdOT?m#O!aU^a{~90h+Zq#8j$#Hd zIeh*BqriVtf~in?Gd`qwNUbE7SGmLUVpxhkEy+&|z2j4D5`^v|XJ*=Yn)Ce+{y2@R z@x7G3 zwHUZvl7P~uOMVv}p-rz3kJC*oOi0mz{N|>c(j*vmF+@Zs0K+MFfV!**Q1QHrzX#D3 zFCIYGRl!1(^v(q9cS9esMUh7$)!;AfD3QWz9o-TT+fYqQ^I*#>Y}7>s34CB1A2w2S z*IT%tkjQ6;>)mSssoe!U-SX*(6_kHN@+R3e8H^+lOI+ElnFk4nR+32%zURZ!EXt1R z8r`wfL92(vDDp+Zh3%+BvTvh32}h&__BVnVnZY;ny@N4l1nS9~K2|6w=XlIy=zte7 zC^-gw1M974G|CA@*#7a8>Y2{1Sag^kTNFR&NYOQ_V4IJ_ zqcE8@sTnt41M}2BxHJ7Lryj*oWm#tSM6L z3(>Db0IKlagp|@CB_eIf<7hqH(!9zLPYBS%*DOt!QcsAPrXV6ow@s~$58Qf&a?J5l zQzL(&Hz@jgqB9!&T=Jm0Jc9!mt)0lKchR_mAE2V_e&Jdkl%xn=4MtnsTJpQk;3=h& zUqX9#chdSCmt_%hx>(3%?)US~`cnPPKN_#mmK7IsgymP;W zHV!bFmDjFQ%$Ikc16$1pNnf`jOHU20Pru&&1Mm$m$};J)Tj@tEawb32-is9M?HaTD za@6@K_pv58aoiYJ!Q$g--`@QLptqNL)wq+|I-KBX$7+&sNRMX;t)0nEGL4=~Ok+y= z&R>jrglk6ax=Cr5`1GU#4X^S6OzMF46wlD!*RPn@DzB7Q) zo~<*h)q+K)qSp3OMW58!lar(>evLs691*Yg)e?H7WF-W|-@^?>szDv}&{yY0%W}dj z!6bvW!<22POOJw7;RWej84qp(s{YJOlS|lcqC4!2)2*r$x~P+F78kGX|2;o9usw<> zTD5gr6YNa{d2ywS%R5AWk7B4n4SLJZUu~S`b}4~rw+*nt2zkOenr3kTK}7ViomfIN z>-UqlJ+WpZftDWCb$oFFwDy_$L;HrU*~3cOP?DDr%dkm?Old^9xrC+lmn2POl-M{E zO}s;!AtTEp9K6jmLmZr#oTQ{z>&b-WQPDMD+B^je`LSg2!W zUAVnN#^&fx9VPP1X0;5LWgqHF;cTYogz3`F$rWaix1Jm7zVX@G;TVo5QpjH^TtUch zUV)kjf{d>*Gbmz@E_2FUcc#~;Lf6=%r`?T%x0K7`X>4ZtvVTlNG6K;uf;cNv0%rjg z^kYZ$#6pl4@L+|+8n{qA$juC}t^6ioVZ6qxFd>bIvbEUMct}LLx(-_zmtNk;hKmkJdctlDyvCA!{0dt~6?;CmK zTNj9u@=$Ux zyU*i2&Jv_1E*cx>h5Ks;wN7lla;I6EdidtV@5M67?|HUf(&JQ+s10pjU=hR(r~GO9 zX~wPW0X0rHANz&m&!@etcd>3^J5-j2ihx5?XyHXa;yyZ~og@f*kzIC*p)*@mg@s}(bvE3XM|4c?(#A!q> zNMDH^*Q3hwySpw}TfM6Y;5*ByJN6SPp>sTCL2fy(fgSPLF(@h8kh?@>W*RrFEi~NL zZH1aWXPl%vSIJCTGPD+;>r}J>yM&PQ&Yq%mg*78&Q6wbfjG(#<02r$z%mj8KF^A#_ zmDXx=D1O5j)M7fB9>6yM$8cI$YyLh~0blmVr@n)sQ&v3ZrC+2ajw$#LUhigQe-y*z z@#iQXt*ZScuykkoX1{Im*Q@H==Veifx1@^rmYZ)6T&s1qvAmO&0Rn?xv%*@?UQ*_& z9S*&g>l2Zt!qL!({Tc?FFvCg$lBmCaB-*3qB0XQB7DyA9Xz1bi2bByN``3;J%TL-n zNoLDaGFG3xMRu7s*v7m3=B0;hVUq9Ebc<8(0&Yyode6#3f7epBVD~sH(CF=qXL080 zu4FZq*o+$R7W~-Zb`rBP*=hRFU6$g2|CLSi%+EpXK(RbcGX6jORW7BkP$o_@!-`Pq zb%&P~uZUV$2y_+dp>DB*&~MAuA=D5zyQq2m78P7_=8N5R4z)@X8t;?hclEgDJW8oN zOG4`|AhjYkBZc+sO`(S&E5 z;#pQJl=`A)S9?q}&W_6rIx6&WgZf>D-eBUAgttXI?y=cSR|G@)>!}GO+BiB&R-h?KaKHwmM5~I@)>=iXaP|-Djy$12wGnv;PfWuUcKkCALBF2rC(0^E2LL(4hedCIyQ}K)-HE$0HwP zkSKbzh-VVrvglS&;Uq+St0}V1oAtWZAmR(9-V@#k+z^&;9609_cB3Z;%WeR&$~&{d zc+x%GarD)~NgYT8H=pVkyx63d`)4z9abBm?MtB9p(d1e}v5Zg0E`9?$ooiZ^X&H>@ zqff3lSi0(?1C|if{VH)6!!<_@8olA_u+mA(vS!fDXrP;O@1u&SA-R|c4tpVCKqu1D z1?~iGc~9*AeeNtOxXe2TWjQf1Cy?b_bFDvmS+j!Oy+I*Uajeo6W_S9Ob{6)L?hI{T z8_q;2{fTOliTIG#pxr6ztH0R|TG7AZZ0nX^^-gPMdhV?%<*SfiHgCW~hwY--7kz>> zAG#azjeT%4*hYRgiWhwaT>?8&58JTL5P$lPip0MesKx9QIs6w@VU_HlaM|zvmf$VY zXewM8;%RtgE8i7UZjyq%ds^)>jY&(&eP!0`Z-n%`Zo{u>YqR@qowOum<@@@NA!pq8 zE_#{&xXc-6;%Ng6{YsZ2Lf9gvi)6X#Y|l5?F)D)VH2(SBv7xC^niHFuHh#exoFF~w z{&z(i7%MKGEbL+|(|D4jn1Jyo&RG$4yGJ{3pE5OoB{d+-kF!8#-;eRc;IhY-0iV-Mt<--dW4M#(o2pKxrpodD*#4_^XCk@Bk+5lTaj=auGBh(g4$6UYJS z6;S&pZr#UhTRCF+Z3W?4N_@arr{Idv3trI*OB>jYo~NK{eL>H?IeOf!&vDX%)1_2* z<`+F8H@BYhAcBceXo}a4y@(;^OFXB6+Qc)8n*V>mohRM&%Pj?$LjMtO^%Hw@85YSWi~OweN1o`__yq%@=gHmQPv*oI)yH zOPV;!r&b1o>=**NT2~l7_Q=b5=UMd=c10uVsnk-T)^i)0tNF$ksqB?I(M+K{RL2Ax zl%k&XI~(vOwlwV9TwCW$?SfANpPtsy8dYlpz6ztG@3?Dq-d&OnVF36a7A13q0VKjj z42P{nO1Na=XVuy+8isGiQD5PRlDfD;d;dD69RL6W?kksJ`4$byVmOwYkM6rOPZ`zV zzcp*M#n@mZx&4XYW5{z;+b$bX;o=Zd7Zj9&!F|rbAtd)doNH2*)pRuoJ}L-j7sl|47t|`%S8M4ENhMdXg2#Ji3EE`t@ADyObdKy zyqmF+W|4vI>vDkNM)n80Y)w|nP(F#iaM<{EhcP`#sR1q~rfcIN#(W!=D$Rvss7g$C zSh))+(w%asH$G*qIYRk{Cqfb;UucG8yefuy`G|2ge{Ba$>_!^8yxeS+y z_ZYC-qy^o?Czjx6;b_1!=~V*zkuL4I35^hOyTsi>N?rXl#`74P@UkaBFV1lJRi!`r zlJlB+YBb*+Hhj3v&g&Ks*CRiw|7hWXUArq5`u5)BTgQLUq74VCCb5@IQ2uXdTb-e?;M3aJ zMPD{0TDxB(Ut+EEX-T2!bjG@D_ZArZ;HeJE(^M>MQuvTYYP3k!l%}9;y+w0>Q4YL} z21vs$7R248CqCUr-fhpILBOFq7DtaHR6a=U_Ip?GTAYafG&ABy_Zfgm*vox^;b{W( zQ?VyMGem;n9sifp=NcSh#c^=8yv%EdS2itr4 zu?Cr}%W_BS0i|m(?wc}-nKdbHVmlu=v9k=}an#7?Y*nn~wbjYOCAih5>u1EpSm_pS zHBh2fuaF9riG#BU3ve%rXj^^}Fb5Hj=&l@D0?9}Vi}}beB^oZV?;5Q|aD!sop! z>DX&88+8Y`stjn#1e>lH76R3S{_4A(*tZ$wrL`XJbM9088$rlGY#2(P{{6fik zdQ!LMW)Zd=5U1Ky+xHPJc=&NpZ_Ds$))E%5C_D0J_=4J%?qf+W6#xBrG`f>+@@4&H zQ~JQmTA9Vn$C_p(KfPFXNz>?=HK?AF1u1q@13fpY;;sCq0{}Sc5vyr6-EVjZxPP(K z1p82TT1yw@g$t5y@k1Gb|10m2-DP5>BCqvI$@B%ZAzh(>H+!(asjRM;o?_>1>j&C< z=n#f`2%m`Jis8j#MGSY}t6K&WZh%kzACqX;@P|p?V=)*}kHKji?eN>v0%8?4E+Z0d zJ1FnW_%Y1JYC69CPx8Hoheqte(CtmC#%6d9gtzBJdT3bU}t<1N$E-_-Ynh`Q@#%s&)U$p8k7%T{HqIo2 zZD@AUwf*g1d-hJU3C5bQdu&C#w~g_x1@?4p^IG> zQriEL1e=9d9o`Q{xsJQVTV^t_lG~xYCSj@mogu$YsJ~nXj8jhi;dF&2ax)o_ig5in zs&BIO_hZgMM_JkYg>nNTvC&QUHG%5n-Wb_3C-xERwKO6)BagqpB8jZM5Pt16mQ*3e zS3q#^VkKLw!fVfpARgTFzpTjJ;Lul=?U!mDSS@|{BW>1Ayqly>c^>d-x zh_mZ=0_o?b?tOVZp*0;9@3`8$>VFyAa`q=Awq-Stli>yFydpjCBGo;AKX+#V_&aK$ zn%_FLlEFem|GxGRZL%5gF*|?1f>DSLkqq>kcB&zb8e5w+8hZqx?8KosI0L;w25%(Q zc!^x@JtxJkux^)$tdNhJt1LN|o%A76K6VbD^IGc%tx1ZY=uSxpwX#lW0t(yAh0Zz4 zXL0KZ?be<4|LQ)5#QRUv!{U}VQ+mMTl5WWCE5BsT_0`#0p8DmTp})N({pC?ondMgN zGak8g<0qngpxEvI#FhiWsT)j>ENTRM4Q^q4L#qDsz+-*Jr1LF4khi}MacY2~aK9(H z&FSpvJ6$4mTB1TG;WqKjBKH=l!Fxd}$LWa6#=cl2tvx=LwU`dx!?@;(!WRl?T{AhP zi=va7{-Z79CDFSfp<65`uT=Z>ti0wKEZu`QRi_{}$8q`rS; z=u=?l%5)7h8q92OS4fC?=hi~U{!3H2BGEI9R_?9v8EFjrVB(1P`xu^iEGrLGv_CwI z8Fk?BQ(3l*2@=jXAQ%O0iV;xA{jzQx|0&=}McT;qsEO)aELj5`{iTUIdyW!^(356E z1o?TCabae22K39V9&B(ITQ^O;+H_QEkP=};R|<@n?&fJyEq3P?!@*@Pb5fFyv0fq{ zRT}CqNlF=xcTwIZq`;(aNtK=Fvj-ijDAxMmzN-3?ZzA*b_6hh>Xz)X=IWYS63C+FE zVo8+~GFxq?pJulpw+@p=ng2vwI|YwDl{-w|JCmWdiofXTF!qL0j$B5drA+K+j+ZBs zu$+ygpCuegjR&ms`F}GdaDx;Q-uV5{k=ww)6-7QBX%i>5icPV}t(O5u9roKKu1M!j z(eHzVThW=aK}T)R-rc5HV^2H z#)DK^0h@nEe1R{!)N6L(*V3@DmF|HJ`iQ?;RB_&!eWJeeBEB{c6PHQke4{KK^RCzB+FX3{Nrts`a4vVKGdT%XQ8AmWCMm{|e+Z*h80oJA5qhUBy%-OK@G>CZs1rJE zdOb{Z;EpUy%jF!4^kql3jf>r5B3ACSHTov+<>n*ifpOAQ48+7bVq>WEf%6m1m1vzG z91e%>ZOwGv-Y(6nZ%xXZ3v^q?4!=26t{@>c*gpQ@TIDqCIol`#85S#(dyc>&+tLfX ze0{Nl{y|FQYp&N=kdM>M)Ur=jC?9vY_(hlM2}W*-n9AAd$-0y2g70f6hoLvG`IZ<~ zq*nkXI7aAF)ToT*0DR^?Te6S=ZjaAXK+GY{vus4^)c4?sdcx>({;el^%bx|WN7$ZY z3xVUd_-b~+k6+e7JZi%Je+i<0d19s5JGu#A_e9>5=Nol_)~7bi&CUnA{JD)l4P*n)3|%qc~^T&o7g&9MhJ;xO!t zZV7X|33!6iZ1b3g5~t2@$yG!tE);(9M=CRRU+Wybtd-SyStA#w+A7s`V@{@0dU;7Y zCbe{CP zKg97l&{C#W|9Ktj1SWw#oW>AmwM+$Qy2T#iQagDI{fNy673vewWc3$%mzXa8l&$yZer-kr_?2dZM*kYzE9hbQ|7>B;J z^2Q7J&?oA^n-I{Hs7g4=7w>QLhKX+&v&yypZ)tPcLQ^;LRK=Hh=;;b%VyCg`**fnu z%8B4912)@Y>>*UqDIK(W+!h7?^$EV`5gdPc0h!q^Q6%(nn-_*6G)Va+g@yviNF-eN5AugZl`0TE5O%ikribhz;W}zc__utQ}?( zKy(35v#RWV6m~ZU$3%3kn>9V-gv9TX*zWjGhpRy1QXU$gTe4PO=(Rxl}Bw>Tdij=A6%GhqV z4~yPyMs{zO$;?y==DU|1*nLmPPUPR%aTsB=^QuI5N!dA~{)1$&R}AUahAI2_xnJAo znbj)eWUNPFKwkO*$>iq6*Zpi~{0tYCS#f?yyMB8DuoVvHu_Tn)nQcVP+Ct*c%%8#clj&cv*58Uh# zm}!UM{#>LH!Vm5tySC+8#g*qWSV@g?*@!%k{mF9l4`2(U`>p5QL0Qk89f9CXNi?$O z!_v#z9MO1d5*kOR37~7?Q>juR!Eo@GAC`xhfRwHZw2yYhnU?PvEfGp2gedHjv6>?{=q3t@7o`05=~Y(E&p3vDnK#@vh>RZ{`}lYgTCthZ zM#Vx6pi*~ne_rzm-6#h9?5uZX=SVHk6$v>W)rJbTMcp2{DxQA)K)udlzQE@qKJA240Eem${R3##U~6X> z*FMu->`&9`+F%H@h}p2r{KPdl@9%5G6<&y>h2MM~ywTHlKg3xeA}ZE6ssN5KNLEOg zt2^o(ZfyE7Q_hVI@ft3P=x=>21HcY3*aM+yo}!Y@X7E;mi1CwHkW=3dIkbGfNTMXx z2c+IE##Vg{%G9M*p2UaQ^(O4#C_wyGX}NfU&P(Z3@36I}8nI_$Va9<^FtbT@+D6_P zFGOROFqfRIki+*=HU0?n8VYr{7Y~ruf-3Ja4IJfdBp4#;MTyt2Z_T%lxiWMrx$FtL z`kOT0qYN?}GCq$d7luJ)o36IFZK5BH2zBU%wS1$78|ED4)Z=^Dnj!ei&owxld9WvS_* zM;Q|-{dg!2h>jUMe76hIr2Jiji&h?dv#WNh3}CGqm@jzE_15oCr8i?$d`~^xO$m$5 zklfyUl-7d+#x2A|E3uy~D5!2Rh&`X!-I4w>jrf-rqf658 zI;S$@F07)%PkSU{*ESEZq5Aqji;wPE#U!{gSG}W@oDQ&XqV2b7Ta_jU%GD^udLex# zs3b&&HT}u>bc=$x=se?BCx(9gH+$coU3Y(egEywp@IEP#+SVDRzEf?di~JzV^LNH! zJnm6~y+;QAG+o1Tq6+ojrm1Dg27uF5Vt2kwD{pb^>MX#KAFJSW>Vt)`|?Gx5&m6fKT3{vTo|i3qOqL0*ncB0`Vx0* z{HN82|5Ut8moWS;ausL+mac%G2dFU3vHS9=o?vM4PNof03R*45i3N^KQO9PZ6RZpG zjQnr(y(XC{UkW_16c)|>LSSoRJ{_S6y|9MJU?p^Rm5a_}Qzs2LohqDIbdOR%GT2VX z7zCSSo;PBjCfDB$NDk=5nIB`vgOqB$?~V)*fqG|4BPPqMlv_n<>1+SWP8T9_bL>;d zA<=%-7qzfxVP!*3CAMk$<9WlPoj|%`fdsfw_^NZASFALtbm_$@tVgEFMrI&$gy8G4 zzwo;JQXkuP#Ks|9hnk&;A>62DTioGko*R1XPLtse69-4rv-HBGKHHN@zonW8pM9L4 zD4`l2r;rJ%b+Iszv{oI5J4r;#YL22}oe0Qlp89fQawdpu?liWK)(c=VQMWB#TojVsDfQ)B@|k<&&SZN; zrSDXf=k9cLz4y;gNYjNe-6sb@&e`u9&b@|;K=M}ePBltGf$H6|ztp2&J%KVvPvBI8 zp^WdAmji&8461-s)hd$uT`wBEOg`mQauSi5-+VvIa$7b|#Y(;A>BqSm-eRKT55LIRoN*A&62#Apj zTeMti8<9q2Bzz7aRt*c~m+lS=(eScxMK0DH`>BcID~X&73A!$7+4YvU0!wpJ2sIQ~LC5VUh-0E-Bc`ovy;|Fi+{MO1iKUvJ- z_OJ{^@gEBm%pA!&$|KBxH&l1bIczo!i#7`fKcEB~_G{xV1lq%2JG zt741&y_K;|NOlY;`TVc};&WbZP6F@ap`|iZ;-DErN>eQMetMH|7s@DoG;Cn}sS+Pg36B#w{W|duI(#$5ns#MN;W|GobrKc8`mjB_1Q zma=?3AR%bnOQ{Ewu+gE6jbtmxfYWs6u8APZ4r3t!(?5yPv{9&Q87tisbSKWl z_0=J@mDlI~r%-`DqBR44A-yi<6RHU@hUd}*>ef1=JS>UlPNbze;t;!MqVzVA#p4g#l zmiAuGk6%0k$e4ECE%>m|&ERd(ItNN*rq$`3b@1_%&)~UanZOS&yA-WCE2ijG#^U!( zNK5Q}rdr2})JB|Sol%}OoGa~LX!wyvA}u!04<*1nLgd>FCZ(Cit0ekgm$xaAry zrI0M#2W(bt9|yyDyECE3nK^L?j8)UccU&A%Ay38FV)RxVXw8!3hsSSjH%o@&aNfex zD)Gf|aO)JH7^ljJjOI^64x$H$sAN)0@VaPUwC`xi@N&_n^8OOEB}u^e@n{DRCXD^V zu!+40H)X_WQo(;_r)%oUysj1L^x05MQ}J@rZoVl@aX>)xqpCRof$Jgn$OEuvCjvX? zW+FLKiL@_!|0fN2UBGr;lxyx*&DM|sd>!WwOJj|?U}h8%E>QCWZLb*)Rr9AjO7{s z8dv2a$mOr)$|t<5)UH=aH%oT>x-=wZzTX~dFPAV7$^C@k4HLf9pYc1bU!OalH^QUA zxJUq_atU8bcX};{!z;(~LD82rN$v^2`DG|-N+f1riKi2*u*cfs*}Lf{U-FemKFrpq z%Pn_&qDa7-+LWH26-~+>fTRH2H(Y}s?|59Nj4j^t?f~NuYOTYMauZ+^1Rv~nV|LSM z4F-0JqoBHjq)GwRk3kk^O!Sw}UCB7Caq>@4}=Kh>sX0kCf16AJ|>sizGzw(BR2% z2=Q07WZMOo?~ZddtJmrhVX9OA0X}PLxFLY-Pwm4NBL*A@ zg+l|78mhB;3X1(`+-K~on|b+UkhY!b!W*H@NZ0BoB(%jS8M zy&ERz?hjki#^d_|>beo13y8w1vWlPhFmmQdYfaXd* zeJp^-KLEi$0RFEWrC%NictYV6dA4ptXe@B#k9L@8rMUa+-t6}2noVsN$TwaGIT&Gx zI05anlH4z|MQm@6)}uzVnC3W%HV(P05EC_MlCN&`8#N=~PzKl3*cU;QUY4g^i;vAK zeCdRy)3!3@YWLmFU?D7C9K@GYfiwFktMC=sC1oR_5q7%IV4DtRn*u{C*^+7`!=;Fd zu8fJ=W|Pr+#(PtL{QrE)m3b8s-ReLejS+u7YD45f;fYS$e4*^8Dbo-82M7v)UP9CX z-@YcAB7JP0wsm3EjPpkz$YGl=kjqX3a^>D+(?3mn(Ca^T0u9CZUg03n66!}^`tmYgwrly>1nrS>g@js{q*Gid4b?=m84 z?us|UIW44CI@@4$tnkpXBw73Yc3*yx)-B(v%!X)aop_O`YCIPWv64GeW7smN2pd-h z7oJo5D|7Kglj&&FSM)eV$b88?eW?`p^Ukc%%E?~sX`9AS3f{|fV`H2OP-J!W*1S}# z3z=}Aq~9=KusUs+?1pvXJ?EZ}Al9aH)NEDg`&egeLP$~cYsb*{X-93@{{X0+)&lyY z4^<<(Izyd5pI7>*7Mbse+1?A>SkLWRA9j~I#{G77tTZkcBnCfRXiT)I7rfA9d$U^` zTj_}(E7_CXI-phRr$_qDhq;2F8Ou-oo&p^u_#1Wezn;VrFi6Sw%Kl>9@AX^>DsR=W z@@zBz?bf(D(}($9E_oh1+YxgZFMR~atVHQrM~O_ZFD|ly3c=qTf4cLP%NqcWk?MI) z{;U9``L>%$a2O)#%WWKwKMZ{VUkOmhTf@xCKbI(lswzhg(?qkIl>`|YQMVDb_-L+t zCXo%$Ao3;QRk7-a8P}$f(v|Q%-qXM*1XfSKxA;(0UP0JSeEx#jQ9>sDDPTKI-Pj@% zW82AkAi*!+y7G0Di%SkR)|`@(%Wf@hb36Tfh!R39(ojn)%g(ED%l9J(f$IZiP&UTN ze%mv142WPvK5l>}Tg1uxq^Y&?xZVc!K-JpL-+%(_yi~U|@tKxNp6qeWi@mHH<=Aqn z(<`tibwLl>ma|Hh)U==-Y!eqwlsxE&SDcp_lxVRSHgEW4ydU^}!az#{iHwXXOwqZ$ z`bCm%JxBxM!Vr(>l7)A!th*F;NRAYghvZOqTxi*)suINTeZ)2*2eg*v#Y-uOT3Z@g~GvY`KJSz&J<%6`4!5@fwipT2K;Fpj(R>JrXIOu9=$gouXlor$QmJZH%% zE2VW(8#(iui-pqjetw|QeWS;3ip>EuGQI?ZWp-28{l1p2{4^E<2H4 z`4IgRN@YbNNSUcuD`ixWrl1^fEi_GI1WoT_AGKEV@fw1*I+;% zL>+cftBe$89dIvrJg_NlHf)F!o8i=0u86B&rtwx6zGqWRq-XUts?I-cv)NE2!92>9 z$~CC*aTZC11LO^EA{)H1e>n%G!#;zX_XGy)d2>?SyqV*H?ti7ZFFHcjygQudofl2( z0=1^T1^d0OmMCx9G28MY)gGB}lb|uIQ|Hbj-x0O&MT?ei&@48#sEQU6nP#ZJ4>OHzWz?o=O}m$_PsQei=x<*$&4W>d9n~`Qh`tV zr;6WP>AfGGHM}MvI1ayL%=}#fEn;9ng95c;1b7pt$6h#;$l+LBYXBLtt!7=mub->+ z;1I==MY$o`7Z`whPYB&{muCHk%Q@EE99zVK`6QC#{-k(48IfV>0fZ4D z)q&3GYBqSFE$7aAv`C;yuS5q*vR^yPI8lD|_nK)?Z z^Xe~o4R^&4SDE`z2!q_Q_=%TO<#r!&hzT>t{gru`l@f0pu~MHH{p;pd+^-jPM4$hL z2$rz7z1ZYrYK!HzH8pb{G3zXFG8r@_g3B&9Ry!P`7zM&Tv{`n#%X) zgSL_r$dBlmh5mOaxCqdeRGMroP3eRnys7&YWo`$p!uQY>oQw*L2L?t3@4p8Z%a%Xu z4$@yRvEN*?H^+z67Dzl}SPcpJO7EDP{oF4aCQHDhW32O>(9V@Ui~Alcack#{(O0}(DICX8GSDsmf z(C4z5)>bEjoP0~Px@`F*m9VnMm?U}@x8oQ%Z3I86!ItX)>=~lsC-0PpZ>k0fw(|?x zvHA})$>a{lqs=Wnj8cE$NWLdA+@5IaY2%}m<9*{T03va+vKWur z?#r5FToA27dY;QLZ^Xrc+5Z5QqmBKRVK~Oav&{PPMNR@5ejA{)=6NQf&?nFtfS+lLt1nL0(Oy! zs@qcnbSpa2zArJd9c@e~hBh{s(QL6Rc%TcLveVSUh7d?4jmrC9%BYHh@&*=yBHKul zQr{2VAaphw2TV!ZP@I6D>Qtq~zBezzQE7BZX$Pv|h?6b1RNjv6w1D(I)?hZs!qPge zK926Q`EJKPSnjh@P=3~TT$=-hunzwK%H^I;m98bSNgBi>Iqe?Hu{v{H9O%tjv6by> zsuPBx+Mm%DhMK|HDg!($Zn)m7fy*EN0MJ5>>lx7h0RE6%!H^WHkN2Y0<7zZB1i7Zi z5FhUal9`MrQmNyf_S7M+AYmHyR6xp@T4uALh!!8^2Ry)cm~W;XL@73#oWj$@^=UW3@&JE0@m+#o>BNs+`Bwp`gF+d1dSYVf9f zF}qF9LK9S{PTLL3f_MRj1ST_jV!}hiaj+h3zonB?ksEZgy`o2MYIN#6y6&8_=(gZ& z?u@u26FVDF?1M;-pTc%x!~ybPn}DrNjDxZ=8Yc3o$v6gTkZ780)N47W0C7FnVH_7V zqn~oL0qtRE#7=XXiKXGoV-=_FYY58Db5sUUxGpNya5s27E^3ma% z)>4L<3nfN0wX!GuD@TM8Kt~l$+GQkTKRQda)oEog+4zS1PD>RRxz+CCZgpB-OG}`U zm+Gj>!Loi~?yHOO1(u87_Z+`)os}PnyGKQ-Q${bu5ykj|%jP~t(bx4>@PxUfxXF~2 zCi}r>7tzMYKUJk-1j|gEj=Y?Tgf+p3Ve@Mjb5G!;6SEfLp+sT^Z(;i@crEvpnRzRC zkCg6Wo9_fSKv9xY) zb!u3fX0ou&NWkp)^;(u7Nit5q^Ev(TMGt}juB)UKP59kr_-#x$Y z6`xAJDsixew2^z5ONyne-^`!HwXQPZCkmNkU1XpThF9<}R~L zO_r))H(#pgQu(bIS)OxsLe}|)%5A2>QKU6wt>Oa+0L1lB^4I`4T1os$^*|&@l`_`a z%BmDKx7c9UJySJ-q&P8>KN10qld=< -# -*- coding: utf-8 -*- -# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 - -import cherrypy -from cherrypy._cpcompat import md5, ntob -from cherrypy.lib import auth_basic -from cherrypy.test import helper - - -class BasicAuthTest(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self): - return "This is public." - index.exposed = True - - class BasicProtected: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - class BasicProtected2: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - userpassdict = {'xuser' : 'xpassword'} - userhashdict = {'xuser' : md5(ntob('xpassword')).hexdigest()} - - def checkpasshash(realm, user, password): - p = userhashdict.get(user) - return p and p == md5(ntob(password)).hexdigest() or False - - conf = {'/basic': {'tools.auth_basic.on': True, - 'tools.auth_basic.realm': 'wonderland', - 'tools.auth_basic.checkpassword': auth_basic.checkpassword_dict(userpassdict)}, - '/basic2': {'tools.auth_basic.on': True, - 'tools.auth_basic.realm': 'wonderland', - 'tools.auth_basic.checkpassword': checkpasshash}, - } - - root = Root() - root.basic = BasicProtected() - root.basic2 = BasicProtected2() - cherrypy.tree.mount(root, config=conf) - setup_server = staticmethod(setup_server) - - def testPublic(self): - self.getPage("/") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.assertBody('This is public.') - - def testBasic(self): - self.getPage("/basic/") - self.assertStatus(401) - self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"') - - self.getPage('/basic/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')]) - self.assertStatus(401) - - self.getPage('/basic/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')]) - self.assertStatus('200 OK') - self.assertBody("Hello xuser, you've been authorized.") - - def testBasic2(self): - self.getPage("/basic2/") - self.assertStatus(401) - self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"') - - self.getPage('/basic2/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')]) - self.assertStatus(401) - - self.getPage('/basic2/', [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')]) - self.assertStatus('200 OK') - self.assertBody("Hello xuser, you've been authorized.") - diff --git a/cherrypy/test/test_auth_digest.py b/cherrypy/test/test_auth_digest.py deleted file mode 100644 index 1960fa81..00000000 --- a/cherrypy/test/test_auth_digest.py +++ /dev/null @@ -1,115 +0,0 @@ -# This file is part of CherryPy -# -*- coding: utf-8 -*- -# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 - - -import cherrypy -from cherrypy.lib import auth_digest - -from cherrypy.test import helper - -class DigestAuthTest(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self): - return "This is public." - index.exposed = True - - class DigestProtected: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - def fetch_users(): - return {'test': 'test'} - - - get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(fetch_users()) - conf = {'/digest': {'tools.auth_digest.on': True, - 'tools.auth_digest.realm': 'localhost', - 'tools.auth_digest.get_ha1': get_ha1, - 'tools.auth_digest.key': 'a565c27146791cfb', - 'tools.auth_digest.debug': 'True'}} - - root = Root() - root.digest = DigestProtected() - cherrypy.tree.mount(root, config=conf) - setup_server = staticmethod(setup_server) - - def testPublic(self): - self.getPage("/") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.assertBody('This is public.') - - def testDigest(self): - self.getPage("/digest/") - self.assertStatus(401) - - value = None - for k, v in self.headers: - if k.lower() == "www-authenticate": - if v.startswith("Digest"): - value = v - break - - if value is None: - self._handlewebError("Digest authentification scheme was not found") - - value = value[7:] - items = value.split(', ') - tokens = {} - for item in items: - key, value = item.split('=') - tokens[key.lower()] = value - - missing_msg = "%s is missing" - bad_value_msg = "'%s' was expecting '%s' but found '%s'" - nonce = None - if 'realm' not in tokens: - self._handlewebError(missing_msg % 'realm') - elif tokens['realm'] != '"localhost"': - self._handlewebError(bad_value_msg % ('realm', '"localhost"', tokens['realm'])) - if 'nonce' not in tokens: - self._handlewebError(missing_msg % 'nonce') - else: - nonce = tokens['nonce'].strip('"') - if 'algorithm' not in tokens: - self._handlewebError(missing_msg % 'algorithm') - elif tokens['algorithm'] != '"MD5"': - self._handlewebError(bad_value_msg % ('algorithm', '"MD5"', tokens['algorithm'])) - if 'qop' not in tokens: - self._handlewebError(missing_msg % 'qop') - elif tokens['qop'] != '"auth"': - self._handlewebError(bad_value_msg % ('qop', '"auth"', tokens['qop'])) - - get_ha1 = auth_digest.get_ha1_dict_plain({'test' : 'test'}) - - # Test user agent response with a wrong value for 'realm' - base_auth = 'Digest username="test", realm="wrong realm", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' - - auth_header = base_auth % (nonce, '11111111111111111111111111111111', '00000001') - auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET') - # calculate the response digest - ha1 = get_ha1(auth.realm, 'test') - response = auth.request_digest(ha1) - # send response with correct response digest, but wrong realm - auth_header = base_auth % (nonce, response, '00000001') - self.getPage('/digest/', [('Authorization', auth_header)]) - self.assertStatus(401) - - # Test that must pass - base_auth = 'Digest username="test", realm="localhost", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' - - auth_header = base_auth % (nonce, '11111111111111111111111111111111', '00000001') - auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET') - # calculate the response digest - ha1 = get_ha1('localhost', 'test') - response = auth.request_digest(ha1) - # send response with correct response digest - auth_header = base_auth % (nonce, response, '00000001') - self.getPage('/digest/', [('Authorization', auth_header)]) - self.assertStatus('200 OK') - self.assertBody("Hello test, you've been authorized.") - diff --git a/cherrypy/test/test_bus.py b/cherrypy/test/test_bus.py deleted file mode 100644 index 51c10220..00000000 --- a/cherrypy/test/test_bus.py +++ /dev/null @@ -1,263 +0,0 @@ -import threading -import time -import unittest - -import cherrypy -from cherrypy._cpcompat import get_daemon, set -from cherrypy.process import wspbus - - -msg = "Listener %d on channel %s: %s." - - -class PublishSubscribeTests(unittest.TestCase): - - def get_listener(self, channel, index): - def listener(arg=None): - self.responses.append(msg % (index, channel, arg)) - return listener - - def test_builtin_channels(self): - b = wspbus.Bus() - - self.responses, expected = [], [] - - for channel in b.listeners: - for index, priority in enumerate([100, 50, 0, 51]): - b.subscribe(channel, self.get_listener(channel, index), priority) - - for channel in b.listeners: - b.publish(channel) - expected.extend([msg % (i, channel, None) for i in (2, 1, 3, 0)]) - b.publish(channel, arg=79347) - expected.extend([msg % (i, channel, 79347) for i in (2, 1, 3, 0)]) - - self.assertEqual(self.responses, expected) - - def test_custom_channels(self): - b = wspbus.Bus() - - self.responses, expected = [], [] - - custom_listeners = ('hugh', 'louis', 'dewey') - for channel in custom_listeners: - for index, priority in enumerate([None, 10, 60, 40]): - b.subscribe(channel, self.get_listener(channel, index), priority) - - for channel in custom_listeners: - b.publish(channel, 'ah so') - expected.extend([msg % (i, channel, 'ah so') for i in (1, 3, 0, 2)]) - b.publish(channel) - expected.extend([msg % (i, channel, None) for i in (1, 3, 0, 2)]) - - self.assertEqual(self.responses, expected) - - def test_listener_errors(self): - b = wspbus.Bus() - - self.responses, expected = [], [] - channels = [c for c in b.listeners if c != 'log'] - - for channel in channels: - b.subscribe(channel, self.get_listener(channel, 1)) - # This will break since the lambda takes no args. - b.subscribe(channel, lambda: None, priority=20) - - for channel in channels: - self.assertRaises(wspbus.ChannelFailures, b.publish, channel, 123) - expected.append(msg % (1, channel, 123)) - - self.assertEqual(self.responses, expected) - - -class BusMethodTests(unittest.TestCase): - - def log(self, bus): - self._log_entries = [] - def logit(msg, level): - self._log_entries.append(msg) - bus.subscribe('log', logit) - - def assertLog(self, entries): - self.assertEqual(self._log_entries, entries) - - def get_listener(self, channel, index): - def listener(arg=None): - self.responses.append(msg % (index, channel, arg)) - return listener - - def test_start(self): - b = wspbus.Bus() - self.log(b) - - self.responses = [] - num = 3 - for index in range(num): - b.subscribe('start', self.get_listener('start', index)) - - b.start() - try: - # The start method MUST call all 'start' listeners. - self.assertEqual(set(self.responses), - set([msg % (i, 'start', None) for i in range(num)])) - # The start method MUST move the state to STARTED - # (or EXITING, if errors occur) - self.assertEqual(b.state, b.states.STARTED) - # The start method MUST log its states. - self.assertLog(['Bus STARTING', 'Bus STARTED']) - finally: - # Exit so the atexit handler doesn't complain. - b.exit() - - def test_stop(self): - b = wspbus.Bus() - self.log(b) - - self.responses = [] - num = 3 - for index in range(num): - b.subscribe('stop', self.get_listener('stop', index)) - - b.stop() - - # The stop method MUST call all 'stop' listeners. - self.assertEqual(set(self.responses), - set([msg % (i, 'stop', None) for i in range(num)])) - # The stop method MUST move the state to STOPPED - self.assertEqual(b.state, b.states.STOPPED) - # The stop method MUST log its states. - self.assertLog(['Bus STOPPING', 'Bus STOPPED']) - - def test_graceful(self): - b = wspbus.Bus() - self.log(b) - - self.responses = [] - num = 3 - for index in range(num): - b.subscribe('graceful', self.get_listener('graceful', index)) - - b.graceful() - - # The graceful method MUST call all 'graceful' listeners. - self.assertEqual(set(self.responses), - set([msg % (i, 'graceful', None) for i in range(num)])) - # The graceful method MUST log its states. - self.assertLog(['Bus graceful']) - - def test_exit(self): - b = wspbus.Bus() - self.log(b) - - self.responses = [] - num = 3 - for index in range(num): - b.subscribe('stop', self.get_listener('stop', index)) - b.subscribe('exit', self.get_listener('exit', index)) - - b.exit() - - # The exit method MUST call all 'stop' listeners, - # and then all 'exit' listeners. - self.assertEqual(set(self.responses), - set([msg % (i, 'stop', None) for i in range(num)] + - [msg % (i, 'exit', None) for i in range(num)])) - # The exit method MUST move the state to EXITING - self.assertEqual(b.state, b.states.EXITING) - # The exit method MUST log its states. - self.assertLog(['Bus STOPPING', 'Bus STOPPED', 'Bus EXITING', 'Bus EXITED']) - - def test_wait(self): - b = wspbus.Bus() - - def f(method): - time.sleep(0.2) - getattr(b, method)() - - for method, states in [('start', [b.states.STARTED]), - ('stop', [b.states.STOPPED]), - ('start', [b.states.STARTING, b.states.STARTED]), - ('exit', [b.states.EXITING]), - ]: - threading.Thread(target=f, args=(method,)).start() - b.wait(states) - - # The wait method MUST wait for the given state(s). - if b.state not in states: - self.fail("State %r not in %r" % (b.state, states)) - - def test_block(self): - b = wspbus.Bus() - self.log(b) - - def f(): - time.sleep(0.2) - b.exit() - def g(): - time.sleep(0.4) - threading.Thread(target=f).start() - threading.Thread(target=g).start() - threads = [t for t in threading.enumerate() if not get_daemon(t)] - self.assertEqual(len(threads), 3) - - b.block() - - # The block method MUST wait for the EXITING state. - self.assertEqual(b.state, b.states.EXITING) - # The block method MUST wait for ALL non-main, non-daemon threads to finish. - threads = [t for t in threading.enumerate() if not get_daemon(t)] - self.assertEqual(len(threads), 1) - # The last message will mention an indeterminable thread name; ignore it - self.assertEqual(self._log_entries[:-1], - ['Bus STOPPING', 'Bus STOPPED', - 'Bus EXITING', 'Bus EXITED', - 'Waiting for child threads to terminate...']) - - def test_start_with_callback(self): - b = wspbus.Bus() - self.log(b) - try: - events = [] - def f(*args, **kwargs): - events.append(("f", args, kwargs)) - def g(): - events.append("g") - b.subscribe("start", g) - b.start_with_callback(f, (1, 3, 5), {"foo": "bar"}) - # Give wait() time to run f() - time.sleep(0.2) - - # The callback method MUST wait for the STARTED state. - self.assertEqual(b.state, b.states.STARTED) - # The callback method MUST run after all start methods. - self.assertEqual(events, ["g", ("f", (1, 3, 5), {"foo": "bar"})]) - finally: - b.exit() - - def test_log(self): - b = wspbus.Bus() - self.log(b) - self.assertLog([]) - - # Try a normal message. - expected = [] - for msg in ["O mah darlin'"] * 3 + ["Clementiiiiiiiine"]: - b.log(msg) - expected.append(msg) - self.assertLog(expected) - - # Try an error message - try: - foo - except NameError: - b.log("You are lost and gone forever", traceback=True) - lastmsg = self._log_entries[-1] - if "Traceback" not in lastmsg or "NameError" not in lastmsg: - self.fail("Last log message %r did not contain " - "the expected traceback." % lastmsg) - else: - self.fail("NameError was not raised as expected.") - - -if __name__ == "__main__": - unittest.main() diff --git a/cherrypy/test/test_caching.py b/cherrypy/test/test_caching.py deleted file mode 100644 index 720a933a..00000000 --- a/cherrypy/test/test_caching.py +++ /dev/null @@ -1,329 +0,0 @@ -import datetime -import gzip -from itertools import count -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import sys -import threading -import time -import urllib - -import cherrypy -from cherrypy._cpcompat import next, ntob, quote, xrange -from cherrypy.lib import httputil - -gif_bytes = ntob('GIF89a\x01\x00\x01\x00\x82\x00\x01\x99"\x1e\x00\x00\x00\x00\x00' - '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x02\x03\x02\x08\t\x00;') - - - -from cherrypy.test import helper - -class CacheTest(helper.CPWebCase): - - def setup_server(): - - class Root: - - _cp_config = {'tools.caching.on': True} - - def __init__(self): - self.counter = 0 - self.control_counter = 0 - self.longlock = threading.Lock() - - def index(self): - self.counter += 1 - msg = "visit #%s" % self.counter - return msg - index.exposed = True - - def control(self): - self.control_counter += 1 - return "visit #%s" % self.control_counter - control.exposed = True - - def a_gif(self): - cherrypy.response.headers['Last-Modified'] = httputil.HTTPDate() - return gif_bytes - a_gif.exposed = True - - def long_process(self, seconds='1'): - try: - self.longlock.acquire() - time.sleep(float(seconds)) - finally: - self.longlock.release() - return 'success!' - long_process.exposed = True - - def clear_cache(self, path): - cherrypy._cache.store[cherrypy.request.base + path].clear() - clear_cache.exposed = True - - class VaryHeaderCachingServer(object): - - _cp_config = {'tools.caching.on': True, - 'tools.response_headers.on': True, - 'tools.response_headers.headers': [('Vary', 'Our-Varying-Header')], - } - - def __init__(self): - self.counter = count(1) - - def index(self): - return "visit #%s" % next(self.counter) - index.exposed = True - - class UnCached(object): - _cp_config = {'tools.expires.on': True, - 'tools.expires.secs': 60, - 'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.root': curdir, - } - - def force(self): - cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' - self._cp_config['tools.expires.force'] = True - self._cp_config['tools.expires.secs'] = 0 - return "being forceful" - force.exposed = True - force._cp_config = {'tools.expires.secs': 0} - - def dynamic(self): - cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' - cherrypy.response.headers['Cache-Control'] = 'private' - return "D-d-d-dynamic!" - dynamic.exposed = True - - def cacheable(self): - cherrypy.response.headers['Etag'] = 'bibbitybobbityboo' - return "Hi, I'm cacheable." - cacheable.exposed = True - - def specific(self): - cherrypy.response.headers['Etag'] = 'need_this_to_make_me_cacheable' - return "I am being specific" - specific.exposed = True - specific._cp_config = {'tools.expires.secs': 86400} - - class Foo(object):pass - - def wrongtype(self): - cherrypy.response.headers['Etag'] = 'need_this_to_make_me_cacheable' - return "Woops" - wrongtype.exposed = True - wrongtype._cp_config = {'tools.expires.secs': Foo()} - - cherrypy.tree.mount(Root()) - cherrypy.tree.mount(UnCached(), "/expires") - cherrypy.tree.mount(VaryHeaderCachingServer(), "/varying_headers") - cherrypy.config.update({'tools.gzip.on': True}) - setup_server = staticmethod(setup_server) - - def testCaching(self): - elapsed = 0.0 - for trial in range(10): - self.getPage("/") - # The response should be the same every time, - # except for the Age response header. - self.assertBody('visit #1') - if trial != 0: - age = int(self.assertHeader("Age")) - self.assert_(age >= elapsed) - elapsed = age - - # POST, PUT, DELETE should not be cached. - self.getPage("/", method="POST") - self.assertBody('visit #2') - # Because gzip is turned on, the Vary header should always Vary for content-encoding - self.assertHeader('Vary', 'Accept-Encoding') - # The previous request should have invalidated the cache, - # so this request will recalc the response. - self.getPage("/", method="GET") - self.assertBody('visit #3') - # ...but this request should get the cached copy. - self.getPage("/", method="GET") - self.assertBody('visit #3') - self.getPage("/", method="DELETE") - self.assertBody('visit #4') - - # The previous request should have invalidated the cache, - # so this request will recalc the response. - self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')]) - self.assertHeader('Content-Encoding', 'gzip') - self.assertHeader('Vary') - self.assertEqual(cherrypy.lib.encoding.decompress(self.body), ntob("visit #5")) - - # Now check that a second request gets the gzip header and gzipped body - # This also tests a bug in 3.0 to 3.0.2 whereby the cached, gzipped - # response body was being gzipped a second time. - self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')]) - self.assertHeader('Content-Encoding', 'gzip') - self.assertEqual(cherrypy.lib.encoding.decompress(self.body), ntob("visit #5")) - - # Now check that a third request that doesn't accept gzip - # skips the cache (because the 'Vary' header denies it). - self.getPage("/", method="GET") - self.assertNoHeader('Content-Encoding') - self.assertBody('visit #6') - - def testVaryHeader(self): - self.getPage("/varying_headers/") - self.assertStatus("200 OK") - self.assertHeaderItemValue('Vary', 'Our-Varying-Header') - self.assertBody('visit #1') - - # Now check that different 'Vary'-fields don't evict each other. - # This test creates 2 requests with different 'Our-Varying-Header' - # and then tests if the first one still exists. - self.getPage("/varying_headers/", headers=[('Our-Varying-Header', 'request 2')]) - self.assertStatus("200 OK") - self.assertBody('visit #2') - - self.getPage("/varying_headers/", headers=[('Our-Varying-Header', 'request 2')]) - self.assertStatus("200 OK") - self.assertBody('visit #2') - - self.getPage("/varying_headers/") - self.assertStatus("200 OK") - self.assertBody('visit #1') - - def testExpiresTool(self): - # test setting an expires header - self.getPage("/expires/specific") - self.assertStatus("200 OK") - self.assertHeader("Expires") - - # test exceptions for bad time values - self.getPage("/expires/wrongtype") - self.assertStatus(500) - self.assertInBody("TypeError") - - # static content should not have "cache prevention" headers - self.getPage("/expires/index.html") - self.assertStatus("200 OK") - self.assertNoHeader("Pragma") - self.assertNoHeader("Cache-Control") - self.assertHeader("Expires") - - # dynamic content that sets indicators should not have - # "cache prevention" headers - self.getPage("/expires/cacheable") - self.assertStatus("200 OK") - self.assertNoHeader("Pragma") - self.assertNoHeader("Cache-Control") - self.assertHeader("Expires") - - self.getPage('/expires/dynamic') - self.assertBody("D-d-d-dynamic!") - # the Cache-Control header should be untouched - self.assertHeader("Cache-Control", "private") - self.assertHeader("Expires") - - # configure the tool to ignore indicators and replace existing headers - self.getPage("/expires/force") - self.assertStatus("200 OK") - # This also gives us a chance to test 0 expiry with no other headers - self.assertHeader("Pragma", "no-cache") - if cherrypy.server.protocol_version == "HTTP/1.1": - self.assertHeader("Cache-Control", "no-cache, must-revalidate") - self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") - - # static content should now have "cache prevention" headers - self.getPage("/expires/index.html") - self.assertStatus("200 OK") - self.assertHeader("Pragma", "no-cache") - if cherrypy.server.protocol_version == "HTTP/1.1": - self.assertHeader("Cache-Control", "no-cache, must-revalidate") - self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") - - # the cacheable handler should now have "cache prevention" headers - self.getPage("/expires/cacheable") - self.assertStatus("200 OK") - self.assertHeader("Pragma", "no-cache") - if cherrypy.server.protocol_version == "HTTP/1.1": - self.assertHeader("Cache-Control", "no-cache, must-revalidate") - self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") - - self.getPage('/expires/dynamic') - self.assertBody("D-d-d-dynamic!") - # dynamic sets Cache-Control to private but it should be - # overwritten here ... - self.assertHeader("Pragma", "no-cache") - if cherrypy.server.protocol_version == "HTTP/1.1": - self.assertHeader("Cache-Control", "no-cache, must-revalidate") - self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT") - - def testLastModified(self): - self.getPage("/a.gif") - self.assertStatus(200) - self.assertBody(gif_bytes) - lm1 = self.assertHeader("Last-Modified") - - # this request should get the cached copy. - self.getPage("/a.gif") - self.assertStatus(200) - self.assertBody(gif_bytes) - self.assertHeader("Age") - lm2 = self.assertHeader("Last-Modified") - self.assertEqual(lm1, lm2) - - # this request should match the cached copy, but raise 304. - self.getPage("/a.gif", [('If-Modified-Since', lm1)]) - self.assertStatus(304) - self.assertNoHeader("Last-Modified") - if not getattr(cherrypy.server, "using_apache", False): - self.assertHeader("Age") - - def test_antistampede(self): - SECONDS = 4 - # We MUST make an initial synchronous request in order to create the - # AntiStampedeCache object, and populate its selecting_headers, - # before the actual stampede. - self.getPage("/long_process?seconds=%d" % SECONDS) - self.assertBody('success!') - self.getPage("/clear_cache?path=" + - quote('/long_process?seconds=%d' % SECONDS, safe='')) - self.assertStatus(200) - sys.stdout.write("prepped... ") - sys.stdout.flush() - - start = datetime.datetime.now() - def run(): - self.getPage("/long_process?seconds=%d" % SECONDS) - # The response should be the same every time - self.assertBody('success!') - ts = [threading.Thread(target=run) for i in xrange(100)] - for t in ts: - t.start() - for t in ts: - t.join() - self.assertEqualDates(start, datetime.datetime.now(), - # Allow a second for our thread/TCP overhead etc. - seconds=SECONDS + 1.1) - - def test_cache_control(self): - self.getPage("/control") - self.assertBody('visit #1') - self.getPage("/control") - self.assertBody('visit #1') - - self.getPage("/control", headers=[('Cache-Control', 'no-cache')]) - self.assertBody('visit #2') - self.getPage("/control") - self.assertBody('visit #2') - - self.getPage("/control", headers=[('Pragma', 'no-cache')]) - self.assertBody('visit #3') - self.getPage("/control") - self.assertBody('visit #3') - - time.sleep(1) - self.getPage("/control", headers=[('Cache-Control', 'max-age=0')]) - self.assertBody('visit #4') - self.getPage("/control") - self.assertBody('visit #4') - diff --git a/cherrypy/test/test_config.py b/cherrypy/test/test_config.py deleted file mode 100644 index a0bd8ab9..00000000 --- a/cherrypy/test/test_config.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Tests for the CherryPy configuration system.""" - -import os, sys -localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - -from cherrypy._cpcompat import ntob, StringIO -import unittest - -import cherrypy - -def setup_server(): - - class Root: - - _cp_config = {'foo': 'this', - 'bar': 'that'} - - def __init__(self): - cherrypy.config.namespaces['db'] = self.db_namespace - - def db_namespace(self, k, v): - if k == "scheme": - self.db = v - - # @cherrypy.expose(alias=('global_', 'xyz')) - def index(self, key): - return cherrypy.request.config.get(key, "None") - index = cherrypy.expose(index, alias=('global_', 'xyz')) - - def repr(self, key): - return repr(cherrypy.request.config.get(key, None)) - repr.exposed = True - - def dbscheme(self): - return self.db - dbscheme.exposed = True - - def plain(self, x): - return x - plain.exposed = True - plain._cp_config = {'request.body.attempt_charsets': ['utf-16']} - - favicon_ico = cherrypy.tools.staticfile.handler( - filename=os.path.join(localDir, '../favicon.ico')) - - class Foo: - - _cp_config = {'foo': 'this2', - 'baz': 'that2'} - - def index(self, key): - return cherrypy.request.config.get(key, "None") - index.exposed = True - nex = index - - def silly(self): - return 'Hello world' - silly.exposed = True - silly._cp_config = {'response.headers.X-silly': 'sillyval'} - - def bar(self, key): - return repr(cherrypy.request.config.get(key, None)) - bar.exposed = True - bar._cp_config = {'foo': 'this3', 'bax': 'this4'} - - class Another: - - def index(self, key): - return str(cherrypy.request.config.get(key, "None")) - index.exposed = True - - - def raw_namespace(key, value): - if key == 'input.map': - handler = cherrypy.request.handler - def wrapper(): - params = cherrypy.request.params - for name, coercer in list(value.items()): - try: - params[name] = coercer(params[name]) - except KeyError: - pass - return handler() - cherrypy.request.handler = wrapper - elif key == 'output': - handler = cherrypy.request.handler - def wrapper(): - # 'value' is a type (like int or str). - return value(handler()) - cherrypy.request.handler = wrapper - - class Raw: - - _cp_config = {'raw.output': repr} - - def incr(self, num): - return num + 1 - incr.exposed = True - incr._cp_config = {'raw.input.map': {'num': int}} - - ioconf = StringIO(""" -[/] -neg: -1234 -filename: os.path.join(sys.prefix, "hello.py") -thing1: cherrypy.lib.httputil.response_codes[404] -thing2: __import__('cherrypy.tutorial', globals(), locals(), ['']).thing2 -complex: 3+2j -ones: "11" -twos: "22" -stradd: %%(ones)s + %%(twos)s + "33" - -[/favicon.ico] -tools.staticfile.filename = %r -""" % os.path.join(localDir, 'static/dirback.jpg')) - - root = Root() - root.foo = Foo() - root.raw = Raw() - app = cherrypy.tree.mount(root, config=ioconf) - app.request_class.namespaces['raw'] = raw_namespace - - cherrypy.tree.mount(Another(), "/another") - cherrypy.config.update({'luxuryyacht': 'throatwobblermangrove', - 'db.scheme': r"sqlite///memory", - }) - - -# Client-side code # - -from cherrypy.test import helper - -class ConfigTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testConfig(self): - tests = [ - ('/', 'nex', 'None'), - ('/', 'foo', 'this'), - ('/', 'bar', 'that'), - ('/xyz', 'foo', 'this'), - ('/foo/', 'foo', 'this2'), - ('/foo/', 'bar', 'that'), - ('/foo/', 'bax', 'None'), - ('/foo/bar', 'baz', "'that2'"), - ('/foo/nex', 'baz', 'that2'), - # If 'foo' == 'this', then the mount point '/another' leaks into '/'. - ('/another/','foo', 'None'), - ] - for path, key, expected in tests: - self.getPage(path + "?key=" + key) - self.assertBody(expected) - - expectedconf = { - # From CP defaults - 'tools.log_headers.on': False, - 'tools.log_tracebacks.on': True, - 'request.show_tracebacks': True, - 'log.screen': False, - 'environment': 'test_suite', - 'engine.autoreload_on': False, - # From global config - 'luxuryyacht': 'throatwobblermangrove', - # From Root._cp_config - 'bar': 'that', - # From Foo._cp_config - 'baz': 'that2', - # From Foo.bar._cp_config - 'foo': 'this3', - 'bax': 'this4', - } - for key, expected in expectedconf.items(): - self.getPage("/foo/bar?key=" + key) - self.assertBody(repr(expected)) - - def testUnrepr(self): - self.getPage("/repr?key=neg") - self.assertBody("-1234") - - self.getPage("/repr?key=filename") - self.assertBody(repr(os.path.join(sys.prefix, "hello.py"))) - - self.getPage("/repr?key=thing1") - self.assertBody(repr(cherrypy.lib.httputil.response_codes[404])) - - if not getattr(cherrypy.server, "using_apache", False): - # The object ID's won't match up when using Apache, since the - # server and client are running in different processes. - self.getPage("/repr?key=thing2") - from cherrypy.tutorial import thing2 - self.assertBody(repr(thing2)) - - self.getPage("/repr?key=complex") - self.assertBody("(3+2j)") - - self.getPage("/repr?key=stradd") - self.assertBody(repr("112233")) - - def testRespNamespaces(self): - self.getPage("/foo/silly") - self.assertHeader('X-silly', 'sillyval') - self.assertBody('Hello world') - - def testCustomNamespaces(self): - self.getPage("/raw/incr?num=12") - self.assertBody("13") - - self.getPage("/dbscheme") - self.assertBody(r"sqlite///memory") - - def testHandlerToolConfigOverride(self): - # Assert that config overrides tool constructor args. Above, we set - # the favicon in the page handler to be '../favicon.ico', - # but then overrode it in config to be './static/dirback.jpg'. - self.getPage("/favicon.ico") - self.assertBody(open(os.path.join(localDir, "static/dirback.jpg"), - "rb").read()) - - def test_request_body_namespace(self): - self.getPage("/plain", method='POST', headers=[ - ('Content-Type', 'application/x-www-form-urlencoded'), - ('Content-Length', '13')], - body=ntob('\xff\xfex\x00=\xff\xfea\x00b\x00c\x00')) - self.assertBody("abc") - - -class VariableSubstitutionTests(unittest.TestCase): - setup_server = staticmethod(setup_server) - - def test_config(self): - from textwrap import dedent - - # variable substitution with [DEFAULT] - conf = dedent(""" - [DEFAULT] - dir = "/some/dir" - my.dir = %(dir)s + "/sub" - - [my] - my.dir = %(dir)s + "/my/dir" - my.dir2 = %(my.dir)s + '/dir2' - - """) - - fp = StringIO(conf) - - cherrypy.config.update(fp) - self.assertEqual(cherrypy.config["my"]["my.dir"], "/some/dir/my/dir") - self.assertEqual(cherrypy.config["my"]["my.dir2"], "/some/dir/my/dir/dir2") - diff --git a/cherrypy/test/test_config_server.py b/cherrypy/test/test_config_server.py deleted file mode 100644 index 0b9718da..00000000 --- a/cherrypy/test/test_config_server.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests for the CherryPy configuration system.""" - -import os, sys -localDir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import socket -import time - -import cherrypy - - -# Client-side code # - -from cherrypy.test import helper - -class ServerConfigTests(helper.CPWebCase): - - def setup_server(): - - class Root: - def index(self): - return cherrypy.request.wsgi_environ['SERVER_PORT'] - index.exposed = True - - def upload(self, file): - return "Size: %s" % len(file.file.read()) - upload.exposed = True - - def tinyupload(self): - return cherrypy.request.body.read() - tinyupload.exposed = True - tinyupload._cp_config = {'request.body.maxbytes': 100} - - cherrypy.tree.mount(Root()) - - cherrypy.config.update({ - 'server.socket_host': '0.0.0.0', - 'server.socket_port': 9876, - 'server.max_request_body_size': 200, - 'server.max_request_header_size': 500, - 'server.socket_timeout': 0.5, - - # Test explicit server.instance - 'server.2.instance': 'cherrypy._cpwsgi_server.CPWSGIServer', - 'server.2.socket_port': 9877, - - # Test non-numeric - # Also test default server.instance = builtin server - 'server.yetanother.socket_port': 9878, - }) - setup_server = staticmethod(setup_server) - - PORT = 9876 - - def testBasicConfig(self): - self.getPage("/") - self.assertBody(str(self.PORT)) - - def testAdditionalServers(self): - if self.scheme == 'https': - return self.skip("not available under ssl") - self.PORT = 9877 - self.getPage("/") - self.assertBody(str(self.PORT)) - self.PORT = 9878 - self.getPage("/") - self.assertBody(str(self.PORT)) - - def testMaxRequestSizePerHandler(self): - if getattr(cherrypy.server, "using_apache", False): - return self.skip("skipped due to known Apache differences... ") - - self.getPage('/tinyupload', method="POST", - headers=[('Content-Type', 'text/plain'), - ('Content-Length', '100')], - body="x" * 100) - self.assertStatus(200) - self.assertBody("x" * 100) - - self.getPage('/tinyupload', method="POST", - headers=[('Content-Type', 'text/plain'), - ('Content-Length', '101')], - body="x" * 101) - self.assertStatus(413) - - def testMaxRequestSize(self): - if getattr(cherrypy.server, "using_apache", False): - return self.skip("skipped due to known Apache differences... ") - - for size in (500, 5000, 50000): - self.getPage("/", headers=[('From', "x" * 500)]) - self.assertStatus(413) - - # Test for http://www.cherrypy.org/ticket/421 - # (Incorrect border condition in readline of SizeCheckWrapper). - # This hangs in rev 891 and earlier. - lines256 = "x" * 248 - self.getPage("/", - headers=[('Host', '%s:%s' % (self.HOST, self.PORT)), - ('From', lines256)]) - - # Test upload - body = '\r\n'.join([ - '--x', - 'Content-Disposition: form-data; name="file"; filename="hello.txt"', - 'Content-Type: text/plain', - '', - '%s', - '--x--']) - partlen = 200 - len(body) - b = body % ("x" * partlen) - h = [("Content-type", "multipart/form-data; boundary=x"), - ("Content-Length", "%s" % len(b))] - self.getPage('/upload', h, "POST", b) - self.assertBody('Size: %d' % partlen) - - b = body % ("x" * 200) - h = [("Content-type", "multipart/form-data; boundary=x"), - ("Content-Length", "%s" % len(b))] - self.getPage('/upload', h, "POST", b) - self.assertStatus(413) - diff --git a/cherrypy/test/test_conn.py b/cherrypy/test/test_conn.py deleted file mode 100644 index 1346f593..00000000 --- a/cherrypy/test/test_conn.py +++ /dev/null @@ -1,734 +0,0 @@ -"""Tests for TCP connection handling, including proper and timely close.""" - -import socket -import sys -import time -timeout = 1 - - -import cherrypy -from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, NotConnected, BadStatusLine -from cherrypy._cpcompat import ntob, urlopen, unicodestr -from cherrypy.test import webtest -from cherrypy import _cperror - - -pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN' - -def setup_server(): - - def raise500(): - raise cherrypy.HTTPError(500) - - class Root: - - def index(self): - return pov - index.exposed = True - page1 = index - page2 = index - page3 = index - - def hello(self): - return "Hello, world!" - hello.exposed = True - - def timeout(self, t): - return str(cherrypy.server.httpserver.timeout) - timeout.exposed = True - - def stream(self, set_cl=False): - if set_cl: - cherrypy.response.headers['Content-Length'] = 10 - - def content(): - for x in range(10): - yield str(x) - - return content() - stream.exposed = True - stream._cp_config = {'response.stream': True} - - def error(self, code=500): - raise cherrypy.HTTPError(code) - error.exposed = True - - def upload(self): - if not cherrypy.request.method == 'POST': - raise AssertionError("'POST' != request.method %r" % - cherrypy.request.method) - return "thanks for '%s'" % cherrypy.request.body.read() - upload.exposed = True - - def custom(self, response_code): - cherrypy.response.status = response_code - return "Code = %s" % response_code - custom.exposed = True - - def err_before_read(self): - return "ok" - err_before_read.exposed = True - err_before_read._cp_config = {'hooks.on_start_resource': raise500} - - def one_megabyte_of_a(self): - return ["a" * 1024] * 1024 - one_megabyte_of_a.exposed = True - - def custom_cl(self, body, cl): - cherrypy.response.headers['Content-Length'] = cl - if not isinstance(body, list): - body = [body] - newbody = [] - for chunk in body: - if isinstance(chunk, unicodestr): - chunk = chunk.encode('ISO-8859-1') - newbody.append(chunk) - return newbody - custom_cl.exposed = True - # Turn off the encoding tool so it doens't collapse - # our response body and reclaculate the Content-Length. - custom_cl._cp_config = {'tools.encode.on': False} - - cherrypy.tree.mount(Root()) - cherrypy.config.update({ - 'server.max_request_body_size': 1001, - 'server.socket_timeout': timeout, - }) - - -from cherrypy.test import helper - -class ConnectionCloseTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_HTTP11(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - self.persistent = True - - # Make the first request and assert there's no "Connection: close". - self.getPage("/") - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader("Connection") - - # Make another request on the same connection. - self.getPage("/page1") - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader("Connection") - - # Test client-side close. - self.getPage("/page2", headers=[("Connection", "close")]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader("Connection", "close") - - # Make another request on the same connection, which should error. - self.assertRaises(NotConnected, self.getPage, "/") - - def test_Streaming_no_len(self): - self._streaming(set_cl=False) - - def test_Streaming_with_len(self): - self._streaming(set_cl=True) - - def _streaming(self, set_cl): - if cherrypy.server.protocol_version == "HTTP/1.1": - self.PROTOCOL = "HTTP/1.1" - - self.persistent = True - - # Make the first request and assert there's no "Connection: close". - self.getPage("/") - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader("Connection") - - # Make another, streamed request on the same connection. - if set_cl: - # When a Content-Length is provided, the content should stream - # without closing the connection. - self.getPage("/stream?set_cl=Yes") - self.assertHeader("Content-Length") - self.assertNoHeader("Connection", "close") - self.assertNoHeader("Transfer-Encoding") - - self.assertStatus('200 OK') - self.assertBody('0123456789') - else: - # When no Content-Length response header is provided, - # streamed output will either close the connection, or use - # chunked encoding, to determine transfer-length. - self.getPage("/stream") - self.assertNoHeader("Content-Length") - self.assertStatus('200 OK') - self.assertBody('0123456789') - - chunked_response = False - for k, v in self.headers: - if k.lower() == "transfer-encoding": - if str(v) == "chunked": - chunked_response = True - - if chunked_response: - self.assertNoHeader("Connection", "close") - else: - self.assertHeader("Connection", "close") - - # Make another request on the same connection, which should error. - self.assertRaises(NotConnected, self.getPage, "/") - - # Try HEAD. See http://www.cherrypy.org/ticket/864. - self.getPage("/stream", method='HEAD') - self.assertStatus('200 OK') - self.assertBody('') - self.assertNoHeader("Transfer-Encoding") - else: - self.PROTOCOL = "HTTP/1.0" - - self.persistent = True - - # Make the first request and assert Keep-Alive. - self.getPage("/", headers=[("Connection", "Keep-Alive")]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader("Connection", "Keep-Alive") - - # Make another, streamed request on the same connection. - if set_cl: - # When a Content-Length is provided, the content should - # stream without closing the connection. - self.getPage("/stream?set_cl=Yes", - headers=[("Connection", "Keep-Alive")]) - self.assertHeader("Content-Length") - self.assertHeader("Connection", "Keep-Alive") - self.assertNoHeader("Transfer-Encoding") - self.assertStatus('200 OK') - self.assertBody('0123456789') - else: - # When a Content-Length is not provided, - # the server should close the connection. - self.getPage("/stream", headers=[("Connection", "Keep-Alive")]) - self.assertStatus('200 OK') - self.assertBody('0123456789') - - self.assertNoHeader("Content-Length") - self.assertNoHeader("Connection", "Keep-Alive") - self.assertNoHeader("Transfer-Encoding") - - # Make another request on the same connection, which should error. - self.assertRaises(NotConnected, self.getPage, "/") - - def test_HTTP10_KeepAlive(self): - self.PROTOCOL = "HTTP/1.0" - if self.scheme == "https": - self.HTTP_CONN = HTTPSConnection - else: - self.HTTP_CONN = HTTPConnection - - # Test a normal HTTP/1.0 request. - self.getPage("/page2") - self.assertStatus('200 OK') - self.assertBody(pov) - # Apache, for example, may emit a Connection header even for HTTP/1.0 -## self.assertNoHeader("Connection") - - # Test a keep-alive HTTP/1.0 request. - self.persistent = True - - self.getPage("/page3", headers=[("Connection", "Keep-Alive")]) - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertHeader("Connection", "Keep-Alive") - - # Remove the keep-alive header again. - self.getPage("/page3") - self.assertStatus('200 OK') - self.assertBody(pov) - # Apache, for example, may emit a Connection header even for HTTP/1.0 -## self.assertNoHeader("Connection") - - -class PipelineTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_HTTP11_Timeout(self): - # If we timeout without sending any data, - # the server will close the conn with a 408. - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Connect but send nothing. - self.persistent = True - conn = self.HTTP_CONN - conn.auto_open = False - conn.connect() - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # The request should have returned 408 already. - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 408) - conn.close() - - # Connect but send half the headers only. - self.persistent = True - conn = self.HTTP_CONN - conn.auto_open = False - conn.connect() - conn.send(ntob('GET /hello HTTP/1.1')) - conn.send(("Host: %s" % self.HOST).encode('ascii')) - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # The conn should have already sent 408. - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 408) - conn.close() - - def test_HTTP11_Timeout_after_request(self): - # If we timeout after at least one request has succeeded, - # the server will close the conn without 408. - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Make an initial request - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/timeout?t=%s" % timeout, skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(str(timeout)) - - # Make a second request on the same socket - conn._output(ntob('GET /hello HTTP/1.1')) - conn._output(ntob("Host: %s" % self.HOST, 'ascii')) - conn._send_output() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody("Hello, world!") - - # Wait for our socket timeout - time.sleep(timeout * 2) - - # Make another request on the same socket, which should error - conn._output(ntob('GET /hello HTTP/1.1')) - conn._output(ntob("Host: %s" % self.HOST, 'ascii')) - conn._send_output() - response = conn.response_class(conn.sock, method="GET") - try: - response.begin() - except: - if not isinstance(sys.exc_info()[1], - (socket.error, BadStatusLine)): - self.fail("Writing to timed out socket didn't fail" - " as it should have: %s" % sys.exc_info()[1]) - else: - if response.status != 408: - self.fail("Writing to timed out socket didn't fail" - " as it should have: %s" % - response.read()) - - conn.close() - - # Make another request on a new socket, which should work - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(pov) - - - # Make another request on the same socket, - # but timeout on the headers - conn.send(ntob('GET /hello HTTP/1.1')) - # Wait for our socket timeout - time.sleep(timeout * 2) - response = conn.response_class(conn.sock, method="GET") - try: - response.begin() - except: - if not isinstance(sys.exc_info()[1], - (socket.error, BadStatusLine)): - self.fail("Writing to timed out socket didn't fail" - " as it should have: %s" % sys.exc_info()[1]) - else: - self.fail("Writing to timed out socket didn't fail" - " as it should have: %s" % - response.read()) - - conn.close() - - # Retry the request on a new connection, which should work - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(pov) - conn.close() - - def test_HTTP11_pipelining(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Test pipelining. httplib doesn't support this directly. - self.persistent = True - conn = self.HTTP_CONN - - # Put request 1 - conn.putrequest("GET", "/hello", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - - for trial in range(5): - # Put next request - conn._output(ntob('GET /hello HTTP/1.1')) - conn._output(ntob("Host: %s" % self.HOST, 'ascii')) - conn._send_output() - - # Retrieve previous response - response = conn.response_class(conn.sock, method="GET") - response.begin() - body = response.read(13) - self.assertEqual(response.status, 200) - self.assertEqual(body, ntob("Hello, world!")) - - # Retrieve final response - response = conn.response_class(conn.sock, method="GET") - response.begin() - body = response.read() - self.assertEqual(response.status, 200) - self.assertEqual(body, ntob("Hello, world!")) - - conn.close() - - def test_100_Continue(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - self.persistent = True - conn = self.HTTP_CONN - - # Try a page without an Expect request header first. - # Note that httplib's response.begin automatically ignores - # 100 Continue responses, so we must manually check for it. - conn.putrequest("POST", "/upload", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Content-Type", "text/plain") - conn.putheader("Content-Length", "4") - conn.endheaders() - conn.send(ntob("d'oh")) - response = conn.response_class(conn.sock, method="POST") - version, status, reason = response._read_status() - self.assertNotEqual(status, 100) - conn.close() - - # Now try a page with an Expect header... - conn.connect() - conn.putrequest("POST", "/upload", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Content-Type", "text/plain") - conn.putheader("Content-Length", "17") - conn.putheader("Expect", "100-continue") - conn.endheaders() - response = conn.response_class(conn.sock, method="POST") - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - line = response.fp.readline().strip() - if line: - self.fail("100 Continue should not output any headers. Got %r" % line) - else: - break - - # ...send the body - body = ntob("I am a small file") - conn.send(body) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody("thanks for '%s'" % body) - conn.close() - - -class ConnectionTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_readall_or_close(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - if self.scheme == "https": - self.HTTP_CONN = HTTPSConnection - else: - self.HTTP_CONN = HTTPConnection - - # Test a max of 0 (the default) and then reset to what it was above. - old_max = cherrypy.server.max_request_body_size - for new_max in (0, old_max): - cherrypy.server.max_request_body_size = new_max - - self.persistent = True - conn = self.HTTP_CONN - - # Get a POST page with an error - conn.putrequest("POST", "/err_before_read", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Content-Type", "text/plain") - conn.putheader("Content-Length", "1000") - conn.putheader("Expect", "100-continue") - conn.endheaders() - response = conn.response_class(conn.sock, method="POST") - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - skip = response.fp.readline().strip() - if not skip: - break - - # ...send the body - conn.send(ntob("x" * 1000)) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(500) - - # Now try a working page with an Expect header... - conn._output(ntob('POST /upload HTTP/1.1')) - conn._output(ntob("Host: %s" % self.HOST, 'ascii')) - conn._output(ntob("Content-Type: text/plain")) - conn._output(ntob("Content-Length: 17")) - conn._output(ntob("Expect: 100-continue")) - conn._send_output() - response = conn.response_class(conn.sock, method="POST") - - # ...assert and then skip the 100 response - version, status, reason = response._read_status() - self.assertEqual(status, 100) - while True: - skip = response.fp.readline().strip() - if not skip: - break - - # ...send the body - body = ntob("I am a small file") - conn.send(body) - - # ...get the final response - response.begin() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody("thanks for '%s'" % body) - conn.close() - - def test_No_Message_Body(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Set our HTTP_CONN to an instance so it persists between requests. - self.persistent = True - - # Make the first request and assert there's no "Connection: close". - self.getPage("/") - self.assertStatus('200 OK') - self.assertBody(pov) - self.assertNoHeader("Connection") - - # Make a 204 request on the same connection. - self.getPage("/custom/204") - self.assertStatus(204) - self.assertNoHeader("Content-Length") - self.assertBody("") - self.assertNoHeader("Connection") - - # Make a 304 request on the same connection. - self.getPage("/custom/304") - self.assertStatus(304) - self.assertNoHeader("Content-Length") - self.assertBody("") - self.assertNoHeader("Connection") - - def test_Chunked_Encoding(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - if (hasattr(self, 'harness') and - "modpython" in self.harness.__class__.__name__.lower()): - # mod_python forbids chunked encoding - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Set our HTTP_CONN to an instance so it persists between requests. - self.persistent = True - conn = self.HTTP_CONN - - # Try a normal chunked request (with extensions) - body = ntob("8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n" - "Content-Type: application/json\r\n" - "\r\n") - conn.putrequest("POST", "/upload", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Transfer-Encoding", "chunked") - conn.putheader("Trailer", "Content-Type") - # Note that this is somewhat malformed: - # we shouldn't be sending Content-Length. - # RFC 2616 says the server should ignore it. - conn.putheader("Content-Length", "3") - conn.endheaders() - conn.send(body) - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus('200 OK') - self.assertBody("thanks for '%s'" % ntob('xx\r\nxxxxyyyyy')) - - # Try a chunked request that exceeds server.max_request_body_size. - # Note that the delimiters and trailer are included. - body = ntob("3e3\r\n" + ("x" * 995) + "\r\n0\r\n\r\n") - conn.putrequest("POST", "/upload", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Transfer-Encoding", "chunked") - conn.putheader("Content-Type", "text/plain") - # Chunked requests don't need a content-length -## conn.putheader("Content-Length", len(body)) - conn.endheaders() - conn.send(body) - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(413) - conn.close() - - def test_Content_Length_in(self): - # Try a non-chunked request where Content-Length exceeds - # server.max_request_body_size. Assert error before body send. - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("POST", "/upload", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader("Content-Type", "text/plain") - conn.putheader("Content-Length", "9999") - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(413) - self.assertBody("The entity sent with the request exceeds " - "the maximum allowed bytes.") - conn.close() - - def test_Content_Length_out_preheaders(self): - # Try a non-chunked response where Content-Length is less than - # the actual bytes in the response body. - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/custom_cl?body=I+have+too+many+bytes&cl=5", - skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(500) - self.assertBody( - "The requested resource returned more bytes than the " - "declared Content-Length.") - conn.close() - - def test_Content_Length_out_postheaders(self): - # Try a non-chunked response where Content-Length is less than - # the actual bytes in the response body. - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/custom_cl?body=I+too&body=+have+too+many&cl=5", - skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.getresponse() - self.status, self.headers, self.body = webtest.shb(response) - self.assertStatus(200) - self.assertBody("I too") - conn.close() - - def test_598(self): - remote_data_conn = urlopen('%s://%s:%s/one_megabyte_of_a/' % - (self.scheme, self.HOST, self.PORT,)) - buf = remote_data_conn.read(512) - time.sleep(timeout * 0.6) - remaining = (1024 * 1024) - 512 - while remaining: - data = remote_data_conn.read(remaining) - if not data: - break - else: - buf += data - remaining -= len(data) - - self.assertEqual(len(buf), 1024 * 1024) - self.assertEqual(buf, ntob("a" * 1024 * 1024)) - self.assertEqual(remaining, 0) - remote_data_conn.close() - - -class BadRequestTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_No_CRLF(self): - self.persistent = True - - conn = self.HTTP_CONN - conn.send(ntob('GET /hello HTTP/1.1\n\n')) - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.body = response.read() - self.assertBody("HTTP requires CRLF terminators") - conn.close() - - conn.connect() - conn.send(ntob('GET /hello HTTP/1.1\r\n\n')) - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.body = response.read() - self.assertBody("HTTP requires CRLF terminators") - conn.close() - diff --git a/cherrypy/test/test_core.py b/cherrypy/test/test_core.py deleted file mode 100644 index 09544e34..00000000 --- a/cherrypy/test/test_core.py +++ /dev/null @@ -1,617 +0,0 @@ -"""Basic tests for the CherryPy core: request handling.""" - -import os -localDir = os.path.dirname(__file__) -import sys -import types - -import cherrypy -from cherrypy._cpcompat import IncompleteRead, itervalues, ntob -from cherrypy import _cptools, tools -from cherrypy.lib import httputil, static - - -favicon_path = os.path.join(os.getcwd(), localDir, "../favicon.ico") - -# Client-side code # - -from cherrypy.test import helper - -class CoreRequestHandlingTest(helper.CPWebCase): - - def setup_server(): - class Root: - - def index(self): - return "hello" - index.exposed = True - - favicon_ico = tools.staticfile.handler(filename=favicon_path) - - def defct(self, newct): - newct = "text/%s" % newct - cherrypy.config.update({'tools.response_headers.on': True, - 'tools.response_headers.headers': - [('Content-Type', newct)]}) - defct.exposed = True - - def baseurl(self, path_info, relative=None): - return cherrypy.url(path_info, relative=bool(relative)) - baseurl.exposed = True - - root = Root() - - if sys.version_info >= (2, 5): - from cherrypy.test._test_decorators import ExposeExamples - root.expose_dec = ExposeExamples() - - - class TestType(type): - """Metaclass which automatically exposes all functions in each subclass, - and adds an instance of the subclass as an attribute of root. - """ - def __init__(cls, name, bases, dct): - type.__init__(cls, name, bases, dct) - for value in itervalues(dct): - if isinstance(value, types.FunctionType): - value.exposed = True - setattr(root, name.lower(), cls()) - class Test(object): - __metaclass__ = TestType - - - class URL(Test): - - _cp_config = {'tools.trailing_slash.on': False} - - def index(self, path_info, relative=None): - if relative != 'server': - relative = bool(relative) - return cherrypy.url(path_info, relative=relative) - - def leaf(self, path_info, relative=None): - if relative != 'server': - relative = bool(relative) - return cherrypy.url(path_info, relative=relative) - - - class Status(Test): - - def index(self): - return "normal" - - def blank(self): - cherrypy.response.status = "" - - # According to RFC 2616, new status codes are OK as long as they - # are between 100 and 599. - - # Here is an illegal code... - def illegal(self): - cherrypy.response.status = 781 - return "oops" - - # ...and here is an unknown but legal code. - def unknown(self): - cherrypy.response.status = "431 My custom error" - return "funky" - - # Non-numeric code - def bad(self): - cherrypy.response.status = "error" - return "bad news" - - - class Redirect(Test): - - class Error: - _cp_config = {"tools.err_redirect.on": True, - "tools.err_redirect.url": "/errpage", - "tools.err_redirect.internal": False, - } - - def index(self): - raise NameError("redirect_test") - index.exposed = True - error = Error() - - def index(self): - return "child" - - def custom(self, url, code): - raise cherrypy.HTTPRedirect(url, code) - - def by_code(self, code): - raise cherrypy.HTTPRedirect("somewhere%20else", code) - by_code._cp_config = {'tools.trailing_slash.extra': True} - - def nomodify(self): - raise cherrypy.HTTPRedirect("", 304) - - def proxy(self): - raise cherrypy.HTTPRedirect("proxy", 305) - - def stringify(self): - return str(cherrypy.HTTPRedirect("/")) - - def fragment(self, frag): - raise cherrypy.HTTPRedirect("/some/url#%s" % frag) - - def login_redir(): - if not getattr(cherrypy.request, "login", None): - raise cherrypy.InternalRedirect("/internalredirect/login") - tools.login_redir = _cptools.Tool('before_handler', login_redir) - - def redir_custom(): - raise cherrypy.InternalRedirect("/internalredirect/custom_err") - - class InternalRedirect(Test): - - def index(self): - raise cherrypy.InternalRedirect("/") - - def choke(self): - return 3 / 0 - choke.exposed = True - choke._cp_config = {'hooks.before_error_response': redir_custom} - - def relative(self, a, b): - raise cherrypy.InternalRedirect("cousin?t=6") - - def cousin(self, t): - assert cherrypy.request.prev.closed - return cherrypy.request.prev.query_string - - def petshop(self, user_id): - if user_id == "parrot": - # Trade it for a slug when redirecting - raise cherrypy.InternalRedirect('/image/getImagesByUser?user_id=slug') - elif user_id == "terrier": - # Trade it for a fish when redirecting - raise cherrypy.InternalRedirect('/image/getImagesByUser?user_id=fish') - else: - # This should pass the user_id through to getImagesByUser - raise cherrypy.InternalRedirect( - '/image/getImagesByUser?user_id=%s' % str(user_id)) - - # We support Python 2.3, but the @-deco syntax would look like this: - # @tools.login_redir() - def secure(self): - return "Welcome!" - secure = tools.login_redir()(secure) - # Since calling the tool returns the same function you pass in, - # you could skip binding the return value, and just write: - # tools.login_redir()(secure) - - def login(self): - return "Please log in" - - def custom_err(self): - return "Something went horribly wrong." - - def early_ir(self, arg): - return "whatever" - early_ir._cp_config = {'hooks.before_request_body': redir_custom} - - - class Image(Test): - - def getImagesByUser(self, user_id): - return "0 images for %s" % user_id - - - class Flatten(Test): - - def as_string(self): - return "content" - - def as_list(self): - return ["con", "tent"] - - def as_yield(self): - yield ntob("content") - - def as_dblyield(self): - yield self.as_yield() - as_dblyield._cp_config = {'tools.flatten.on': True} - - def as_refyield(self): - for chunk in self.as_yield(): - yield chunk - - - class Ranges(Test): - - def get_ranges(self, bytes): - return repr(httputil.get_ranges('bytes=%s' % bytes, 8)) - - def slice_file(self): - path = os.path.join(os.getcwd(), os.path.dirname(__file__)) - return static.serve_file(os.path.join(path, "static/index.html")) - - - class Cookies(Test): - - def single(self, name): - cookie = cherrypy.request.cookie[name] - # Python2's SimpleCookie.__setitem__ won't take unicode keys. - cherrypy.response.cookie[str(name)] = cookie.value - - def multiple(self, names): - for name in names: - cookie = cherrypy.request.cookie[name] - # Python2's SimpleCookie.__setitem__ won't take unicode keys. - cherrypy.response.cookie[str(name)] = cookie.value - - - cherrypy.tree.mount(root) - setup_server = staticmethod(setup_server) - - - def testStatus(self): - self.getPage("/status/") - self.assertBody('normal') - self.assertStatus(200) - - self.getPage("/status/blank") - self.assertBody('') - self.assertStatus(200) - - self.getPage("/status/illegal") - self.assertStatus(500) - msg = "Illegal response status from server (781 is out of range)." - self.assertErrorPage(500, msg) - - if not getattr(cherrypy.server, 'using_apache', False): - self.getPage("/status/unknown") - self.assertBody('funky') - self.assertStatus(431) - - self.getPage("/status/bad") - self.assertStatus(500) - msg = "Illegal response status from server ('error' is non-numeric)." - self.assertErrorPage(500, msg) - - def testSlashes(self): - # Test that requests for index methods without a trailing slash - # get redirected to the same URI path with a trailing slash. - # Make sure GET params are preserved. - self.getPage("/redirect?id=3") - self.assertStatus(301) - self.assertInBody("" - "%s/redirect/?id=3" % (self.base(), self.base())) - - if self.prefix(): - # Corner case: the "trailing slash" redirect could be tricky if - # we're using a virtual root and the URI is "/vroot" (no slash). - self.getPage("") - self.assertStatus(301) - self.assertInBody("%s/" % - (self.base(), self.base())) - - # Test that requests for NON-index methods WITH a trailing slash - # get redirected to the same URI path WITHOUT a trailing slash. - # Make sure GET params are preserved. - self.getPage("/redirect/by_code/?code=307") - self.assertStatus(301) - self.assertInBody("" - "%s/redirect/by_code?code=307" - % (self.base(), self.base())) - - # If the trailing_slash tool is off, CP should just continue - # as if the slashes were correct. But it needs some help - # inside cherrypy.url to form correct output. - self.getPage('/url?path_info=page1') - self.assertBody('%s/url/page1' % self.base()) - self.getPage('/url/leaf/?path_info=page1') - self.assertBody('%s/url/page1' % self.base()) - - def testRedirect(self): - self.getPage("/redirect/") - self.assertBody('child') - self.assertStatus(200) - - self.getPage("/redirect/by_code?code=300") - self.assertMatchesBody(r"\1somewhere%20else") - self.assertStatus(300) - - self.getPage("/redirect/by_code?code=301") - self.assertMatchesBody(r"\1somewhere%20else") - self.assertStatus(301) - - self.getPage("/redirect/by_code?code=302") - self.assertMatchesBody(r"\1somewhere%20else") - self.assertStatus(302) - - self.getPage("/redirect/by_code?code=303") - self.assertMatchesBody(r"\1somewhere%20else") - self.assertStatus(303) - - self.getPage("/redirect/by_code?code=307") - self.assertMatchesBody(r"\1somewhere%20else") - self.assertStatus(307) - - self.getPage("/redirect/nomodify") - self.assertBody('') - self.assertStatus(304) - - self.getPage("/redirect/proxy") - self.assertBody('') - self.assertStatus(305) - - # HTTPRedirect on error - self.getPage("/redirect/error/") - self.assertStatus(('302 Found', '303 See Other')) - self.assertInBody('/errpage') - - # Make sure str(HTTPRedirect()) works. - self.getPage("/redirect/stringify", protocol="HTTP/1.0") - self.assertStatus(200) - self.assertBody("(['%s/'], 302)" % self.base()) - if cherrypy.server.protocol_version == "HTTP/1.1": - self.getPage("/redirect/stringify", protocol="HTTP/1.1") - self.assertStatus(200) - self.assertBody("(['%s/'], 303)" % self.base()) - - # check that #fragments are handled properly - # http://skrb.org/ietf/http_errata.html#location-fragments - frag = "foo" - self.getPage("/redirect/fragment/%s" % frag) - self.assertMatchesBody(r"\1\/some\/url\#%s" % (frag, frag)) - loc = self.assertHeader('Location') - assert loc.endswith("#%s" % frag) - self.assertStatus(('302 Found', '303 See Other')) - - # check injection protection - # See http://www.cherrypy.org/ticket/1003 - self.getPage("/redirect/custom?code=303&url=/foobar/%0d%0aSet-Cookie:%20somecookie=someval") - self.assertStatus(303) - loc = self.assertHeader('Location') - assert 'Set-Cookie' in loc - self.assertNoHeader('Set-Cookie') - - def test_InternalRedirect(self): - # InternalRedirect - self.getPage("/internalredirect/") - self.assertBody('hello') - self.assertStatus(200) - - # Test passthrough - self.getPage("/internalredirect/petshop?user_id=Sir-not-appearing-in-this-film") - self.assertBody('0 images for Sir-not-appearing-in-this-film') - self.assertStatus(200) - - # Test args - self.getPage("/internalredirect/petshop?user_id=parrot") - self.assertBody('0 images for slug') - self.assertStatus(200) - - # Test POST - self.getPage("/internalredirect/petshop", method="POST", - body="user_id=terrier") - self.assertBody('0 images for fish') - self.assertStatus(200) - - # Test ir before body read - self.getPage("/internalredirect/early_ir", method="POST", - body="arg=aha!") - self.assertBody("Something went horribly wrong.") - self.assertStatus(200) - - self.getPage("/internalredirect/secure") - self.assertBody('Please log in') - self.assertStatus(200) - - # Relative path in InternalRedirect. - # Also tests request.prev. - self.getPage("/internalredirect/relative?a=3&b=5") - self.assertBody("a=3&b=5") - self.assertStatus(200) - - # InternalRedirect on error - self.getPage("/internalredirect/choke") - self.assertStatus(200) - self.assertBody("Something went horribly wrong.") - - def testFlatten(self): - for url in ["/flatten/as_string", "/flatten/as_list", - "/flatten/as_yield", "/flatten/as_dblyield", - "/flatten/as_refyield"]: - self.getPage(url) - self.assertBody('content') - - def testRanges(self): - self.getPage("/ranges/get_ranges?bytes=3-6") - self.assertBody("[(3, 7)]") - - # Test multiple ranges and a suffix-byte-range-spec, for good measure. - self.getPage("/ranges/get_ranges?bytes=2-4,-1") - self.assertBody("[(2, 5), (7, 8)]") - - # Get a partial file. - if cherrypy.server.protocol_version == "HTTP/1.1": - self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')]) - self.assertStatus(206) - self.assertHeader("Content-Type", "text/html;charset=utf-8") - self.assertHeader("Content-Range", "bytes 2-5/14") - self.assertBody("llo,") - - # What happens with overlapping ranges (and out of order, too)? - self.getPage("/ranges/slice_file", [('Range', 'bytes=4-6,2-5')]) - self.assertStatus(206) - ct = self.assertHeader("Content-Type") - expected_type = "multipart/byteranges; boundary=" - self.assert_(ct.startswith(expected_type)) - boundary = ct[len(expected_type):] - expected_body = ("\r\n--%s\r\n" - "Content-type: text/html\r\n" - "Content-range: bytes 4-6/14\r\n" - "\r\n" - "o, \r\n" - "--%s\r\n" - "Content-type: text/html\r\n" - "Content-range: bytes 2-5/14\r\n" - "\r\n" - "llo,\r\n" - "--%s--\r\n" % (boundary, boundary, boundary)) - self.assertBody(expected_body) - self.assertHeader("Content-Length") - - # Test "416 Requested Range Not Satisfiable" - self.getPage("/ranges/slice_file", [('Range', 'bytes=2300-2900')]) - self.assertStatus(416) - # "When this status code is returned for a byte-range request, - # the response SHOULD include a Content-Range entity-header - # field specifying the current length of the selected resource" - self.assertHeader("Content-Range", "bytes */14") - elif cherrypy.server.protocol_version == "HTTP/1.0": - # Test Range behavior with HTTP/1.0 request - self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')]) - self.assertStatus(200) - self.assertBody("Hello, world\r\n") - - def testFavicon(self): - # favicon.ico is served by staticfile. - icofilename = os.path.join(localDir, "../favicon.ico") - icofile = open(icofilename, "rb") - data = icofile.read() - icofile.close() - - self.getPage("/favicon.ico") - self.assertBody(data) - - def testCookies(self): - if sys.version_info >= (2, 5): - header_value = lambda x: x - else: - header_value = lambda x: x+';' - - self.getPage("/cookies/single?name=First", - [('Cookie', 'First=Dinsdale;')]) - self.assertHeader('Set-Cookie', header_value('First=Dinsdale')) - - self.getPage("/cookies/multiple?names=First&names=Last", - [('Cookie', 'First=Dinsdale; Last=Piranha;'), - ]) - self.assertHeader('Set-Cookie', header_value('First=Dinsdale')) - self.assertHeader('Set-Cookie', header_value('Last=Piranha')) - - self.getPage("/cookies/single?name=Something-With:Colon", - [('Cookie', 'Something-With:Colon=some-value')]) - self.assertStatus(400) - - def testDefaultContentType(self): - self.getPage('/') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.getPage('/defct/plain') - self.getPage('/') - self.assertHeader('Content-Type', 'text/plain;charset=utf-8') - self.getPage('/defct/html') - - def test_cherrypy_url(self): - # Input relative to current - self.getPage('/url/leaf?path_info=page1') - self.assertBody('%s/url/page1' % self.base()) - self.getPage('/url/?path_info=page1') - self.assertBody('%s/url/page1' % self.base()) - # Other host header - host = 'www.mydomain.example' - self.getPage('/url/leaf?path_info=page1', - headers=[('Host', host)]) - self.assertBody('%s://%s/url/page1' % (self.scheme, host)) - - # Input is 'absolute'; that is, relative to script_name - self.getPage('/url/leaf?path_info=/page1') - self.assertBody('%s/page1' % self.base()) - self.getPage('/url/?path_info=/page1') - self.assertBody('%s/page1' % self.base()) - - # Single dots - self.getPage('/url/leaf?path_info=./page1') - self.assertBody('%s/url/page1' % self.base()) - self.getPage('/url/leaf?path_info=other/./page1') - self.assertBody('%s/url/other/page1' % self.base()) - self.getPage('/url/?path_info=/other/./page1') - self.assertBody('%s/other/page1' % self.base()) - - # Double dots - self.getPage('/url/leaf?path_info=../page1') - self.assertBody('%s/page1' % self.base()) - self.getPage('/url/leaf?path_info=other/../page1') - self.assertBody('%s/url/page1' % self.base()) - self.getPage('/url/leaf?path_info=/other/../page1') - self.assertBody('%s/page1' % self.base()) - - # Output relative to current path or script_name - self.getPage('/url/?path_info=page1&relative=True') - self.assertBody('page1') - self.getPage('/url/leaf?path_info=/page1&relative=True') - self.assertBody('../page1') - self.getPage('/url/leaf?path_info=page1&relative=True') - self.assertBody('page1') - self.getPage('/url/leaf?path_info=leaf/page1&relative=True') - self.assertBody('leaf/page1') - self.getPage('/url/leaf?path_info=../page1&relative=True') - self.assertBody('../page1') - self.getPage('/url/?path_info=other/../page1&relative=True') - self.assertBody('page1') - - # Output relative to / - self.getPage('/baseurl?path_info=ab&relative=True') - self.assertBody('ab') - # Output relative to / - self.getPage('/baseurl?path_info=/ab&relative=True') - self.assertBody('ab') - - # absolute-path references ("server-relative") - # Input relative to current - self.getPage('/url/leaf?path_info=page1&relative=server') - self.assertBody('/url/page1') - self.getPage('/url/?path_info=page1&relative=server') - self.assertBody('/url/page1') - # Input is 'absolute'; that is, relative to script_name - self.getPage('/url/leaf?path_info=/page1&relative=server') - self.assertBody('/page1') - self.getPage('/url/?path_info=/page1&relative=server') - self.assertBody('/page1') - - def test_expose_decorator(self): - if not sys.version_info >= (2, 5): - return self.skip("skipped (Python 2.5+ only) ") - - # Test @expose - self.getPage("/expose_dec/no_call") - self.assertStatus(200) - self.assertBody("Mr E. R. Bradshaw") - - # Test @expose() - self.getPage("/expose_dec/call_empty") - self.assertStatus(200) - self.assertBody("Mrs. B.J. Smegma") - - # Test @expose("alias") - self.getPage("/expose_dec/call_alias") - self.assertStatus(200) - self.assertBody("Mr Nesbitt") - # Does the original name work? - self.getPage("/expose_dec/nesbitt") - self.assertStatus(200) - self.assertBody("Mr Nesbitt") - - # Test @expose(["alias1", "alias2"]) - self.getPage("/expose_dec/alias1") - self.assertStatus(200) - self.assertBody("Mr Ken Andrews") - self.getPage("/expose_dec/alias2") - self.assertStatus(200) - self.assertBody("Mr Ken Andrews") - # Does the original name work? - self.getPage("/expose_dec/andrews") - self.assertStatus(200) - self.assertBody("Mr Ken Andrews") - - # Test @expose(alias="alias") - self.getPage("/expose_dec/alias3") - self.assertStatus(200) - self.assertBody("Mr. and Mrs. Watson") - diff --git a/cherrypy/test/test_dynamicobjectmapping.py b/cherrypy/test/test_dynamicobjectmapping.py deleted file mode 100644 index 1e04d089..00000000 --- a/cherrypy/test/test_dynamicobjectmapping.py +++ /dev/null @@ -1,403 +0,0 @@ -import cherrypy -from cherrypy._cptree import Application -from cherrypy.test import helper - -script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"] - - - -def setup_server(): - class SubSubRoot: - def index(self): - return "SubSubRoot index" - index.exposed = True - - def default(self, *args): - return "SubSubRoot default" - default.exposed = True - - def handler(self): - return "SubSubRoot handler" - handler.exposed = True - - def dispatch(self): - return "SubSubRoot dispatch" - dispatch.exposed = True - - subsubnodes = { - '1': SubSubRoot(), - '2': SubSubRoot(), - } - - class SubRoot: - def index(self): - return "SubRoot index" - index.exposed = True - - def default(self, *args): - return "SubRoot %s" % (args,) - default.exposed = True - - def handler(self): - return "SubRoot handler" - handler.exposed = True - - def _cp_dispatch(self, vpath): - return subsubnodes.get(vpath[0], None) - - subnodes = { - '1': SubRoot(), - '2': SubRoot(), - } - class Root: - def index(self): - return "index" - index.exposed = True - - def default(self, *args): - return "default %s" % (args,) - default.exposed = True - - def handler(self): - return "handler" - handler.exposed = True - - def _cp_dispatch(self, vpath): - return subnodes.get(vpath[0]) - - #-------------------------------------------------------------------------- - # DynamicNodeAndMethodDispatcher example. - # This example exposes a fairly naive HTTP api - class User(object): - def __init__(self, id, name): - self.id = id - self.name = name - - def __unicode__(self): - return unicode(self.name) - - user_lookup = { - 1: User(1, 'foo'), - 2: User(2, 'bar'), - } - - def make_user(name, id=None): - if not id: - id = max(*user_lookup.keys()) + 1 - user_lookup[id] = User(id, name) - return id - - class UserContainerNode(object): - exposed = True - - def POST(self, name): - """ - Allow the creation of a new Object - """ - return "POST %d" % make_user(name) - - def GET(self): - keys = user_lookup.keys() - keys.sort() - return unicode(keys) - - def dynamic_dispatch(self, vpath): - try: - id = int(vpath[0]) - except (ValueError, IndexError): - return None - return UserInstanceNode(id) - - class UserInstanceNode(object): - exposed = True - def __init__(self, id): - self.id = id - self.user = user_lookup.get(id, None) - - # For all but PUT methods there MUST be a valid user identified - # by self.id - if not self.user and cherrypy.request.method != 'PUT': - raise cherrypy.HTTPError(404) - - def GET(self, *args, **kwargs): - """ - Return the appropriate representation of the instance. - """ - return unicode(self.user) - - def POST(self, name): - """ - Update the fields of the user instance. - """ - self.user.name = name - return "POST %d" % self.user.id - - def PUT(self, name): - """ - Create a new user with the specified id, or edit it if it already exists - """ - if self.user: - # Edit the current user - self.user.name = name - return "PUT %d" % self.user.id - else: - # Make a new user with said attributes. - return "PUT %d" % make_user(name, self.id) - - def DELETE(self): - """ - Delete the user specified at the id. - """ - id = self.user.id - del user_lookup[self.user.id] - del self.user - return "DELETE %d" % id - - - class ABHandler: - class CustomDispatch: - def index(self, a, b): - return "custom" - index.exposed = True - - def _cp_dispatch(self, vpath): - """Make sure that if we don't pop anything from vpath, - processing still works. - """ - return self.CustomDispatch() - - def index(self, a, b=None): - body = [ 'a:' + str(a) ] - if b is not None: - body.append(',b:' + str(b)) - return ''.join(body) - index.exposed = True - - def delete(self, a, b): - return 'deleting ' + str(a) + ' and ' + str(b) - delete.exposed = True - - class IndexOnly: - def _cp_dispatch(self, vpath): - """Make sure that popping ALL of vpath still shows the index - handler. - """ - while vpath: - vpath.pop() - return self - - def index(self): - return "IndexOnly index" - index.exposed = True - - class DecoratedPopArgs: - """Test _cp_dispatch with @cherrypy.popargs.""" - def index(self): - return "no params" - index.exposed = True - - def hi(self): - return "hi was not interpreted as 'a' param" - hi.exposed = True - DecoratedPopArgs = cherrypy.popargs('a', 'b', handler=ABHandler())(DecoratedPopArgs) - - class NonDecoratedPopArgs: - """Test _cp_dispatch = cherrypy.popargs()""" - - _cp_dispatch = cherrypy.popargs('a') - - def index(self, a): - return "index: " + str(a) - index.exposed = True - - class ParameterizedHandler: - """Special handler created for each request""" - - def __init__(self, a): - self.a = a - - def index(self): - if 'a' in cherrypy.request.params: - raise Exception("Parameterized handler argument ended up in request.params") - return self.a - index.exposed = True - - class ParameterizedPopArgs: - """Test cherrypy.popargs() with a function call handler""" - ParameterizedPopArgs = cherrypy.popargs('a', handler=ParameterizedHandler)(ParameterizedPopArgs) - - Root.decorated = DecoratedPopArgs() - Root.undecorated = NonDecoratedPopArgs() - Root.index_only = IndexOnly() - Root.parameter_test = ParameterizedPopArgs() - - Root.users = UserContainerNode() - - md = cherrypy.dispatch.MethodDispatcher('dynamic_dispatch') - for url in script_names: - conf = {'/': { - 'user': (url or "/").split("/")[-2], - }, - '/users': { - 'request.dispatch': md - }, - } - cherrypy.tree.mount(Root(), url, conf) - -class DynamicObjectMappingTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testObjectMapping(self): - for url in script_names: - prefix = self.script_name = url - - self.getPage('/') - self.assertBody('index') - - self.getPage('/handler') - self.assertBody('handler') - - # Dynamic dispatch will succeed here for the subnodes - # so the subroot gets called - self.getPage('/1/') - self.assertBody('SubRoot index') - - self.getPage('/2/') - self.assertBody('SubRoot index') - - self.getPage('/1/handler') - self.assertBody('SubRoot handler') - - self.getPage('/2/handler') - self.assertBody('SubRoot handler') - - # Dynamic dispatch will fail here for the subnodes - # so the default gets called - self.getPage('/asdf/') - self.assertBody("default ('asdf',)") - - self.getPage('/asdf/asdf') - self.assertBody("default ('asdf', 'asdf')") - - self.getPage('/asdf/handler') - self.assertBody("default ('asdf', 'handler')") - - # Dynamic dispatch will succeed here for the subsubnodes - # so the subsubroot gets called - self.getPage('/1/1/') - self.assertBody('SubSubRoot index') - - self.getPage('/2/2/') - self.assertBody('SubSubRoot index') - - self.getPage('/1/1/handler') - self.assertBody('SubSubRoot handler') - - self.getPage('/2/2/handler') - self.assertBody('SubSubRoot handler') - - self.getPage('/2/2/dispatch') - self.assertBody('SubSubRoot dispatch') - - # The exposed dispatch will not be called as a dispatch - # method. - self.getPage('/2/2/foo/foo') - self.assertBody("SubSubRoot default") - - # Dynamic dispatch will fail here for the subsubnodes - # so the SubRoot gets called - self.getPage('/1/asdf/') - self.assertBody("SubRoot ('asdf',)") - - self.getPage('/1/asdf/asdf') - self.assertBody("SubRoot ('asdf', 'asdf')") - - self.getPage('/1/asdf/handler') - self.assertBody("SubRoot ('asdf', 'handler')") - - def testMethodDispatch(self): - # GET acts like a container - self.getPage("/users") - self.assertBody("[1, 2]") - self.assertHeader('Allow', 'GET, HEAD, POST') - - # POST to the container URI allows creation - self.getPage("/users", method="POST", body="name=baz") - self.assertBody("POST 3") - self.assertHeader('Allow', 'GET, HEAD, POST') - - # POST to a specific instanct URI results in a 404 - # as the resource does not exit. - self.getPage("/users/5", method="POST", body="name=baz") - self.assertStatus(404) - - # PUT to a specific instanct URI results in creation - self.getPage("/users/5", method="PUT", body="name=boris") - self.assertBody("PUT 5") - self.assertHeader('Allow', 'DELETE, GET, HEAD, POST, PUT') - - # GET acts like a container - self.getPage("/users") - self.assertBody("[1, 2, 3, 5]") - self.assertHeader('Allow', 'GET, HEAD, POST') - - test_cases = ( - (1, 'foo', 'fooupdated', 'DELETE, GET, HEAD, POST, PUT'), - (2, 'bar', 'barupdated', 'DELETE, GET, HEAD, POST, PUT'), - (3, 'baz', 'bazupdated', 'DELETE, GET, HEAD, POST, PUT'), - (5, 'boris', 'borisupdated', 'DELETE, GET, HEAD, POST, PUT'), - ) - for id, name, updatedname, headers in test_cases: - self.getPage("/users/%d" % id) - self.assertBody(name) - self.assertHeader('Allow', headers) - - # Make sure POSTs update already existings resources - self.getPage("/users/%d" % id, method='POST', body="name=%s" % updatedname) - self.assertBody("POST %d" % id) - self.assertHeader('Allow', headers) - - # Make sure PUTs Update already existing resources. - self.getPage("/users/%d" % id, method='PUT', body="name=%s" % updatedname) - self.assertBody("PUT %d" % id) - self.assertHeader('Allow', headers) - - # Make sure DELETES Remove already existing resources. - self.getPage("/users/%d" % id, method='DELETE') - self.assertBody("DELETE %d" % id) - self.assertHeader('Allow', headers) - - - # GET acts like a container - self.getPage("/users") - self.assertBody("[]") - self.assertHeader('Allow', 'GET, HEAD, POST') - - def testVpathDispatch(self): - self.getPage("/decorated/") - self.assertBody("no params") - - self.getPage("/decorated/hi") - self.assertBody("hi was not interpreted as 'a' param") - - self.getPage("/decorated/yo/") - self.assertBody("a:yo") - - self.getPage("/decorated/yo/there/") - self.assertBody("a:yo,b:there") - - self.getPage("/decorated/yo/there/delete") - self.assertBody("deleting yo and there") - - self.getPage("/decorated/yo/there/handled_by_dispatch/") - self.assertBody("custom") - - self.getPage("/undecorated/blah/") - self.assertBody("index: blah") - - self.getPage("/index_only/a/b/c/d/e/f/g/") - self.assertBody("IndexOnly index") - - self.getPage("/parameter_test/argument2/") - self.assertBody("argument2") - diff --git a/cherrypy/test/test_encoding.py b/cherrypy/test/test_encoding.py deleted file mode 100644 index 67b28ede..00000000 --- a/cherrypy/test/test_encoding.py +++ /dev/null @@ -1,363 +0,0 @@ - -import gzip -import sys - -import cherrypy -from cherrypy._cpcompat import BytesIO, IncompleteRead, ntob, ntou - -europoundUnicode = ntou('\x80\xa3') -sing = u"\u6bdb\u6cfd\u4e1c: Sing, Little Birdie?" -sing8 = sing.encode('utf-8') -sing16 = sing.encode('utf-16') - - -from cherrypy.test import helper - - -class EncodingTests(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self, param): - assert param == europoundUnicode, "%r != %r" % (param, europoundUnicode) - yield europoundUnicode - index.exposed = True - - def mao_zedong(self): - return sing - mao_zedong.exposed = True - - def utf8(self): - return sing8 - utf8.exposed = True - utf8._cp_config = {'tools.encode.encoding': 'utf-8'} - - def cookies_and_headers(self): - # if the headers have non-ascii characters and a cookie has - # any part which is unicode (even ascii), the response - # should not fail. - cherrypy.response.cookie['candy'] = 'bar' - cherrypy.response.cookie['candy']['domain'] = 'cherrypy.org' - cherrypy.response.headers['Some-Header'] = 'My d\xc3\xb6g has fleas' - return 'Any content' - cookies_and_headers.exposed = True - - def reqparams(self, *args, **kwargs): - return ntob(', ').join([": ".join((k, v)).encode('utf8') - for k, v in cherrypy.request.params.items()]) - reqparams.exposed = True - - def nontext(self, *args, **kwargs): - cherrypy.response.headers['Content-Type'] = 'application/binary' - return '\x00\x01\x02\x03' - nontext.exposed = True - nontext._cp_config = {'tools.encode.text_only': False, - 'tools.encode.add_charset': True, - } - - class GZIP: - def index(self): - yield "Hello, world" - index.exposed = True - - def noshow(self): - # Test for ticket #147, where yield showed no exceptions (content- - # encoding was still gzip even though traceback wasn't zipped). - raise IndexError() - yield "Here be dragons" - noshow.exposed = True - # Turn encoding off so the gzip tool is the one doing the collapse. - noshow._cp_config = {'tools.encode.on': False} - - def noshow_stream(self): - # Test for ticket #147, where yield showed no exceptions (content- - # encoding was still gzip even though traceback wasn't zipped). - raise IndexError() - yield "Here be dragons" - noshow_stream.exposed = True - noshow_stream._cp_config = {'response.stream': True} - - class Decode: - def extra_charset(self, *args, **kwargs): - return ', '.join([": ".join((k, v)) - for k, v in cherrypy.request.params.items()]) - extra_charset.exposed = True - extra_charset._cp_config = { - 'tools.decode.on': True, - 'tools.decode.default_encoding': ['utf-16'], - } - - def force_charset(self, *args, **kwargs): - return ', '.join([": ".join((k, v)) - for k, v in cherrypy.request.params.items()]) - force_charset.exposed = True - force_charset._cp_config = { - 'tools.decode.on': True, - 'tools.decode.encoding': 'utf-16', - } - - root = Root() - root.gzip = GZIP() - root.decode = Decode() - cherrypy.tree.mount(root, config={'/gzip': {'tools.gzip.on': True}}) - setup_server = staticmethod(setup_server) - - def test_query_string_decoding(self): - europoundUtf8 = europoundUnicode.encode('utf-8') - self.getPage(ntob('/?param=') + europoundUtf8) - self.assertBody(europoundUtf8) - - # Encoded utf8 query strings MUST be parsed correctly. - # Here, q is the POUND SIGN U+00A3 encoded in utf8 and then %HEX - self.getPage("/reqparams?q=%C2%A3") - # The return value will be encoded as utf8. - self.assertBody(ntob("q: \xc2\xa3")) - - # Query strings that are incorrectly encoded MUST raise 404. - # Here, q is the POUND SIGN U+00A3 encoded in latin1 and then %HEX - self.getPage("/reqparams?q=%A3") - self.assertStatus(404) - self.assertErrorPage(404, - "The given query string could not be processed. Query " - "strings for this resource must be encoded with 'utf8'.") - - def test_urlencoded_decoding(self): - # Test the decoding of an application/x-www-form-urlencoded entity. - europoundUtf8 = europoundUnicode.encode('utf-8') - body=ntob("param=") + europoundUtf8 - self.getPage('/', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(europoundUtf8) - - # Encoded utf8 entities MUST be parsed and decoded correctly. - # Here, q is the POUND SIGN U+00A3 encoded in utf8 - body = ntob("q=\xc2\xa3") - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("q: \xc2\xa3")) - - # ...and in utf16, which is not in the default attempt_charsets list: - body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded;charset=utf-16"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("q: \xc2\xa3")) - - # Entities that are incorrectly encoded MUST raise 400. - # Here, q is the POUND SIGN U+00A3 encoded in utf16, but - # the Content-Type incorrectly labels it utf-8. - body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded;charset=utf-8"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertStatus(400) - self.assertErrorPage(400, - "The request entity could not be decoded. The following charsets " - "were attempted: ['utf-8']") - - def test_decode_tool(self): - # An extra charset should be tried first, and succeed if it matches. - # Here, we add utf-16 as a charset and pass a utf-16 body. - body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00") - self.getPage('/decode/extra_charset', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("q: \xc2\xa3")) - - # An extra charset should be tried first, and continue to other default - # charsets if it doesn't match. - # Here, we add utf-16 as a charset but still pass a utf-8 body. - body = ntob("q=\xc2\xa3") - self.getPage('/decode/extra_charset', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("q: \xc2\xa3")) - - # An extra charset should error if force is True and it doesn't match. - # Here, we force utf-16 as a charset but still pass a utf-8 body. - body = ntob("q=\xc2\xa3") - self.getPage('/decode/force_charset', method='POST', - headers=[("Content-Type", "application/x-www-form-urlencoded"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertErrorPage(400, - "The request entity could not be decoded. The following charsets " - "were attempted: ['utf-16']") - - def test_multipart_decoding(self): - # Test the decoding of a multipart entity when the charset (utf16) is - # explicitly given. - body=ntob('\r\n'.join(['--X', - 'Content-Type: text/plain;charset=utf-16', - 'Content-Disposition: form-data; name="text"', - '', - '\xff\xfea\x00b\x00\x1c c\x00', - '--X', - 'Content-Type: text/plain;charset=utf-16', - 'Content-Disposition: form-data; name="submit"', - '', - '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00', - '--X--'])) - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "multipart/form-data;boundary=X"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("text: ab\xe2\x80\x9cc, submit: Create")) - - def test_multipart_decoding_no_charset(self): - # Test the decoding of a multipart entity when the charset (utf8) is - # NOT explicitly given, but is in the list of charsets to attempt. - body=ntob('\r\n'.join(['--X', - 'Content-Disposition: form-data; name="text"', - '', - '\xe2\x80\x9c', - '--X', - 'Content-Disposition: form-data; name="submit"', - '', - 'Create', - '--X--'])) - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "multipart/form-data;boundary=X"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(ntob("text: \xe2\x80\x9c, submit: Create")) - - def test_multipart_decoding_no_successful_charset(self): - # Test the decoding of a multipart entity when the charset (utf16) is - # NOT explicitly given, and is NOT in the list of charsets to attempt. - body=ntob('\r\n'.join(['--X', - 'Content-Disposition: form-data; name="text"', - '', - '\xff\xfea\x00b\x00\x1c c\x00', - '--X', - 'Content-Disposition: form-data; name="submit"', - '', - '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00', - '--X--'])) - self.getPage('/reqparams', method='POST', - headers=[("Content-Type", "multipart/form-data;boundary=X"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertStatus(400) - self.assertErrorPage(400, - "The request entity could not be decoded. The following charsets " - "were attempted: ['us-ascii', 'utf-8']") - - def test_nontext(self): - self.getPage('/nontext') - self.assertHeader('Content-Type', 'application/binary;charset=utf-8') - self.assertBody('\x00\x01\x02\x03') - - def testEncoding(self): - # Default encoding should be utf-8 - self.getPage('/mao_zedong') - self.assertBody(sing8) - - # Ask for utf-16. - self.getPage('/mao_zedong', [('Accept-Charset', 'utf-16')]) - self.assertHeader('Content-Type', 'text/html;charset=utf-16') - self.assertBody(sing16) - - # Ask for multiple encodings. ISO-8859-1 should fail, and utf-16 - # should be produced. - self.getPage('/mao_zedong', [('Accept-Charset', - 'iso-8859-1;q=1, utf-16;q=0.5')]) - self.assertBody(sing16) - - # The "*" value should default to our default_encoding, utf-8 - self.getPage('/mao_zedong', [('Accept-Charset', '*;q=1, utf-7;q=.2')]) - self.assertBody(sing8) - - # Only allow iso-8859-1, which should fail and raise 406. - self.getPage('/mao_zedong', [('Accept-Charset', 'iso-8859-1, *;q=0')]) - self.assertStatus("406 Not Acceptable") - self.assertInBody("Your client sent this Accept-Charset header: " - "iso-8859-1, *;q=0. We tried these charsets: " - "iso-8859-1.") - - # Ask for x-mac-ce, which should be unknown. See ticket #569. - self.getPage('/mao_zedong', [('Accept-Charset', - 'us-ascii, ISO-8859-1, x-mac-ce')]) - self.assertStatus("406 Not Acceptable") - self.assertInBody("Your client sent this Accept-Charset header: " - "us-ascii, ISO-8859-1, x-mac-ce. We tried these " - "charsets: ISO-8859-1, us-ascii, x-mac-ce.") - - # Test the 'encoding' arg to encode. - self.getPage('/utf8') - self.assertBody(sing8) - self.getPage('/utf8', [('Accept-Charset', 'us-ascii, ISO-8859-1')]) - self.assertStatus("406 Not Acceptable") - - def testGzip(self): - zbuf = BytesIO() - zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9) - zfile.write(ntob("Hello, world")) - zfile.close() - - self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip")]) - self.assertInBody(zbuf.getvalue()[:3]) - self.assertHeader("Vary", "Accept-Encoding") - self.assertHeader("Content-Encoding", "gzip") - - # Test when gzip is denied. - self.getPage('/gzip/', headers=[("Accept-Encoding", "identity")]) - self.assertHeader("Vary", "Accept-Encoding") - self.assertNoHeader("Content-Encoding") - self.assertBody("Hello, world") - - self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip;q=0")]) - self.assertHeader("Vary", "Accept-Encoding") - self.assertNoHeader("Content-Encoding") - self.assertBody("Hello, world") - - self.getPage('/gzip/', headers=[("Accept-Encoding", "*;q=0")]) - self.assertStatus(406) - self.assertNoHeader("Content-Encoding") - self.assertErrorPage(406, "identity, gzip") - - # Test for ticket #147 - self.getPage('/gzip/noshow', headers=[("Accept-Encoding", "gzip")]) - self.assertNoHeader('Content-Encoding') - self.assertStatus(500) - self.assertErrorPage(500, pattern="IndexError\n") - - # In this case, there's nothing we can do to deliver a - # readable page, since 1) the gzip header is already set, - # and 2) we may have already written some of the body. - # The fix is to never stream yields when using gzip. - if (cherrypy.server.protocol_version == "HTTP/1.0" or - getattr(cherrypy.server, "using_apache", False)): - self.getPage('/gzip/noshow_stream', - headers=[("Accept-Encoding", "gzip")]) - self.assertHeader('Content-Encoding', 'gzip') - self.assertInBody('\x1f\x8b\x08\x00') - else: - # The wsgiserver will simply stop sending data, and the HTTP client - # will error due to an incomplete chunk-encoded stream. - self.assertRaises((ValueError, IncompleteRead), self.getPage, - '/gzip/noshow_stream', - headers=[("Accept-Encoding", "gzip")]) - - def test_UnicodeHeaders(self): - self.getPage('/cookies_and_headers') - self.assertBody('Any content') - diff --git a/cherrypy/test/test_etags.py b/cherrypy/test/test_etags.py deleted file mode 100644 index 026f9d65..00000000 --- a/cherrypy/test/test_etags.py +++ /dev/null @@ -1,81 +0,0 @@ -import cherrypy -from cherrypy.test import helper - - -class ETagTest(helper.CPWebCase): - - def setup_server(): - class Root: - def resource(self): - return "Oh wah ta goo Siam." - resource.exposed = True - - def fail(self, code): - code = int(code) - if 300 <= code <= 399: - raise cherrypy.HTTPRedirect([], code) - else: - raise cherrypy.HTTPError(code) - fail.exposed = True - - def unicoded(self): - return u'I am a \u1ee4nicode string.' - unicoded.exposed = True - unicoded._cp_config = {'tools.encode.on': True} - - conf = {'/': {'tools.etags.on': True, - 'tools.etags.autotags': True, - }} - cherrypy.tree.mount(Root(), config=conf) - setup_server = staticmethod(setup_server) - - def test_etags(self): - self.getPage("/resource") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.assertBody('Oh wah ta goo Siam.') - etag = self.assertHeader('ETag') - - # Test If-Match (both valid and invalid) - self.getPage("/resource", headers=[('If-Match', etag)]) - self.assertStatus("200 OK") - self.getPage("/resource", headers=[('If-Match', "*")]) - self.assertStatus("200 OK") - self.getPage("/resource", headers=[('If-Match', "*")], method="POST") - self.assertStatus("200 OK") - self.getPage("/resource", headers=[('If-Match', "a bogus tag")]) - self.assertStatus("412 Precondition Failed") - - # Test If-None-Match (both valid and invalid) - self.getPage("/resource", headers=[('If-None-Match', etag)]) - self.assertStatus(304) - self.getPage("/resource", method='POST', headers=[('If-None-Match', etag)]) - self.assertStatus("412 Precondition Failed") - self.getPage("/resource", headers=[('If-None-Match', "*")]) - self.assertStatus(304) - self.getPage("/resource", headers=[('If-None-Match', "a bogus tag")]) - self.assertStatus("200 OK") - - def test_errors(self): - self.getPage("/resource") - self.assertStatus(200) - etag = self.assertHeader('ETag') - - # Test raising errors in page handler - self.getPage("/fail/412", headers=[('If-Match', etag)]) - self.assertStatus(412) - self.getPage("/fail/304", headers=[('If-Match', etag)]) - self.assertStatus(304) - self.getPage("/fail/412", headers=[('If-None-Match', "*")]) - self.assertStatus(412) - self.getPage("/fail/304", headers=[('If-None-Match', "*")]) - self.assertStatus(304) - - def test_unicode_body(self): - self.getPage("/unicoded") - self.assertStatus(200) - etag1 = self.assertHeader('ETag') - self.getPage("/unicoded", headers=[('If-Match', etag1)]) - self.assertStatus(200) - self.assertHeader('ETag', etag1) - diff --git a/cherrypy/test/test_http.py b/cherrypy/test/test_http.py deleted file mode 100644 index eb72b5bf..00000000 --- a/cherrypy/test/test_http.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Tests for managing HTTP issues (malformed requests, etc).""" - -import mimetypes - -import cherrypy -from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob - - -def encode_multipart_formdata(files): - """Return (content_type, body) ready for httplib.HTTP instance. - - files: a sequence of (name, filename, value) tuples for multipart uploads. - """ - BOUNDARY = '________ThIs_Is_tHe_bouNdaRY_$' - L = [] - for key, filename, value in files: - L.append('--' + BOUNDARY) - L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % - (key, filename)) - ct = mimetypes.guess_type(filename)[0] or 'application/octet-stream' - L.append('Content-Type: %s' % ct) - L.append('') - L.append(value) - L.append('--' + BOUNDARY + '--') - L.append('') - body = '\r\n'.join(L) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, body - - - - -from cherrypy.test import helper - -class HTTPTests(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self, *args, **kwargs): - return "Hello world!" - index.exposed = True - - def no_body(self, *args, **kwargs): - return "Hello world!" - no_body.exposed = True - no_body._cp_config = {'request.process_request_body': False} - - def post_multipart(self, file): - """Return a summary ("a * 65536\nb * 65536") of the uploaded file.""" - contents = file.file.read() - summary = [] - curchar = "" - count = 0 - for c in contents: - if c == curchar: - count += 1 - else: - if count: - summary.append("%s * %d" % (curchar, count)) - count = 1 - curchar = c - if count: - summary.append("%s * %d" % (curchar, count)) - return ", ".join(summary) - post_multipart.exposed = True - - cherrypy.tree.mount(Root()) - cherrypy.config.update({'server.max_request_body_size': 30000000}) - setup_server = staticmethod(setup_server) - - def test_no_content_length(self): - # "The presence of a message-body in a request is signaled by the - # inclusion of a Content-Length or Transfer-Encoding header field in - # the request's message-headers." - # - # Send a message with neither header and no body. Even though - # the request is of method POST, this should be OK because we set - # request.process_request_body to False for our handler. - if self.scheme == "https": - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.request("POST", "/no_body") - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(ntob('Hello world!')) - - # Now send a message that has no Content-Length, but does send a body. - # Verify that CP times out the socket and responds - # with 411 Length Required. - if self.scheme == "https": - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.request("POST", "/") - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(411) - - def test_post_multipart(self): - alphabet = "abcdefghijklmnopqrstuvwxyz" - # generate file contents for a large post - contents = "".join([c * 65536 for c in alphabet]) - - # encode as multipart form data - files=[('file', 'file.txt', contents)] - content_type, body = encode_multipart_formdata(files) - body = body.encode('Latin-1') - - # post file - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.putrequest('POST', '/post_multipart') - c.putheader('Content-Type', content_type) - c.putheader('Content-Length', str(len(body))) - c.endheaders() - c.send(body) - - response = c.getresponse() - self.body = response.fp.read() - self.status = str(response.status) - self.assertStatus(200) - self.assertBody(", ".join(["%s * 65536" % c for c in alphabet])) - - def test_malformed_request_line(self): - if getattr(cherrypy.server, "using_apache", False): - return self.skip("skipped due to known Apache differences...") - - # Test missing version in Request-Line - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c._output(ntob('GET /')) - c._send_output() - if hasattr(c, 'strict'): - response = c.response_class(c.sock, strict=c.strict, method='GET') - else: - # Python 3.2 removed the 'strict' feature, saying: - # "http.client now always assumes HTTP/1.x compliant servers." - response = c.response_class(c.sock, method='GET') - response.begin() - self.assertEqual(response.status, 400) - self.assertEqual(response.fp.read(22), ntob("Malformed Request-Line")) - c.close() - - def test_malformed_header(self): - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - c.putrequest('GET', '/') - c.putheader('Content-Type', 'text/plain') - # See http://www.cherrypy.org/ticket/941 - c._output(ntob('Re, 1.2.3.4#015#012')) - c.endheaders() - - response = c.getresponse() - self.status = str(response.status) - self.assertStatus(400) - self.body = response.fp.read(20) - self.assertBody("Illegal header line.") - diff --git a/cherrypy/test/test_httpauth.py b/cherrypy/test/test_httpauth.py deleted file mode 100644 index 9d0eecb2..00000000 --- a/cherrypy/test/test_httpauth.py +++ /dev/null @@ -1,151 +0,0 @@ -import cherrypy -from cherrypy._cpcompat import md5, sha, ntob -from cherrypy.lib import httpauth - -from cherrypy.test import helper - -class HTTPAuthTest(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self): - return "This is public." - index.exposed = True - - class DigestProtected: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - class BasicProtected: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - class BasicProtected2: - def index(self): - return "Hello %s, you've been authorized." % cherrypy.request.login - index.exposed = True - - def fetch_users(): - return {'test': 'test'} - - def sha_password_encrypter(password): - return sha(ntob(password)).hexdigest() - - def fetch_password(username): - return sha(ntob('test')).hexdigest() - - conf = {'/digest': {'tools.digest_auth.on': True, - 'tools.digest_auth.realm': 'localhost', - 'tools.digest_auth.users': fetch_users}, - '/basic': {'tools.basic_auth.on': True, - 'tools.basic_auth.realm': 'localhost', - 'tools.basic_auth.users': {'test': md5(ntob('test')).hexdigest()}}, - '/basic2': {'tools.basic_auth.on': True, - 'tools.basic_auth.realm': 'localhost', - 'tools.basic_auth.users': fetch_password, - 'tools.basic_auth.encrypt': sha_password_encrypter}} - - root = Root() - root.digest = DigestProtected() - root.basic = BasicProtected() - root.basic2 = BasicProtected2() - cherrypy.tree.mount(root, config=conf) - setup_server = staticmethod(setup_server) - - - def testPublic(self): - self.getPage("/") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.assertBody('This is public.') - - def testBasic(self): - self.getPage("/basic/") - self.assertStatus(401) - self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"') - - self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZX60')]) - self.assertStatus(401) - - self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZXN0')]) - self.assertStatus('200 OK') - self.assertBody("Hello test, you've been authorized.") - - def testBasic2(self): - self.getPage("/basic2/") - self.assertStatus(401) - self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"') - - self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZX60')]) - self.assertStatus(401) - - self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZXN0')]) - self.assertStatus('200 OK') - self.assertBody("Hello test, you've been authorized.") - - def testDigest(self): - self.getPage("/digest/") - self.assertStatus(401) - - value = None - for k, v in self.headers: - if k.lower() == "www-authenticate": - if v.startswith("Digest"): - value = v - break - - if value is None: - self._handlewebError("Digest authentification scheme was not found") - - value = value[7:] - items = value.split(', ') - tokens = {} - for item in items: - key, value = item.split('=') - tokens[key.lower()] = value - - missing_msg = "%s is missing" - bad_value_msg = "'%s' was expecting '%s' but found '%s'" - nonce = None - if 'realm' not in tokens: - self._handlewebError(missing_msg % 'realm') - elif tokens['realm'] != '"localhost"': - self._handlewebError(bad_value_msg % ('realm', '"localhost"', tokens['realm'])) - if 'nonce' not in tokens: - self._handlewebError(missing_msg % 'nonce') - else: - nonce = tokens['nonce'].strip('"') - if 'algorithm' not in tokens: - self._handlewebError(missing_msg % 'algorithm') - elif tokens['algorithm'] != '"MD5"': - self._handlewebError(bad_value_msg % ('algorithm', '"MD5"', tokens['algorithm'])) - if 'qop' not in tokens: - self._handlewebError(missing_msg % 'qop') - elif tokens['qop'] != '"auth"': - self._handlewebError(bad_value_msg % ('qop', '"auth"', tokens['qop'])) - - # Test a wrong 'realm' value - base_auth = 'Digest username="test", realm="wrong realm", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' - - auth = base_auth % (nonce, '', '00000001') - params = httpauth.parseAuthorization(auth) - response = httpauth._computeDigestResponse(params, 'test') - - auth = base_auth % (nonce, response, '00000001') - self.getPage('/digest/', [('Authorization', auth)]) - self.assertStatus(401) - - # Test that must pass - base_auth = 'Digest username="test", realm="localhost", nonce="%s", uri="/digest/", algorithm=MD5, response="%s", qop=auth, nc=%s, cnonce="1522e61005789929"' - - auth = base_auth % (nonce, '', '00000001') - params = httpauth.parseAuthorization(auth) - response = httpauth._computeDigestResponse(params, 'test') - - auth = base_auth % (nonce, response, '00000001') - self.getPage('/digest/', [('Authorization', auth)]) - self.assertStatus('200 OK') - self.assertBody("Hello test, you've been authorized.") - diff --git a/cherrypy/test/test_httplib.py b/cherrypy/test/test_httplib.py deleted file mode 100644 index 5dc40fd2..00000000 --- a/cherrypy/test/test_httplib.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Tests for cherrypy/lib/httputil.py.""" - -import unittest -from cherrypy.lib import httputil - - -class UtilityTests(unittest.TestCase): - - def test_urljoin(self): - # Test all slash+atom combinations for SCRIPT_NAME and PATH_INFO - self.assertEqual(httputil.urljoin("/sn/", "/pi/"), "/sn/pi/") - self.assertEqual(httputil.urljoin("/sn/", "/pi"), "/sn/pi") - self.assertEqual(httputil.urljoin("/sn/", "/"), "/sn/") - self.assertEqual(httputil.urljoin("/sn/", ""), "/sn/") - self.assertEqual(httputil.urljoin("/sn", "/pi/"), "/sn/pi/") - self.assertEqual(httputil.urljoin("/sn", "/pi"), "/sn/pi") - self.assertEqual(httputil.urljoin("/sn", "/"), "/sn/") - self.assertEqual(httputil.urljoin("/sn", ""), "/sn") - self.assertEqual(httputil.urljoin("/", "/pi/"), "/pi/") - self.assertEqual(httputil.urljoin("/", "/pi"), "/pi") - self.assertEqual(httputil.urljoin("/", "/"), "/") - self.assertEqual(httputil.urljoin("/", ""), "/") - self.assertEqual(httputil.urljoin("", "/pi/"), "/pi/") - self.assertEqual(httputil.urljoin("", "/pi"), "/pi") - self.assertEqual(httputil.urljoin("", "/"), "/") - self.assertEqual(httputil.urljoin("", ""), "/") - -if __name__ == '__main__': - unittest.main() diff --git a/cherrypy/test/test_json.py b/cherrypy/test/test_json.py deleted file mode 100644 index a02c0767..00000000 --- a/cherrypy/test/test_json.py +++ /dev/null @@ -1,79 +0,0 @@ -import cherrypy -from cherrypy.test import helper - -from cherrypy._cpcompat import json - -class JsonTest(helper.CPWebCase): - def setup_server(): - class Root(object): - def plain(self): - return 'hello' - plain.exposed = True - - def json_string(self): - return 'hello' - json_string.exposed = True - json_string._cp_config = {'tools.json_out.on': True} - - def json_list(self): - return ['a', 'b', 42] - json_list.exposed = True - json_list._cp_config = {'tools.json_out.on': True} - - def json_dict(self): - return {'answer': 42} - json_dict.exposed = True - json_dict._cp_config = {'tools.json_out.on': True} - - def json_post(self): - if cherrypy.request.json == [13, 'c']: - return 'ok' - else: - return 'nok' - json_post.exposed = True - json_post._cp_config = {'tools.json_in.on': True} - - root = Root() - cherrypy.tree.mount(root) - setup_server = staticmethod(setup_server) - - def test_json_output(self): - if json is None: - self.skip("json not found ") - return - - self.getPage("/plain") - self.assertBody("hello") - - self.getPage("/json_string") - self.assertBody('"hello"') - - self.getPage("/json_list") - self.assertBody('["a", "b", 42]') - - self.getPage("/json_dict") - self.assertBody('{"answer": 42}') - - def test_json_input(self): - if json is None: - self.skip("json not found ") - return - - body = '[13, "c"]' - headers = [('Content-Type', 'application/json'), - ('Content-Length', str(len(body)))] - self.getPage("/json_post", method="POST", headers=headers, body=body) - self.assertBody('ok') - - body = '[13, "c"]' - headers = [('Content-Type', 'text/plain'), - ('Content-Length', str(len(body)))] - self.getPage("/json_post", method="POST", headers=headers, body=body) - self.assertStatus(415, 'Expected an application/json content type') - - body = '[13, -]' - headers = [('Content-Type', 'application/json'), - ('Content-Length', str(len(body)))] - self.getPage("/json_post", method="POST", headers=headers, body=body) - self.assertStatus(400, 'Invalid JSON document') - diff --git a/cherrypy/test/test_logging.py b/cherrypy/test/test_logging.py deleted file mode 100644 index 5a13cd4a..00000000 --- a/cherrypy/test/test_logging.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Basic tests for the CherryPy core: request handling.""" - -import os -localDir = os.path.dirname(__file__) - -import cherrypy - -access_log = os.path.join(localDir, "access.log") -error_log = os.path.join(localDir, "error.log") - -# Some unicode strings. -tartaros = u'\u03a4\u1f71\u03c1\u03c4\u03b1\u03c1\u03bf\u03c2' -erebos = u'\u0388\u03c1\u03b5\u03b2\u03bf\u03c2.com' - - -def setup_server(): - class Root: - - def index(self): - return "hello" - index.exposed = True - - def uni_code(self): - cherrypy.request.login = tartaros - cherrypy.request.remote.name = erebos - uni_code.exposed = True - - def slashes(self): - cherrypy.request.request_line = r'GET /slashed\path HTTP/1.1' - slashes.exposed = True - - def whitespace(self): - # User-Agent = "User-Agent" ":" 1*( product | comment ) - # comment = "(" *( ctext | quoted-pair | comment ) ")" - # ctext = - # TEXT = - # LWS = [CRLF] 1*( SP | HT ) - cherrypy.request.headers['User-Agent'] = 'Browzuh (1.0\r\n\t\t.3)' - whitespace.exposed = True - - def as_string(self): - return "content" - as_string.exposed = True - - def as_yield(self): - yield "content" - as_yield.exposed = True - - def error(self): - raise ValueError() - error.exposed = True - error._cp_config = {'tools.log_tracebacks.on': True} - - root = Root() - - - cherrypy.config.update({'log.error_file': error_log, - 'log.access_file': access_log, - }) - cherrypy.tree.mount(root) - - - -from cherrypy.test import helper, logtest - -class AccessLogTests(helper.CPWebCase, logtest.LogCase): - setup_server = staticmethod(setup_server) - - logfile = access_log - - def testNormalReturn(self): - self.markLog() - self.getPage("/as_string", - headers=[('Referer', 'http://www.cherrypy.org/'), - ('User-Agent', 'Mozilla/5.0')]) - self.assertBody('content') - self.assertStatus(200) - - intro = '%s - - [' % self.interface() - - self.assertLog(-1, intro) - - if [k for k, v in self.headers if k.lower() == 'content-length']: - self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 7 ' - '"http://www.cherrypy.org/" "Mozilla/5.0"' - % self.prefix()) - else: - self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 - ' - '"http://www.cherrypy.org/" "Mozilla/5.0"' - % self.prefix()) - - def testNormalYield(self): - self.markLog() - self.getPage("/as_yield") - self.assertBody('content') - self.assertStatus(200) - - intro = '%s - - [' % self.interface() - - self.assertLog(-1, intro) - if [k for k, v in self.headers if k.lower() == 'content-length']: - self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 7 "" ""' % - self.prefix()) - else: - self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 - "" ""' - % self.prefix()) - - def testEscapedOutput(self): - # Test unicode in access log pieces. - self.markLog() - self.getPage("/uni_code") - self.assertStatus(200) - self.assertLog(-1, repr(tartaros.encode('utf8'))[1:-1]) - # Test the erebos value. Included inline for your enlightenment. - # Note the 'r' prefix--those backslashes are literals. - self.assertLog(-1, r'\xce\x88\xcf\x81\xce\xb5\xce\xb2\xce\xbf\xcf\x82') - - # Test backslashes in output. - self.markLog() - self.getPage("/slashes") - self.assertStatus(200) - self.assertLog(-1, r'"GET /slashed\\path HTTP/1.1"') - - # Test whitespace in output. - self.markLog() - self.getPage("/whitespace") - self.assertStatus(200) - # Again, note the 'r' prefix. - self.assertLog(-1, r'"Browzuh (1.0\r\n\t\t.3)"') - - -class ErrorLogTests(helper.CPWebCase, logtest.LogCase): - setup_server = staticmethod(setup_server) - - logfile = error_log - - def testTracebacks(self): - # Test that tracebacks get written to the error log. - self.markLog() - ignore = helper.webtest.ignored_exceptions - ignore.append(ValueError) - try: - self.getPage("/error") - self.assertInBody("raise ValueError()") - self.assertLog(0, 'HTTP Traceback (most recent call last):') - self.assertLog(-3, 'raise ValueError()') - finally: - ignore.pop() - diff --git a/cherrypy/test/test_mime.py b/cherrypy/test/test_mime.py deleted file mode 100644 index 605071b8..00000000 --- a/cherrypy/test/test_mime.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Tests for various MIME issues, including the safe_multipart Tool.""" - -import cherrypy -from cherrypy._cpcompat import ntob, ntou, sorted - -def setup_server(): - - class Root: - - def multipart(self, parts): - return repr(parts) - multipart.exposed = True - - def multipart_form_data(self, **kwargs): - return repr(list(sorted(kwargs.items()))) - multipart_form_data.exposed = True - - def flashupload(self, Filedata, Upload, Filename): - return ("Upload: %r, Filename: %r, Filedata: %r" % - (Upload, Filename, Filedata.file.read())) - flashupload.exposed = True - - cherrypy.config.update({'server.max_request_body_size': 0}) - cherrypy.tree.mount(Root()) - - -# Client-side code # - -from cherrypy.test import helper - -class MultipartTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_multipart(self): - text_part = ntou("This is the text version") - html_part = ntou(""" - - - - - - -This is the HTML version - - -""") - body = '\r\n'.join([ - "--123456789", - "Content-Type: text/plain; charset='ISO-8859-1'", - "Content-Transfer-Encoding: 7bit", - "", - text_part, - "--123456789", - "Content-Type: text/html; charset='ISO-8859-1'", - "", - html_part, - "--123456789--"]) - headers = [ - ('Content-Type', 'multipart/mixed; boundary=123456789'), - ('Content-Length', str(len(body))), - ] - self.getPage('/multipart', headers, "POST", body) - self.assertBody(repr([text_part, html_part])) - - def test_multipart_form_data(self): - body='\r\n'.join(['--X', - 'Content-Disposition: form-data; name="foo"', - '', - 'bar', - '--X', - # Test a param with more than one value. - # See http://www.cherrypy.org/ticket/1028 - 'Content-Disposition: form-data; name="baz"', - '', - '111', - '--X', - 'Content-Disposition: form-data; name="baz"', - '', - '333', - '--X--']) - self.getPage('/multipart_form_data', method='POST', - headers=[("Content-Type", "multipart/form-data;boundary=X"), - ("Content-Length", str(len(body))), - ], - body=body), - self.assertBody(repr([('baz', [u'111', u'333']), ('foo', u'bar')])) - - -class SafeMultipartHandlingTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_Flash_Upload(self): - headers = [ - ('Accept', 'text/*'), - ('Content-Type', 'multipart/form-data; ' - 'boundary=----------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6'), - ('User-Agent', 'Shockwave Flash'), - ('Host', 'www.example.com:8080'), - ('Content-Length', '499'), - ('Connection', 'Keep-Alive'), - ('Cache-Control', 'no-cache'), - ] - filedata = ntob('\r\n' - '\r\n' - '\r\n') - body = (ntob( - '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' - 'Content-Disposition: form-data; name="Filename"\r\n' - '\r\n' - '.project\r\n' - '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' - 'Content-Disposition: form-data; ' - 'name="Filedata"; filename=".project"\r\n' - 'Content-Type: application/octet-stream\r\n' - '\r\n') - + filedata + - ntob('\r\n' - '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n' - 'Content-Disposition: form-data; name="Upload"\r\n' - '\r\n' - 'Submit Query\r\n' - # Flash apps omit the trailing \r\n on the last line: - '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6--' - )) - self.getPage('/flashupload', headers, "POST", body) - self.assertBody("Upload: u'Submit Query', Filename: u'.project', " - "Filedata: %r" % filedata) - diff --git a/cherrypy/test/test_misc_tools.py b/cherrypy/test/test_misc_tools.py deleted file mode 100644 index fb94e860..00000000 --- a/cherrypy/test/test_misc_tools.py +++ /dev/null @@ -1,202 +0,0 @@ -import os -localDir = os.path.dirname(__file__) -logfile = os.path.join(localDir, "test_misc_tools.log") - -import cherrypy -from cherrypy import tools - - -def setup_server(): - class Root: - def index(self): - yield "Hello, world" - index.exposed = True - h = [("Content-Language", "en-GB"), ('Content-Type', 'text/plain')] - tools.response_headers(headers=h)(index) - - def other(self): - return "salut" - other.exposed = True - other._cp_config = { - 'tools.response_headers.on': True, - 'tools.response_headers.headers': [("Content-Language", "fr"), - ('Content-Type', 'text/plain')], - 'tools.log_hooks.on': True, - } - - - class Accept: - _cp_config = {'tools.accept.on': True} - - def index(self): - return 'Atom feed' - index.exposed = True - - # In Python 2.4+, we could use a decorator instead: - # @tools.accept('application/atom+xml') - def feed(self): - return """ - - Unknown Blog -""" - feed.exposed = True - feed._cp_config = {'tools.accept.media': 'application/atom+xml'} - - def select(self): - # We could also write this: mtype = cherrypy.lib.accept.accept(...) - mtype = tools.accept.callable(['text/html', 'text/plain']) - if mtype == 'text/html': - return "

Page Title

" - else: - return "PAGE TITLE" - select.exposed = True - - class Referer: - def accept(self): - return "Accepted!" - accept.exposed = True - reject = accept - - class AutoVary: - def index(self): - # Read a header directly with 'get' - ae = cherrypy.request.headers.get('Accept-Encoding') - # Read a header directly with '__getitem__' - cl = cherrypy.request.headers['Host'] - # Read a header directly with '__contains__' - hasif = 'If-Modified-Since' in cherrypy.request.headers - # Read a header directly with 'has_key' - has = cherrypy.request.headers.has_key('Range') - # Call a lib function - mtype = tools.accept.callable(['text/html', 'text/plain']) - return "Hello, world!" - index.exposed = True - - conf = {'/referer': {'tools.referer.on': True, - 'tools.referer.pattern': r'http://[^/]*example\.com', - }, - '/referer/reject': {'tools.referer.accept': False, - 'tools.referer.accept_missing': True, - }, - '/autovary': {'tools.autovary.on': True}, - } - - root = Root() - root.referer = Referer() - root.accept = Accept() - root.autovary = AutoVary() - cherrypy.tree.mount(root, config=conf) - cherrypy.config.update({'log.error_file': logfile}) - - -from cherrypy.test import helper - -class ResponseHeadersTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testResponseHeadersDecorator(self): - self.getPage('/') - self.assertHeader("Content-Language", "en-GB") - self.assertHeader('Content-Type', 'text/plain;charset=utf-8') - - def testResponseHeaders(self): - self.getPage('/other') - self.assertHeader("Content-Language", "fr") - self.assertHeader('Content-Type', 'text/plain;charset=utf-8') - - -class RefererTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testReferer(self): - self.getPage('/referer/accept') - self.assertErrorPage(403, 'Forbidden Referer header.') - - self.getPage('/referer/accept', - headers=[('Referer', 'http://www.example.com/')]) - self.assertStatus(200) - self.assertBody('Accepted!') - - # Reject - self.getPage('/referer/reject') - self.assertStatus(200) - self.assertBody('Accepted!') - - self.getPage('/referer/reject', - headers=[('Referer', 'http://www.example.com/')]) - self.assertErrorPage(403, 'Forbidden Referer header.') - - -class AcceptTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_Accept_Tool(self): - # Test with no header provided - self.getPage('/accept/feed') - self.assertStatus(200) - self.assertInBody('Unknown Blog') - - # Specify exact media type - self.getPage('/accept/feed', headers=[('Accept', 'application/atom+xml')]) - self.assertStatus(200) - self.assertInBody('Unknown Blog') - - # Specify matching media range - self.getPage('/accept/feed', headers=[('Accept', 'application/*')]) - self.assertStatus(200) - self.assertInBody('Unknown Blog') - - # Specify all media ranges - self.getPage('/accept/feed', headers=[('Accept', '*/*')]) - self.assertStatus(200) - self.assertInBody('Unknown Blog') - - # Specify unacceptable media types - self.getPage('/accept/feed', headers=[('Accept', 'text/html')]) - self.assertErrorPage(406, - "Your client sent this Accept header: text/html. " - "But this resource only emits these media types: " - "application/atom+xml.") - - # Test resource where tool is 'on' but media is None (not set). - self.getPage('/accept/') - self.assertStatus(200) - self.assertBody('Atom feed') - - def test_accept_selection(self): - # Try both our expected media types - self.getPage('/accept/select', [('Accept', 'text/html')]) - self.assertStatus(200) - self.assertBody('

Page Title

') - self.getPage('/accept/select', [('Accept', 'text/plain')]) - self.assertStatus(200) - self.assertBody('PAGE TITLE') - self.getPage('/accept/select', [('Accept', 'text/plain, text/*;q=0.5')]) - self.assertStatus(200) - self.assertBody('PAGE TITLE') - - # text/* and */* should prefer text/html since it comes first - # in our 'media' argument to tools.accept - self.getPage('/accept/select', [('Accept', 'text/*')]) - self.assertStatus(200) - self.assertBody('

Page Title

') - self.getPage('/accept/select', [('Accept', '*/*')]) - self.assertStatus(200) - self.assertBody('

Page Title

') - - # Try unacceptable media types - self.getPage('/accept/select', [('Accept', 'application/xml')]) - self.assertErrorPage(406, - "Your client sent this Accept header: application/xml. " - "But this resource only emits these media types: " - "text/html, text/plain.") - - -class AutoVaryTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testAutoVary(self): - self.getPage('/autovary/') - self.assertHeader( - "Vary", 'Accept, Accept-Charset, Accept-Encoding, Host, If-Modified-Since, Range') - diff --git a/cherrypy/test/test_objectmapping.py b/cherrypy/test/test_objectmapping.py deleted file mode 100644 index 46816fcb..00000000 --- a/cherrypy/test/test_objectmapping.py +++ /dev/null @@ -1,403 +0,0 @@ -import cherrypy -from cherrypy._cptree import Application -from cherrypy.test import helper - -script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"] - - -class ObjectMappingTest(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self, name="world"): - return name - index.exposed = True - - def foobar(self): - return "bar" - foobar.exposed = True - - def default(self, *params, **kwargs): - return "default:" + repr(params) - default.exposed = True - - def other(self): - return "other" - other.exposed = True - - def extra(self, *p): - return repr(p) - extra.exposed = True - - def redirect(self): - raise cherrypy.HTTPRedirect('dir1/', 302) - redirect.exposed = True - - def notExposed(self): - return "not exposed" - - def confvalue(self): - return cherrypy.request.config.get("user") - confvalue.exposed = True - - def redirect_via_url(self, path): - raise cherrypy.HTTPRedirect(cherrypy.url(path)) - redirect_via_url.exposed = True - - def translate_html(self): - return "OK" - translate_html.exposed = True - - def mapped_func(self, ID=None): - return "ID is %s" % ID - mapped_func.exposed = True - setattr(Root, "Von B\xfclow", mapped_func) - - - class Exposing: - def base(self): - return "expose works!" - cherrypy.expose(base) - cherrypy.expose(base, "1") - cherrypy.expose(base, "2") - - class ExposingNewStyle(object): - def base(self): - return "expose works!" - cherrypy.expose(base) - cherrypy.expose(base, "1") - cherrypy.expose(base, "2") - - - class Dir1: - def index(self): - return "index for dir1" - index.exposed = True - - def myMethod(self): - return "myMethod from dir1, path_info is:" + repr(cherrypy.request.path_info) - myMethod.exposed = True - myMethod._cp_config = {'tools.trailing_slash.extra': True} - - def default(self, *params): - return "default for dir1, param is:" + repr(params) - default.exposed = True - - - class Dir2: - def index(self): - return "index for dir2, path is:" + cherrypy.request.path_info - index.exposed = True - - def script_name(self): - return cherrypy.tree.script_name() - script_name.exposed = True - - def cherrypy_url(self): - return cherrypy.url("/extra") - cherrypy_url.exposed = True - - def posparam(self, *vpath): - return "/".join(vpath) - posparam.exposed = True - - - class Dir3: - def default(self): - return "default for dir3, not exposed" - - class Dir4: - def index(self): - return "index for dir4, not exposed" - - class DefNoIndex: - def default(self, *args): - raise cherrypy.HTTPRedirect("contact") - default.exposed = True - - # MethodDispatcher code - class ByMethod: - exposed = True - - def __init__(self, *things): - self.things = list(things) - - def GET(self): - return repr(self.things) - - def POST(self, thing): - self.things.append(thing) - - class Collection: - default = ByMethod('a', 'bit') - - Root.exposing = Exposing() - Root.exposingnew = ExposingNewStyle() - Root.dir1 = Dir1() - Root.dir1.dir2 = Dir2() - Root.dir1.dir2.dir3 = Dir3() - Root.dir1.dir2.dir3.dir4 = Dir4() - Root.defnoindex = DefNoIndex() - Root.bymethod = ByMethod('another') - Root.collection = Collection() - - d = cherrypy.dispatch.MethodDispatcher() - for url in script_names: - conf = {'/': {'user': (url or "/").split("/")[-2]}, - '/bymethod': {'request.dispatch': d}, - '/collection': {'request.dispatch': d}, - } - cherrypy.tree.mount(Root(), url, conf) - - - class Isolated: - def index(self): - return "made it!" - index.exposed = True - - cherrypy.tree.mount(Isolated(), "/isolated") - - class AnotherApp: - - exposed = True - - def GET(self): - return "milk" - - cherrypy.tree.mount(AnotherApp(), "/app", {'/': {'request.dispatch': d}}) - setup_server = staticmethod(setup_server) - - - def testObjectMapping(self): - for url in script_names: - prefix = self.script_name = url - - self.getPage('/') - self.assertBody('world') - - self.getPage("/dir1/myMethod") - self.assertBody("myMethod from dir1, path_info is:'/dir1/myMethod'") - - self.getPage("/this/method/does/not/exist") - self.assertBody("default:('this', 'method', 'does', 'not', 'exist')") - - self.getPage("/extra/too/much") - self.assertBody("('too', 'much')") - - self.getPage("/other") - self.assertBody('other') - - self.getPage("/notExposed") - self.assertBody("default:('notExposed',)") - - self.getPage("/dir1/dir2/") - self.assertBody('index for dir2, path is:/dir1/dir2/') - - # Test omitted trailing slash (should be redirected by default). - self.getPage("/dir1/dir2") - self.assertStatus(301) - self.assertHeader('Location', '%s/dir1/dir2/' % self.base()) - - # Test extra trailing slash (should be redirected if configured). - self.getPage("/dir1/myMethod/") - self.assertStatus(301) - self.assertHeader('Location', '%s/dir1/myMethod' % self.base()) - - # Test that default method must be exposed in order to match. - self.getPage("/dir1/dir2/dir3/dir4/index") - self.assertBody("default for dir1, param is:('dir2', 'dir3', 'dir4', 'index')") - - # Test *vpath when default() is defined but not index() - # This also tests HTTPRedirect with default. - self.getPage("/defnoindex") - self.assertStatus((302, 303)) - self.assertHeader('Location', '%s/contact' % self.base()) - self.getPage("/defnoindex/") - self.assertStatus((302, 303)) - self.assertHeader('Location', '%s/defnoindex/contact' % self.base()) - self.getPage("/defnoindex/page") - self.assertStatus((302, 303)) - self.assertHeader('Location', '%s/defnoindex/contact' % self.base()) - - self.getPage("/redirect") - self.assertStatus('302 Found') - self.assertHeader('Location', '%s/dir1/' % self.base()) - - if not getattr(cherrypy.server, "using_apache", False): - # Test that we can use URL's which aren't all valid Python identifiers - # This should also test the %XX-unquoting of URL's. - self.getPage("/Von%20B%fclow?ID=14") - self.assertBody("ID is 14") - - # Test that %2F in the path doesn't get unquoted too early; - # that is, it should not be used to separate path components. - # See ticket #393. - self.getPage("/page%2Fname") - self.assertBody("default:('page/name',)") - - self.getPage("/dir1/dir2/script_name") - self.assertBody(url) - self.getPage("/dir1/dir2/cherrypy_url") - self.assertBody("%s/extra" % self.base()) - - # Test that configs don't overwrite each other from diferent apps - self.getPage("/confvalue") - self.assertBody((url or "/").split("/")[-2]) - - self.script_name = "" - - # Test absoluteURI's in the Request-Line - self.getPage('http://%s:%s/' % (self.interface(), self.PORT)) - self.assertBody('world') - - self.getPage('http://%s:%s/abs/?service=http://192.168.0.1/x/y/z' % - (self.interface(), self.PORT)) - self.assertBody("default:('abs',)") - - self.getPage('/rel/?service=http://192.168.120.121:8000/x/y/z') - self.assertBody("default:('rel',)") - - # Test that the "isolated" app doesn't leak url's into the root app. - # If it did leak, Root.default() would answer with - # "default:('isolated', 'doesnt', 'exist')". - self.getPage("/isolated/") - self.assertStatus("200 OK") - self.assertBody("made it!") - self.getPage("/isolated/doesnt/exist") - self.assertStatus("404 Not Found") - - # Make sure /foobar maps to Root.foobar and not to the app - # mounted at /foo. See http://www.cherrypy.org/ticket/573 - self.getPage("/foobar") - self.assertBody("bar") - - def test_translate(self): - self.getPage("/translate_html") - self.assertStatus("200 OK") - self.assertBody("OK") - - self.getPage("/translate.html") - self.assertStatus("200 OK") - self.assertBody("OK") - - self.getPage("/translate-html") - self.assertStatus("200 OK") - self.assertBody("OK") - - def test_redir_using_url(self): - for url in script_names: - prefix = self.script_name = url - - # Test the absolute path to the parent (leading slash) - self.getPage('/redirect_via_url?path=./') - self.assertStatus(('302 Found', '303 See Other')) - self.assertHeader('Location', '%s/' % self.base()) - - # Test the relative path to the parent (no leading slash) - self.getPage('/redirect_via_url?path=./') - self.assertStatus(('302 Found', '303 See Other')) - self.assertHeader('Location', '%s/' % self.base()) - - # Test the absolute path to the parent (leading slash) - self.getPage('/redirect_via_url/?path=./') - self.assertStatus(('302 Found', '303 See Other')) - self.assertHeader('Location', '%s/' % self.base()) - - # Test the relative path to the parent (no leading slash) - self.getPage('/redirect_via_url/?path=./') - self.assertStatus(('302 Found', '303 See Other')) - self.assertHeader('Location', '%s/' % self.base()) - - def testPositionalParams(self): - self.getPage("/dir1/dir2/posparam/18/24/hut/hike") - self.assertBody("18/24/hut/hike") - - # intermediate index methods should not receive posparams; - # only the "final" index method should do so. - self.getPage("/dir1/dir2/5/3/sir") - self.assertBody("default for dir1, param is:('dir2', '5', '3', 'sir')") - - # test that extra positional args raises an 404 Not Found - # See http://www.cherrypy.org/ticket/733. - self.getPage("/dir1/dir2/script_name/extra/stuff") - self.assertStatus(404) - - def testExpose(self): - # Test the cherrypy.expose function/decorator - self.getPage("/exposing/base") - self.assertBody("expose works!") - - self.getPage("/exposing/1") - self.assertBody("expose works!") - - self.getPage("/exposing/2") - self.assertBody("expose works!") - - self.getPage("/exposingnew/base") - self.assertBody("expose works!") - - self.getPage("/exposingnew/1") - self.assertBody("expose works!") - - self.getPage("/exposingnew/2") - self.assertBody("expose works!") - - def testMethodDispatch(self): - self.getPage("/bymethod") - self.assertBody("['another']") - self.assertHeader('Allow', 'GET, HEAD, POST') - - self.getPage("/bymethod", method="HEAD") - self.assertBody("") - self.assertHeader('Allow', 'GET, HEAD, POST') - - self.getPage("/bymethod", method="POST", body="thing=one") - self.assertBody("") - self.assertHeader('Allow', 'GET, HEAD, POST') - - self.getPage("/bymethod") - self.assertBody("['another', u'one']") - self.assertHeader('Allow', 'GET, HEAD, POST') - - self.getPage("/bymethod", method="PUT") - self.assertErrorPage(405) - self.assertHeader('Allow', 'GET, HEAD, POST') - - # Test default with posparams - self.getPage("/collection/silly", method="POST") - self.getPage("/collection", method="GET") - self.assertBody("['a', 'bit', 'silly']") - - # Test custom dispatcher set on app root (see #737). - self.getPage("/app") - self.assertBody("milk") - - def testTreeMounting(self): - class Root(object): - def hello(self): - return "Hello world!" - hello.exposed = True - - # When mounting an application instance, - # we can't specify a different script name in the call to mount. - a = Application(Root(), '/somewhere') - self.assertRaises(ValueError, cherrypy.tree.mount, a, '/somewhereelse') - - # When mounting an application instance... - a = Application(Root(), '/somewhere') - # ...we MUST allow in identical script name in the call to mount... - cherrypy.tree.mount(a, '/somewhere') - self.getPage('/somewhere/hello') - self.assertStatus(200) - # ...and MUST allow a missing script_name. - del cherrypy.tree.apps['/somewhere'] - cherrypy.tree.mount(a) - self.getPage('/somewhere/hello') - self.assertStatus(200) - - # In addition, we MUST be able to create an Application using - # script_name == None for access to the wsgi_environ. - a = Application(Root(), script_name=None) - # However, this does not apply to tree.mount - self.assertRaises(TypeError, cherrypy.tree.mount, a, None) - diff --git a/cherrypy/test/test_proxy.py b/cherrypy/test/test_proxy.py deleted file mode 100644 index 2fbb619a..00000000 --- a/cherrypy/test/test_proxy.py +++ /dev/null @@ -1,129 +0,0 @@ -import cherrypy -from cherrypy.test import helper - -script_names = ["", "/path/to/myapp"] - - -class ProxyTest(helper.CPWebCase): - - def setup_server(): - - # Set up site - cherrypy.config.update({ - 'tools.proxy.on': True, - 'tools.proxy.base': 'www.mydomain.test', - }) - - # Set up application - - class Root: - - def __init__(self, sn): - # Calculate a URL outside of any requests. - self.thisnewpage = cherrypy.url("/this/new/page", script_name=sn) - - def pageurl(self): - return self.thisnewpage - pageurl.exposed = True - - def index(self): - raise cherrypy.HTTPRedirect('dummy') - index.exposed = True - - def remoteip(self): - return cherrypy.request.remote.ip - remoteip.exposed = True - - def xhost(self): - raise cherrypy.HTTPRedirect('blah') - xhost.exposed = True - xhost._cp_config = {'tools.proxy.local': 'X-Host', - 'tools.trailing_slash.extra': True, - } - - def base(self): - return cherrypy.request.base - base.exposed = True - - def ssl(self): - return cherrypy.request.base - ssl.exposed = True - ssl._cp_config = {'tools.proxy.scheme': 'X-Forwarded-Ssl'} - - def newurl(self): - return ("Browse to this page." - % cherrypy.url("/this/new/page")) - newurl.exposed = True - - for sn in script_names: - cherrypy.tree.mount(Root(sn), sn) - setup_server = staticmethod(setup_server) - - def testProxy(self): - self.getPage("/") - self.assertHeader('Location', - "%s://www.mydomain.test%s/dummy" % - (self.scheme, self.prefix())) - - # Test X-Forwarded-Host (Apache 1.3.33+ and Apache 2) - self.getPage("/", headers=[('X-Forwarded-Host', 'http://www.example.test')]) - self.assertHeader('Location', "http://www.example.test/dummy") - self.getPage("/", headers=[('X-Forwarded-Host', 'www.example.test')]) - self.assertHeader('Location', "%s://www.example.test/dummy" % self.scheme) - # Test multiple X-Forwarded-Host headers - self.getPage("/", headers=[ - ('X-Forwarded-Host', 'http://www.example.test, www.cherrypy.test'), - ]) - self.assertHeader('Location', "http://www.example.test/dummy") - - # Test X-Forwarded-For (Apache2) - self.getPage("/remoteip", - headers=[('X-Forwarded-For', '192.168.0.20')]) - self.assertBody("192.168.0.20") - self.getPage("/remoteip", - headers=[('X-Forwarded-For', '67.15.36.43, 192.168.0.20')]) - self.assertBody("192.168.0.20") - - # Test X-Host (lighttpd; see https://trac.lighttpd.net/trac/ticket/418) - self.getPage("/xhost", headers=[('X-Host', 'www.example.test')]) - self.assertHeader('Location', "%s://www.example.test/blah" % self.scheme) - - # Test X-Forwarded-Proto (lighttpd) - self.getPage("/base", headers=[('X-Forwarded-Proto', 'https')]) - self.assertBody("https://www.mydomain.test") - - # Test X-Forwarded-Ssl (webfaction?) - self.getPage("/ssl", headers=[('X-Forwarded-Ssl', 'on')]) - self.assertBody("https://www.mydomain.test") - - # Test cherrypy.url() - for sn in script_names: - # Test the value inside requests - self.getPage(sn + "/newurl") - self.assertBody("Browse to this page.") - self.getPage(sn + "/newurl", headers=[('X-Forwarded-Host', - 'http://www.example.test')]) - self.assertBody("Browse to this page.") - - # Test the value outside requests - port = "" - if self.scheme == "http" and self.PORT != 80: - port = ":%s" % self.PORT - elif self.scheme == "https" and self.PORT != 443: - port = ":%s" % self.PORT - host = self.HOST - if host in ('0.0.0.0', '::'): - import socket - host = socket.gethostname() - expected = ("%s://%s%s%s/this/new/page" - % (self.scheme, host, port, sn)) - self.getPage(sn + "/pageurl") - self.assertBody(expected) - - # Test trailing slash (see http://www.cherrypy.org/ticket/562). - self.getPage("/xhost/", headers=[('X-Host', 'www.example.test')]) - self.assertHeader('Location', "%s://www.example.test/xhost" - % self.scheme) - diff --git a/cherrypy/test/test_refleaks.py b/cherrypy/test/test_refleaks.py deleted file mode 100644 index 4df1f082..00000000 --- a/cherrypy/test/test_refleaks.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Tests for refleaks.""" - -import gc -from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob -import threading - -import cherrypy -from cherrypy import _cprequest - - -data = object() - -def get_instances(cls): - return [x for x in gc.get_objects() if isinstance(x, cls)] - - -from cherrypy.test import helper - - -class ReferenceTests(helper.CPWebCase): - - def setup_server(): - - class Root: - def index(self, *args, **kwargs): - cherrypy.request.thing = data - return "Hello world!" - index.exposed = True - - def gc_stats(self): - output = ["Statistics:"] - - # Uncollectable garbage - - # gc_collect isn't perfectly synchronous, because it may - # break reference cycles that then take time to fully - # finalize. Call it twice and hope for the best. - gc.collect() - unreachable = gc.collect() - if unreachable: - output.append("\n%s unreachable objects:" % unreachable) - trash = {} - for x in gc.garbage: - trash[type(x)] = trash.get(type(x), 0) + 1 - trash = [(v, k) for k, v in trash.items()] - trash.sort() - for pair in trash: - output.append(" " + repr(pair)) - - # Request references - reqs = get_instances(_cprequest.Request) - lenreqs = len(reqs) - if lenreqs < 2: - output.append("\nMissing Request reference. Should be 1 in " - "this request thread and 1 in the main thread.") - elif lenreqs > 2: - output.append("\nToo many Request references (%r)." % lenreqs) - for req in reqs: - output.append("Referrers for %s:" % repr(req)) - for ref in gc.get_referrers(req): - if ref is not reqs: - output.append(" %s" % repr(ref)) - - # Response references - resps = get_instances(_cprequest.Response) - lenresps = len(resps) - if lenresps < 2: - output.append("\nMissing Response reference. Should be 1 in " - "this request thread and 1 in the main thread.") - elif lenresps > 2: - output.append("\nToo many Response references (%r)." % lenresps) - for resp in resps: - output.append("Referrers for %s:" % repr(resp)) - for ref in gc.get_referrers(resp): - if ref is not resps: - output.append(" %s" % repr(ref)) - - return "\n".join(output) - gc_stats.exposed = True - - cherrypy.tree.mount(Root()) - setup_server = staticmethod(setup_server) - - - def test_threadlocal_garbage(self): - success = [] - - def getpage(): - host = '%s:%s' % (self.interface(), self.PORT) - if self.scheme == 'https': - c = HTTPSConnection(host) - else: - c = HTTPConnection(host) - try: - c.putrequest('GET', '/') - c.endheaders() - response = c.getresponse() - body = response.read() - self.assertEqual(response.status, 200) - self.assertEqual(body, ntob("Hello world!")) - finally: - c.close() - success.append(True) - - ITERATIONS = 25 - ts = [] - for _ in range(ITERATIONS): - t = threading.Thread(target=getpage) - ts.append(t) - t.start() - - for t in ts: - t.join() - - self.assertEqual(len(success), ITERATIONS) - - self.getPage("/gc_stats") - self.assertBody("Statistics:") - diff --git a/cherrypy/test/test_request_obj.py b/cherrypy/test/test_request_obj.py deleted file mode 100644 index 91ee4fd0..00000000 --- a/cherrypy/test/test_request_obj.py +++ /dev/null @@ -1,722 +0,0 @@ -"""Basic tests for the cherrypy.Request object.""" - -import os -localDir = os.path.dirname(__file__) -import sys -import types -from cherrypy._cpcompat import IncompleteRead, ntob, unicodestr - -import cherrypy -from cherrypy import _cptools, tools -from cherrypy.lib import httputil - -defined_http_methods = ("OPTIONS", "GET", "HEAD", "POST", "PUT", "DELETE", - "TRACE", "PROPFIND") - - -# Client-side code # - -from cherrypy.test import helper - -class RequestObjectTests(helper.CPWebCase): - - def setup_server(): - class Root: - - def index(self): - return "hello" - index.exposed = True - - def scheme(self): - return cherrypy.request.scheme - scheme.exposed = True - - root = Root() - - - class TestType(type): - """Metaclass which automatically exposes all functions in each subclass, - and adds an instance of the subclass as an attribute of root. - """ - def __init__(cls, name, bases, dct): - type.__init__(cls, name, bases, dct) - for value in dct.values(): - if isinstance(value, types.FunctionType): - value.exposed = True - setattr(root, name.lower(), cls()) - class Test(object): - __metaclass__ = TestType - - - class Params(Test): - - def index(self, thing): - return repr(thing) - - def ismap(self, x, y): - return "Coordinates: %s, %s" % (x, y) - - def default(self, *args, **kwargs): - return "args: %s kwargs: %s" % (args, kwargs) - default._cp_config = {'request.query_string_encoding': 'latin1'} - - - class ParamErrorsCallable(object): - exposed = True - def __call__(self): - return "data" - - class ParamErrors(Test): - - def one_positional(self, param1): - return "data" - one_positional.exposed = True - - def one_positional_args(self, param1, *args): - return "data" - one_positional_args.exposed = True - - def one_positional_args_kwargs(self, param1, *args, **kwargs): - return "data" - one_positional_args_kwargs.exposed = True - - def one_positional_kwargs(self, param1, **kwargs): - return "data" - one_positional_kwargs.exposed = True - - def no_positional(self): - return "data" - no_positional.exposed = True - - def no_positional_args(self, *args): - return "data" - no_positional_args.exposed = True - - def no_positional_args_kwargs(self, *args, **kwargs): - return "data" - no_positional_args_kwargs.exposed = True - - def no_positional_kwargs(self, **kwargs): - return "data" - no_positional_kwargs.exposed = True - - callable_object = ParamErrorsCallable() - - def raise_type_error(self, **kwargs): - raise TypeError("Client Error") - raise_type_error.exposed = True - - def raise_type_error_with_default_param(self, x, y=None): - return '%d' % 'a' # throw an exception - raise_type_error_with_default_param.exposed = True - - def callable_error_page(status, **kwargs): - return "Error %s - Well, I'm very sorry but you haven't paid!" % status - - - class Error(Test): - - _cp_config = {'tools.log_tracebacks.on': True, - } - - def reason_phrase(self): - raise cherrypy.HTTPError("410 Gone fishin'") - - def custom(self, err='404'): - raise cherrypy.HTTPError(int(err), "No, really, not found!") - custom._cp_config = {'error_page.404': os.path.join(localDir, "static/index.html"), - 'error_page.401': callable_error_page, - } - - def custom_default(self): - return 1 + 'a' # raise an unexpected error - custom_default._cp_config = {'error_page.default': callable_error_page} - - def noexist(self): - raise cherrypy.HTTPError(404, "No, really, not found!") - noexist._cp_config = {'error_page.404': "nonexistent.html"} - - def page_method(self): - raise ValueError() - - def page_yield(self): - yield "howdy" - raise ValueError() - - def page_streamed(self): - yield "word up" - raise ValueError() - yield "very oops" - page_streamed._cp_config = {"response.stream": True} - - def cause_err_in_finalize(self): - # Since status must start with an int, this should error. - cherrypy.response.status = "ZOO OK" - cause_err_in_finalize._cp_config = {'request.show_tracebacks': False} - - def rethrow(self): - """Test that an error raised here will be thrown out to the server.""" - raise ValueError() - rethrow._cp_config = {'request.throw_errors': True} - - - class Expect(Test): - - def expectation_failed(self): - expect = cherrypy.request.headers.elements("Expect") - if expect and expect[0].value != '100-continue': - raise cherrypy.HTTPError(400) - raise cherrypy.HTTPError(417, 'Expectation Failed') - - class Headers(Test): - - def default(self, headername): - """Spit back out the value for the requested header.""" - return cherrypy.request.headers[headername] - - def doubledheaders(self): - # From http://www.cherrypy.org/ticket/165: - # "header field names should not be case sensitive sayes the rfc. - # if i set a headerfield in complete lowercase i end up with two - # header fields, one in lowercase, the other in mixed-case." - - # Set the most common headers - hMap = cherrypy.response.headers - hMap['content-type'] = "text/html" - hMap['content-length'] = 18 - hMap['server'] = 'CherryPy headertest' - hMap['location'] = ('%s://%s:%s/headers/' - % (cherrypy.request.local.ip, - cherrypy.request.local.port, - cherrypy.request.scheme)) - - # Set a rare header for fun - hMap['Expires'] = 'Thu, 01 Dec 2194 16:00:00 GMT' - - return "double header test" - - def ifmatch(self): - val = cherrypy.request.headers['If-Match'] - assert isinstance(val, unicodestr) - cherrypy.response.headers['ETag'] = val - return val - - - class HeaderElements(Test): - - def get_elements(self, headername): - e = cherrypy.request.headers.elements(headername) - return "\n".join([unicodestr(x) for x in e]) - - - class Method(Test): - - def index(self): - m = cherrypy.request.method - if m in defined_http_methods or m == "CONNECT": - return m - - if m == "LINK": - raise cherrypy.HTTPError(405) - else: - raise cherrypy.HTTPError(501) - - def parameterized(self, data): - return data - - def request_body(self): - # This should be a file object (temp file), - # which CP will just pipe back out if we tell it to. - return cherrypy.request.body - - def reachable(self): - return "success" - - class Divorce: - """HTTP Method handlers shouldn't collide with normal method names. - For example, a GET-handler shouldn't collide with a method named 'get'. - - If you build HTTP method dispatching into CherryPy, rewrite this class - to use your new dispatch mechanism and make sure that: - "GET /divorce HTTP/1.1" maps to divorce.index() and - "GET /divorce/get?ID=13 HTTP/1.1" maps to divorce.get() - """ - - documents = {} - - def index(self): - yield "

Choose your document

\n" - yield "
    \n" - for id, contents in self.documents.items(): - yield ("
  • %s: %s
  • \n" - % (id, id, contents)) - yield "
" - index.exposed = True - - def get(self, ID): - return ("Divorce document %s: %s" % - (ID, self.documents.get(ID, "empty"))) - get.exposed = True - - root.divorce = Divorce() - - - class ThreadLocal(Test): - - def index(self): - existing = repr(getattr(cherrypy.request, "asdf", None)) - cherrypy.request.asdf = "rassfrassin" - return existing - - appconf = { - '/method': {'request.methods_with_bodies': ("POST", "PUT", "PROPFIND")}, - } - cherrypy.tree.mount(root, config=appconf) - setup_server = staticmethod(setup_server) - - def test_scheme(self): - self.getPage("/scheme") - self.assertBody(self.scheme) - - def testParams(self): - self.getPage("/params/?thing=a") - self.assertBody("u'a'") - - self.getPage("/params/?thing=a&thing=b&thing=c") - self.assertBody("[u'a', u'b', u'c']") - - # Test friendly error message when given params are not accepted. - cherrypy.config.update({"request.show_mismatched_params": True}) - self.getPage("/params/?notathing=meeting") - self.assertInBody("Missing parameters: thing") - self.getPage("/params/?thing=meeting¬athing=meeting") - self.assertInBody("Unexpected query string parameters: notathing") - - # Test ability to turn off friendly error messages - cherrypy.config.update({"request.show_mismatched_params": False}) - self.getPage("/params/?notathing=meeting") - self.assertInBody("Not Found") - self.getPage("/params/?thing=meeting¬athing=meeting") - self.assertInBody("Not Found") - - # Test "% HEX HEX"-encoded URL, param keys, and values - self.getPage("/params/%d4%20%e3/cheese?Gruy%E8re=Bulgn%e9ville") - self.assertBody(r"args: ('\xd4 \xe3', 'cheese') " - r"kwargs: {'Gruy\xe8re': u'Bulgn\xe9ville'}") - - # Make sure that encoded = and & get parsed correctly - self.getPage("/params/code?url=http%3A//cherrypy.org/index%3Fa%3D1%26b%3D2") - self.assertBody(r"args: ('code',) " - r"kwargs: {'url': u'http://cherrypy.org/index?a=1&b=2'}") - - # Test coordinates sent by - self.getPage("/params/ismap?223,114") - self.assertBody("Coordinates: 223, 114") - - # Test "name[key]" dict-like params - self.getPage("/params/dictlike?a[1]=1&a[2]=2&b=foo&b[bar]=baz") - self.assertBody( - "args: ('dictlike',) " - "kwargs: {'a[1]': u'1', 'b[bar]': u'baz', 'b': u'foo', 'a[2]': u'2'}") - - def testParamErrors(self): - - # test that all of the handlers work when given - # the correct parameters in order to ensure that the - # errors below aren't coming from some other source. - for uri in ( - '/paramerrors/one_positional?param1=foo', - '/paramerrors/one_positional_args?param1=foo', - '/paramerrors/one_positional_args/foo', - '/paramerrors/one_positional_args/foo/bar/baz', - '/paramerrors/one_positional_args_kwargs?param1=foo¶m2=bar', - '/paramerrors/one_positional_args_kwargs/foo?param2=bar¶m3=baz', - '/paramerrors/one_positional_args_kwargs/foo/bar/baz?param2=bar¶m3=baz', - '/paramerrors/one_positional_kwargs?param1=foo¶m2=bar¶m3=baz', - '/paramerrors/one_positional_kwargs/foo?param4=foo¶m2=bar¶m3=baz', - '/paramerrors/no_positional', - '/paramerrors/no_positional_args/foo', - '/paramerrors/no_positional_args/foo/bar/baz', - '/paramerrors/no_positional_args_kwargs?param1=foo¶m2=bar', - '/paramerrors/no_positional_args_kwargs/foo?param2=bar', - '/paramerrors/no_positional_args_kwargs/foo/bar/baz?param2=bar¶m3=baz', - '/paramerrors/no_positional_kwargs?param1=foo¶m2=bar', - '/paramerrors/callable_object', - ): - self.getPage(uri) - self.assertStatus(200) - - # query string parameters are part of the URI, so if they are wrong - # for a particular handler, the status MUST be a 404. - error_msgs = [ - 'Missing parameters', - 'Nothing matches the given URI', - 'Multiple values for parameters', - 'Unexpected query string parameters', - 'Unexpected body parameters', - ] - for uri, msg in ( - ('/paramerrors/one_positional', error_msgs[0]), - ('/paramerrors/one_positional?foo=foo', error_msgs[0]), - ('/paramerrors/one_positional/foo/bar/baz', error_msgs[1]), - ('/paramerrors/one_positional/foo?param1=foo', error_msgs[2]), - ('/paramerrors/one_positional/foo?param1=foo¶m2=foo', error_msgs[2]), - ('/paramerrors/one_positional_args/foo?param1=foo¶m2=foo', error_msgs[2]), - ('/paramerrors/one_positional_args/foo/bar/baz?param2=foo', error_msgs[3]), - ('/paramerrors/one_positional_args_kwargs/foo/bar/baz?param1=bar¶m3=baz', error_msgs[2]), - ('/paramerrors/one_positional_kwargs/foo?param1=foo¶m2=bar¶m3=baz', error_msgs[2]), - ('/paramerrors/no_positional/boo', error_msgs[1]), - ('/paramerrors/no_positional?param1=foo', error_msgs[3]), - ('/paramerrors/no_positional_args/boo?param1=foo', error_msgs[3]), - ('/paramerrors/no_positional_kwargs/boo?param1=foo', error_msgs[1]), - ('/paramerrors/callable_object?param1=foo', error_msgs[3]), - ('/paramerrors/callable_object/boo', error_msgs[1]), - ): - for show_mismatched_params in (True, False): - cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) - self.getPage(uri) - self.assertStatus(404) - if show_mismatched_params: - self.assertInBody(msg) - else: - self.assertInBody("Not Found") - - # if body parameters are wrong, a 400 must be returned. - for uri, body, msg in ( - ('/paramerrors/one_positional/foo', 'param1=foo', error_msgs[2]), - ('/paramerrors/one_positional/foo', 'param1=foo¶m2=foo', error_msgs[2]), - ('/paramerrors/one_positional_args/foo', 'param1=foo¶m2=foo', error_msgs[2]), - ('/paramerrors/one_positional_args/foo/bar/baz', 'param2=foo', error_msgs[4]), - ('/paramerrors/one_positional_args_kwargs/foo/bar/baz', 'param1=bar¶m3=baz', error_msgs[2]), - ('/paramerrors/one_positional_kwargs/foo', 'param1=foo¶m2=bar¶m3=baz', error_msgs[2]), - ('/paramerrors/no_positional', 'param1=foo', error_msgs[4]), - ('/paramerrors/no_positional_args/boo', 'param1=foo', error_msgs[4]), - ('/paramerrors/callable_object', 'param1=foo', error_msgs[4]), - ): - for show_mismatched_params in (True, False): - cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) - self.getPage(uri, method='POST', body=body) - self.assertStatus(400) - if show_mismatched_params: - self.assertInBody(msg) - else: - self.assertInBody("Bad Request") - - - # even if body parameters are wrong, if we get the uri wrong, then - # it's a 404 - for uri, body, msg in ( - ('/paramerrors/one_positional?param2=foo', 'param1=foo', error_msgs[3]), - ('/paramerrors/one_positional/foo/bar', 'param2=foo', error_msgs[1]), - ('/paramerrors/one_positional_args/foo/bar?param2=foo', 'param3=foo', error_msgs[3]), - ('/paramerrors/one_positional_kwargs/foo/bar', 'param2=bar¶m3=baz', error_msgs[1]), - ('/paramerrors/no_positional?param1=foo', 'param2=foo', error_msgs[3]), - ('/paramerrors/no_positional_args/boo?param2=foo', 'param1=foo', error_msgs[3]), - ('/paramerrors/callable_object?param2=bar', 'param1=foo', error_msgs[3]), - ): - for show_mismatched_params in (True, False): - cherrypy.config.update({'request.show_mismatched_params': show_mismatched_params}) - self.getPage(uri, method='POST', body=body) - self.assertStatus(404) - if show_mismatched_params: - self.assertInBody(msg) - else: - self.assertInBody("Not Found") - - # In the case that a handler raises a TypeError we should - # let that type error through. - for uri in ( - '/paramerrors/raise_type_error', - '/paramerrors/raise_type_error_with_default_param?x=0', - '/paramerrors/raise_type_error_with_default_param?x=0&y=0', - ): - self.getPage(uri, method='GET') - self.assertStatus(500) - self.assertTrue('Client Error', self.body) - - def testErrorHandling(self): - self.getPage("/error/missing") - self.assertStatus(404) - self.assertErrorPage(404, "The path '/error/missing' was not found.") - - ignore = helper.webtest.ignored_exceptions - ignore.append(ValueError) - try: - valerr = '\n raise ValueError()\nValueError' - self.getPage("/error/page_method") - self.assertErrorPage(500, pattern=valerr) - - self.getPage("/error/page_yield") - self.assertErrorPage(500, pattern=valerr) - - if (cherrypy.server.protocol_version == "HTTP/1.0" or - getattr(cherrypy.server, "using_apache", False)): - self.getPage("/error/page_streamed") - # Because this error is raised after the response body has - # started, the status should not change to an error status. - self.assertStatus(200) - self.assertBody("word up") - else: - # Under HTTP/1.1, the chunked transfer-coding is used. - # The HTTP client will choke when the output is incomplete. - self.assertRaises((ValueError, IncompleteRead), self.getPage, - "/error/page_streamed") - - # No traceback should be present - self.getPage("/error/cause_err_in_finalize") - msg = "Illegal response status from server ('ZOO' is non-numeric)." - self.assertErrorPage(500, msg, None) - finally: - ignore.pop() - - # Test HTTPError with a reason-phrase in the status arg. - self.getPage('/error/reason_phrase') - self.assertStatus("410 Gone fishin'") - - # Test custom error page for a specific error. - self.getPage("/error/custom") - self.assertStatus(404) - self.assertBody("Hello, world\r\n" + (" " * 499)) - - # Test custom error page for a specific error. - self.getPage("/error/custom?err=401") - self.assertStatus(401) - self.assertBody("Error 401 Unauthorized - Well, I'm very sorry but you haven't paid!") - - # Test default custom error page. - self.getPage("/error/custom_default") - self.assertStatus(500) - self.assertBody("Error 500 Internal Server Error - Well, I'm very sorry but you haven't paid!".ljust(513)) - - # Test error in custom error page (ticket #305). - # Note that the message is escaped for HTML (ticket #310). - self.getPage("/error/noexist") - self.assertStatus(404) - msg = ("No, <b>really</b>, not found!
" - "In addition, the custom error page failed:\n
" - "IOError: [Errno 2] No such file or directory: 'nonexistent.html'") - self.assertInBody(msg) - - if getattr(cherrypy.server, "using_apache", False): - pass - else: - # Test throw_errors (ticket #186). - self.getPage("/error/rethrow") - self.assertInBody("raise ValueError()") - - def testExpect(self): - e = ('Expect', '100-continue') - self.getPage("/headerelements/get_elements?headername=Expect", [e]) - self.assertBody('100-continue') - - self.getPage("/expect/expectation_failed", [e]) - self.assertStatus(417) - - def testHeaderElements(self): - # Accept-* header elements should be sorted, with most preferred first. - h = [('Accept', 'audio/*; q=0.2, audio/basic')] - self.getPage("/headerelements/get_elements?headername=Accept", h) - self.assertStatus(200) - self.assertBody("audio/basic\n" - "audio/*;q=0.2") - - h = [('Accept', 'text/plain; q=0.5, text/html, text/x-dvi; q=0.8, text/x-c')] - self.getPage("/headerelements/get_elements?headername=Accept", h) - self.assertStatus(200) - self.assertBody("text/x-c\n" - "text/html\n" - "text/x-dvi;q=0.8\n" - "text/plain;q=0.5") - - # Test that more specific media ranges get priority. - h = [('Accept', 'text/*, text/html, text/html;level=1, */*')] - self.getPage("/headerelements/get_elements?headername=Accept", h) - self.assertStatus(200) - self.assertBody("text/html;level=1\n" - "text/html\n" - "text/*\n" - "*/*") - - # Test Accept-Charset - h = [('Accept-Charset', 'iso-8859-5, unicode-1-1;q=0.8')] - self.getPage("/headerelements/get_elements?headername=Accept-Charset", h) - self.assertStatus("200 OK") - self.assertBody("iso-8859-5\n" - "unicode-1-1;q=0.8") - - # Test Accept-Encoding - h = [('Accept-Encoding', 'gzip;q=1.0, identity; q=0.5, *;q=0')] - self.getPage("/headerelements/get_elements?headername=Accept-Encoding", h) - self.assertStatus("200 OK") - self.assertBody("gzip;q=1.0\n" - "identity;q=0.5\n" - "*;q=0") - - # Test Accept-Language - h = [('Accept-Language', 'da, en-gb;q=0.8, en;q=0.7')] - self.getPage("/headerelements/get_elements?headername=Accept-Language", h) - self.assertStatus("200 OK") - self.assertBody("da\n" - "en-gb;q=0.8\n" - "en;q=0.7") - - # Test malformed header parsing. See http://www.cherrypy.org/ticket/763. - self.getPage("/headerelements/get_elements?headername=Content-Type", - # Note the illegal trailing ";" - headers=[('Content-Type', 'text/html; charset=utf-8;')]) - self.assertStatus(200) - self.assertBody("text/html;charset=utf-8") - - def test_repeated_headers(self): - # Test that two request headers are collapsed into one. - # See http://www.cherrypy.org/ticket/542. - self.getPage("/headers/Accept-Charset", - headers=[("Accept-Charset", "iso-8859-5"), - ("Accept-Charset", "unicode-1-1;q=0.8")]) - self.assertBody("iso-8859-5, unicode-1-1;q=0.8") - - # Tests that each header only appears once, regardless of case. - self.getPage("/headers/doubledheaders") - self.assertBody("double header test") - hnames = [name.title() for name, val in self.headers] - for key in ['Content-Length', 'Content-Type', 'Date', - 'Expires', 'Location', 'Server']: - self.assertEqual(hnames.count(key), 1, self.headers) - - def test_encoded_headers(self): - # First, make sure the innards work like expected. - self.assertEqual(httputil.decode_TEXT(u"=?utf-8?q?f=C3=BCr?="), u"f\xfcr") - - if cherrypy.server.protocol_version == "HTTP/1.1": - # Test RFC-2047-encoded request and response header values - u = u'\u212bngstr\xf6m' - c = u"=E2=84=ABngstr=C3=B6m" - self.getPage("/headers/ifmatch", [('If-Match', u'=?utf-8?q?%s?=' % c)]) - # The body should be utf-8 encoded. - self.assertBody("\xe2\x84\xabngstr\xc3\xb6m") - # But the Etag header should be RFC-2047 encoded (binary) - self.assertHeader("ETag", u'=?utf-8?b?4oSrbmdzdHLDtm0=?=') - - # Test a *LONG* RFC-2047-encoded request and response header value - self.getPage("/headers/ifmatch", - [('If-Match', u'=?utf-8?q?%s?=' % (c * 10))]) - self.assertBody("\xe2\x84\xabngstr\xc3\xb6m" * 10) - # Note: this is different output for Python3, but it decodes fine. - etag = self.assertHeader("ETag", - '=?utf-8?b?4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' - '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' - '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt' - '4oSrbmdzdHLDtm0=?=') - self.assertEqual(httputil.decode_TEXT(etag), u * 10) - - def test_header_presence(self): - # If we don't pass a Content-Type header, it should not be present - # in cherrypy.request.headers - self.getPage("/headers/Content-Type", - headers=[]) - self.assertStatus(500) - - # If Content-Type is present in the request, it should be present in - # cherrypy.request.headers - self.getPage("/headers/Content-Type", - headers=[("Content-type", "application/json")]) - self.assertBody("application/json") - - def test_basic_HTTPMethods(self): - helper.webtest.methods_with_bodies = ("POST", "PUT", "PROPFIND") - - # Test that all defined HTTP methods work. - for m in defined_http_methods: - self.getPage("/method/", method=m) - - # HEAD requests should not return any body. - if m == "HEAD": - self.assertBody("") - elif m == "TRACE": - # Some HTTP servers (like modpy) have their own TRACE support - self.assertEqual(self.body[:5], ntob("TRACE")) - else: - self.assertBody(m) - - # Request a PUT method with a form-urlencoded body - self.getPage("/method/parameterized", method="PUT", - body="data=on+top+of+other+things") - self.assertBody("on top of other things") - - # Request a PUT method with a file body - b = "one thing on top of another" - h = [("Content-Type", "text/plain"), - ("Content-Length", str(len(b)))] - self.getPage("/method/request_body", headers=h, method="PUT", body=b) - self.assertStatus(200) - self.assertBody(b) - - # Request a PUT method with a file body but no Content-Type. - # See http://www.cherrypy.org/ticket/790. - b = ntob("one thing on top of another") - self.persistent = True - try: - conn = self.HTTP_CONN - conn.putrequest("PUT", "/method/request_body", skip_host=True) - conn.putheader("Host", self.HOST) - conn.putheader('Content-Length', str(len(b))) - conn.endheaders() - conn.send(b) - response = conn.response_class(conn.sock, method="PUT") - response.begin() - self.assertEqual(response.status, 200) - self.body = response.read() - self.assertBody(b) - finally: - self.persistent = False - - # Request a PUT method with no body whatsoever (not an empty one). - # See http://www.cherrypy.org/ticket/650. - # Provide a C-T or webtest will provide one (and a C-L) for us. - h = [("Content-Type", "text/plain")] - self.getPage("/method/reachable", headers=h, method="PUT") - self.assertStatus(411) - - # Request a custom method with a request body - b = ('\n\n' - '' - '') - h = [('Content-Type', 'text/xml'), - ('Content-Length', str(len(b)))] - self.getPage("/method/request_body", headers=h, method="PROPFIND", body=b) - self.assertStatus(200) - self.assertBody(b) - - # Request a disallowed method - self.getPage("/method/", method="LINK") - self.assertStatus(405) - - # Request an unknown method - self.getPage("/method/", method="SEARCH") - self.assertStatus(501) - - # For method dispatchers: make sure that an HTTP method doesn't - # collide with a virtual path atom. If you build HTTP-method - # dispatching into the core, rewrite these handlers to use - # your dispatch idioms. - self.getPage("/divorce/get?ID=13") - self.assertBody('Divorce document 13: empty') - self.assertStatus(200) - self.getPage("/divorce/", method="GET") - self.assertBody('

Choose your document

\n
    \n
') - self.assertStatus(200) - - def test_CONNECT_method(self): - if getattr(cherrypy.server, "using_apache", False): - return self.skip("skipped due to known Apache differences... ") - - self.getPage("/method/", method="CONNECT") - self.assertBody("CONNECT") - - def testEmptyThreadlocals(self): - results = [] - for x in range(20): - self.getPage("/threadlocal/") - results.append(self.body) - self.assertEqual(results, [ntob("None")] * 20) - diff --git a/cherrypy/test/test_routes.py b/cherrypy/test/test_routes.py deleted file mode 100644 index a8062f8f..00000000 --- a/cherrypy/test/test_routes.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - -import cherrypy - -from cherrypy.test import helper -import nose - -class RoutesDispatchTest(helper.CPWebCase): - - def setup_server(): - - try: - import routes - except ImportError: - raise nose.SkipTest('Install routes to test RoutesDispatcher code') - - class Dummy: - def index(self): - return "I said good day!" - - class City: - - def __init__(self, name): - self.name = name - self.population = 10000 - - def index(self, **kwargs): - return "Welcome to %s, pop. %s" % (self.name, self.population) - index._cp_config = {'tools.response_headers.on': True, - 'tools.response_headers.headers': [('Content-Language', 'en-GB')]} - - def update(self, **kwargs): - self.population = kwargs['pop'] - return "OK" - - d = cherrypy.dispatch.RoutesDispatcher() - d.connect(action='index', name='hounslow', route='/hounslow', - controller=City('Hounslow')) - d.connect(name='surbiton', route='/surbiton', controller=City('Surbiton'), - action='index', conditions=dict(method=['GET'])) - d.mapper.connect('/surbiton', controller='surbiton', - action='update', conditions=dict(method=['POST'])) - d.connect('main', ':action', controller=Dummy()) - - conf = {'/': {'request.dispatch': d}} - cherrypy.tree.mount(root=None, config=conf) - setup_server = staticmethod(setup_server) - - def test_Routes_Dispatch(self): - self.getPage("/hounslow") - self.assertStatus("200 OK") - self.assertBody("Welcome to Hounslow, pop. 10000") - - self.getPage("/foo") - self.assertStatus("404 Not Found") - - self.getPage("/surbiton") - self.assertStatus("200 OK") - self.assertBody("Welcome to Surbiton, pop. 10000") - - self.getPage("/surbiton", method="POST", body="pop=1327") - self.assertStatus("200 OK") - self.assertBody("OK") - self.getPage("/surbiton") - self.assertStatus("200 OK") - self.assertHeader("Content-Language", "en-GB") - self.assertBody("Welcome to Surbiton, pop. 1327") - diff --git a/cherrypy/test/test_session.py b/cherrypy/test/test_session.py deleted file mode 100755 index 874023e2..00000000 --- a/cherrypy/test/test_session.py +++ /dev/null @@ -1,464 +0,0 @@ -import os -localDir = os.path.dirname(__file__) -import sys -import threading -import time - -import cherrypy -from cherrypy._cpcompat import copykeys, HTTPConnection, HTTPSConnection -from cherrypy.lib import sessions -from cherrypy.lib.httputil import response_codes - -def http_methods_allowed(methods=['GET', 'HEAD']): - method = cherrypy.request.method.upper() - if method not in methods: - cherrypy.response.headers['Allow'] = ", ".join(methods) - raise cherrypy.HTTPError(405) - -cherrypy.tools.allow = cherrypy.Tool('on_start_resource', http_methods_allowed) - - -def setup_server(): - - class Root: - - _cp_config = {'tools.sessions.on': True, - 'tools.sessions.storage_type' : 'ram', - 'tools.sessions.storage_path' : localDir, - 'tools.sessions.timeout': (1.0 / 60), - 'tools.sessions.clean_freq': (1.0 / 60), - } - - def clear(self): - cherrypy.session.cache.clear() - clear.exposed = True - - def data(self): - cherrypy.session['aha'] = 'foo' - return repr(cherrypy.session._data) - data.exposed = True - - def testGen(self): - counter = cherrypy.session.get('counter', 0) + 1 - cherrypy.session['counter'] = counter - yield str(counter) - testGen.exposed = True - - def testStr(self): - counter = cherrypy.session.get('counter', 0) + 1 - cherrypy.session['counter'] = counter - return str(counter) - testStr.exposed = True - - def setsessiontype(self, newtype): - self.__class__._cp_config.update({'tools.sessions.storage_type': newtype}) - if hasattr(cherrypy, "session"): - del cherrypy.session - cls = getattr(sessions, newtype.title() + 'Session') - if cls.clean_thread: - cls.clean_thread.stop() - cls.clean_thread.unsubscribe() - del cls.clean_thread - setsessiontype.exposed = True - setsessiontype._cp_config = {'tools.sessions.on': False} - - def index(self): - sess = cherrypy.session - c = sess.get('counter', 0) + 1 - time.sleep(0.01) - sess['counter'] = c - return str(c) - index.exposed = True - - def keyin(self, key): - return str(key in cherrypy.session) - keyin.exposed = True - - def delete(self): - cherrypy.session.delete() - sessions.expire() - return "done" - delete.exposed = True - - def delkey(self, key): - del cherrypy.session[key] - return "OK" - delkey.exposed = True - - def blah(self): - return self._cp_config['tools.sessions.storage_type'] - blah.exposed = True - - def iredir(self): - raise cherrypy.InternalRedirect('/blah') - iredir.exposed = True - - def restricted(self): - return cherrypy.request.method - restricted.exposed = True - restricted._cp_config = {'tools.allow.on': True, - 'tools.allow.methods': ['GET']} - - def regen(self): - cherrypy.tools.sessions.regenerate() - return "logged in" - regen.exposed = True - - def length(self): - return str(len(cherrypy.session)) - length.exposed = True - - def session_cookie(self): - # Must load() to start the clean thread. - cherrypy.session.load() - return cherrypy.session.id - session_cookie.exposed = True - session_cookie._cp_config = { - 'tools.sessions.path': '/session_cookie', - 'tools.sessions.name': 'temp', - 'tools.sessions.persistent': False} - - cherrypy.tree.mount(Root()) - - -from cherrypy.test import helper - -class SessionTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def tearDown(self): - # Clean up sessions. - for fname in os.listdir(localDir): - if fname.startswith(sessions.FileSession.SESSION_PREFIX): - os.unlink(os.path.join(localDir, fname)) - - def test_0_Session(self): - self.getPage('/setsessiontype/ram') - self.getPage('/clear') - - # Test that a normal request gets the same id in the cookies. - # Note: this wouldn't work if /data didn't load the session. - self.getPage('/data') - self.assertBody("{'aha': 'foo'}") - c = self.cookies[0] - self.getPage('/data', self.cookies) - self.assertEqual(self.cookies[0], c) - - self.getPage('/testStr') - self.assertBody('1') - cookie_parts = dict([p.strip().split('=') - for p in self.cookies[0][1].split(";")]) - # Assert there is an 'expires' param - self.assertEqual(set(cookie_parts.keys()), - set(['session_id', 'expires', 'Path'])) - self.getPage('/testGen', self.cookies) - self.assertBody('2') - self.getPage('/testStr', self.cookies) - self.assertBody('3') - self.getPage('/data', self.cookies) - self.assertBody("{'aha': 'foo', 'counter': 3}") - self.getPage('/length', self.cookies) - self.assertBody('2') - self.getPage('/delkey?key=counter', self.cookies) - self.assertStatus(200) - - self.getPage('/setsessiontype/file') - self.getPage('/testStr') - self.assertBody('1') - self.getPage('/testGen', self.cookies) - self.assertBody('2') - self.getPage('/testStr', self.cookies) - self.assertBody('3') - self.getPage('/delkey?key=counter', self.cookies) - self.assertStatus(200) - - # Wait for the session.timeout (1 second) - time.sleep(2) - self.getPage('/') - self.assertBody('1') - self.getPage('/length', self.cookies) - self.assertBody('1') - - # Test session __contains__ - self.getPage('/keyin?key=counter', self.cookies) - self.assertBody("True") - cookieset1 = self.cookies - - # Make a new session and test __len__ again - self.getPage('/') - self.getPage('/length', self.cookies) - self.assertBody('2') - - # Test session delete - self.getPage('/delete', self.cookies) - self.assertBody("done") - self.getPage('/delete', cookieset1) - self.assertBody("done") - f = lambda: [x for x in os.listdir(localDir) if x.startswith('session-')] - self.assertEqual(f(), []) - - # Wait for the cleanup thread to delete remaining session files - self.getPage('/') - f = lambda: [x for x in os.listdir(localDir) if x.startswith('session-')] - self.assertNotEqual(f(), []) - time.sleep(2) - self.assertEqual(f(), []) - - def test_1_Ram_Concurrency(self): - self.getPage('/setsessiontype/ram') - self._test_Concurrency() - - def test_2_File_Concurrency(self): - self.getPage('/setsessiontype/file') - self._test_Concurrency() - - def _test_Concurrency(self): - client_thread_count = 5 - request_count = 30 - - # Get initial cookie - self.getPage("/") - self.assertBody("1") - cookies = self.cookies - - data_dict = {} - errors = [] - - def request(index): - if self.scheme == 'https': - c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) - else: - c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) - for i in range(request_count): - c.putrequest('GET', '/') - for k, v in cookies: - c.putheader(k, v) - c.endheaders() - response = c.getresponse() - body = response.read() - if response.status != 200 or not body.isdigit(): - errors.append((response.status, body)) - else: - data_dict[index] = max(data_dict[index], int(body)) - # Uncomment the following line to prove threads overlap. -## sys.stdout.write("%d " % index) - - # Start requests from each of - # concurrent clients - ts = [] - for c in range(client_thread_count): - data_dict[c] = 0 - t = threading.Thread(target=request, args=(c,)) - ts.append(t) - t.start() - - for t in ts: - t.join() - - hitcount = max(data_dict.values()) - expected = 1 + (client_thread_count * request_count) - - for e in errors: - print(e) - self.assertEqual(hitcount, expected) - - def test_3_Redirect(self): - # Start a new session - self.getPage('/testStr') - self.getPage('/iredir', self.cookies) - self.assertBody("file") - - def test_4_File_deletion(self): - # Start a new session - self.getPage('/testStr') - # Delete the session file manually and retry. - id = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] - path = os.path.join(localDir, "session-" + id) - os.unlink(path) - self.getPage('/testStr', self.cookies) - - def test_5_Error_paths(self): - self.getPage('/unknown/page') - self.assertErrorPage(404, "The path '/unknown/page' was not found.") - - # Note: this path is *not* the same as above. The above - # takes a normal route through the session code; this one - # skips the session code's before_handler and only calls - # before_finalize (save) and on_end (close). So the session - # code has to survive calling save/close without init. - self.getPage('/restricted', self.cookies, method='POST') - self.assertErrorPage(405, response_codes[405]) - - def test_6_regenerate(self): - self.getPage('/testStr') - # grab the cookie ID - id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] - self.getPage('/regen') - self.assertBody('logged in') - id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] - self.assertNotEqual(id1, id2) - - self.getPage('/testStr') - # grab the cookie ID - id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] - self.getPage('/testStr', - headers=[('Cookie', - 'session_id=maliciousid; ' - 'expires=Sat, 27 Oct 2017 04:18:28 GMT; Path=/;')]) - id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1] - self.assertNotEqual(id1, id2) - self.assertNotEqual(id2, 'maliciousid') - - def test_7_session_cookies(self): - self.getPage('/setsessiontype/ram') - self.getPage('/clear') - self.getPage('/session_cookie') - # grab the cookie ID - cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) - # Assert there is no 'expires' param - self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) - id1 = cookie_parts['temp'] - self.assertEqual(copykeys(sessions.RamSession.cache), [id1]) - - # Send another request in the same "browser session". - self.getPage('/session_cookie', self.cookies) - cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) - # Assert there is no 'expires' param - self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) - self.assertBody(id1) - self.assertEqual(copykeys(sessions.RamSession.cache), [id1]) - - # Simulate a browser close by just not sending the cookies - self.getPage('/session_cookie') - # grab the cookie ID - cookie_parts = dict([p.strip().split('=') for p in self.cookies[0][1].split(";")]) - # Assert there is no 'expires' param - self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path'])) - # Assert a new id has been generated... - id2 = cookie_parts['temp'] - self.assertNotEqual(id1, id2) - self.assertEqual(set(sessions.RamSession.cache.keys()), set([id1, id2])) - - # Wait for the session.timeout on both sessions - time.sleep(2.5) - cache = copykeys(sessions.RamSession.cache) - if cache: - if cache == [id2]: - self.fail("The second session did not time out.") - else: - self.fail("Unknown session id in cache: %r", cache) - - -import socket -try: - import memcache - - host, port = '127.0.0.1', 11211 - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - raise - break -except (ImportError, socket.error): - class MemcachedSessionTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test(self): - return self.skip("memcached not reachable ") -else: - class MemcachedSessionTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def test_0_Session(self): - self.getPage('/setsessiontype/memcached') - - self.getPage('/testStr') - self.assertBody('1') - self.getPage('/testGen', self.cookies) - self.assertBody('2') - self.getPage('/testStr', self.cookies) - self.assertBody('3') - self.getPage('/length', self.cookies) - self.assertErrorPage(500) - self.assertInBody("NotImplementedError") - self.getPage('/delkey?key=counter', self.cookies) - self.assertStatus(200) - - # Wait for the session.timeout (1 second) - time.sleep(1.25) - self.getPage('/') - self.assertBody('1') - - # Test session __contains__ - self.getPage('/keyin?key=counter', self.cookies) - self.assertBody("True") - - # Test session delete - self.getPage('/delete', self.cookies) - self.assertBody("done") - - def test_1_Concurrency(self): - client_thread_count = 5 - request_count = 30 - - # Get initial cookie - self.getPage("/") - self.assertBody("1") - cookies = self.cookies - - data_dict = {} - - def request(index): - for i in range(request_count): - self.getPage("/", cookies) - # Uncomment the following line to prove threads overlap. -## sys.stdout.write("%d " % index) - if not self.body.isdigit(): - self.fail(self.body) - data_dict[index] = v = int(self.body) - - # Start concurrent requests from - # each of clients - ts = [] - for c in range(client_thread_count): - data_dict[c] = 0 - t = threading.Thread(target=request, args=(c,)) - ts.append(t) - t.start() - - for t in ts: - t.join() - - hitcount = max(data_dict.values()) - expected = 1 + (client_thread_count * request_count) - self.assertEqual(hitcount, expected) - - def test_3_Redirect(self): - # Start a new session - self.getPage('/testStr') - self.getPage('/iredir', self.cookies) - self.assertBody("memcached") - - def test_5_Error_paths(self): - self.getPage('/unknown/page') - self.assertErrorPage(404, "The path '/unknown/page' was not found.") - - # Note: this path is *not* the same as above. The above - # takes a normal route through the session code; this one - # skips the session code's before_handler and only calls - # before_finalize (save) and on_end (close). So the session - # code has to survive calling save/close without init. - self.getPage('/restricted', self.cookies, method='POST') - self.assertErrorPage(405, response_codes[405]) - diff --git a/cherrypy/test/test_sessionauthenticate.py b/cherrypy/test/test_sessionauthenticate.py deleted file mode 100644 index ab1fe51e..00000000 --- a/cherrypy/test/test_sessionauthenticate.py +++ /dev/null @@ -1,62 +0,0 @@ -import cherrypy -from cherrypy.test import helper - - -class SessionAuthenticateTest(helper.CPWebCase): - - def setup_server(): - - def check(username, password): - # Dummy check_username_and_password function - if username != 'test' or password != 'password': - return 'Wrong login/password' - - def augment_params(): - # A simple tool to add some things to request.params - # This is to check to make sure that session_auth can handle request - # params (ticket #780) - cherrypy.request.params["test"] = "test" - - cherrypy.tools.augment_params = cherrypy.Tool('before_handler', - augment_params, None, priority=30) - - class Test: - - _cp_config = {'tools.sessions.on': True, - 'tools.session_auth.on': True, - 'tools.session_auth.check_username_and_password': check, - 'tools.augment_params.on': True, - } - - def index(self, **kwargs): - return "Hi %s, you are logged in" % cherrypy.request.login - index.exposed = True - - cherrypy.tree.mount(Test()) - setup_server = staticmethod(setup_server) - - - def testSessionAuthenticate(self): - # request a page and check for login form - self.getPage('/') - self.assertInBody('
') - - # setup credentials - login_body = 'username=test&password=password&from_page=/' - - # attempt a login - self.getPage('/do_login', method='POST', body=login_body) - self.assertStatus((302, 303)) - - # get the page now that we are logged in - self.getPage('/', self.cookies) - self.assertBody('Hi test, you are logged in') - - # do a logout - self.getPage('/do_logout', self.cookies, method='POST') - self.assertStatus((302, 303)) - - # verify we are logged out - self.getPage('/', self.cookies) - self.assertInBody('') - diff --git a/cherrypy/test/test_states.py b/cherrypy/test/test_states.py deleted file mode 100644 index 0f973374..00000000 --- a/cherrypy/test/test_states.py +++ /dev/null @@ -1,436 +0,0 @@ -from cherrypy._cpcompat import BadStatusLine, ntob -import os -import sys -import threading -import time - -import cherrypy -engine = cherrypy.engine -thisdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - - -class Dependency: - - def __init__(self, bus): - self.bus = bus - self.running = False - self.startcount = 0 - self.gracecount = 0 - self.threads = {} - - def subscribe(self): - self.bus.subscribe('start', self.start) - self.bus.subscribe('stop', self.stop) - self.bus.subscribe('graceful', self.graceful) - self.bus.subscribe('start_thread', self.startthread) - self.bus.subscribe('stop_thread', self.stopthread) - - def start(self): - self.running = True - self.startcount += 1 - - def stop(self): - self.running = False - - def graceful(self): - self.gracecount += 1 - - def startthread(self, thread_id): - self.threads[thread_id] = None - - def stopthread(self, thread_id): - del self.threads[thread_id] - -db_connection = Dependency(engine) - -def setup_server(): - class Root: - def index(self): - return "Hello World" - index.exposed = True - - def ctrlc(self): - raise KeyboardInterrupt() - ctrlc.exposed = True - - def graceful(self): - engine.graceful() - return "app was (gracefully) restarted succesfully" - graceful.exposed = True - - def block_explicit(self): - while True: - if cherrypy.response.timed_out: - cherrypy.response.timed_out = False - return "broken!" - time.sleep(0.01) - block_explicit.exposed = True - - def block_implicit(self): - time.sleep(0.5) - return "response.timeout = %s" % cherrypy.response.timeout - block_implicit.exposed = True - - cherrypy.tree.mount(Root()) - cherrypy.config.update({ - 'environment': 'test_suite', - 'engine.deadlock_poll_freq': 0.1, - }) - - db_connection.subscribe() - - - -# ------------ Enough helpers. Time for real live test cases. ------------ # - - -from cherrypy.test import helper - -class ServerStateTests(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def setUp(self): - cherrypy.server.socket_timeout = 0.1 - - def test_0_NormalStateFlow(self): - engine.stop() - # Our db_connection should not be running - self.assertEqual(db_connection.running, False) - self.assertEqual(db_connection.startcount, 1) - self.assertEqual(len(db_connection.threads), 0) - - # Test server start - engine.start() - self.assertEqual(engine.state, engine.states.STARTED) - - host = cherrypy.server.socket_host - port = cherrypy.server.socket_port - self.assertRaises(IOError, cherrypy._cpserver.check_port, host, port) - - # The db_connection should be running now - self.assertEqual(db_connection.running, True) - self.assertEqual(db_connection.startcount, 2) - self.assertEqual(len(db_connection.threads), 0) - - self.getPage("/") - self.assertBody("Hello World") - self.assertEqual(len(db_connection.threads), 1) - - # Test engine stop. This will also stop the HTTP server. - engine.stop() - self.assertEqual(engine.state, engine.states.STOPPED) - - # Verify that our custom stop function was called - self.assertEqual(db_connection.running, False) - self.assertEqual(len(db_connection.threads), 0) - - # Block the main thread now and verify that exit() works. - def exittest(): - self.getPage("/") - self.assertBody("Hello World") - engine.exit() - cherrypy.server.start() - engine.start_with_callback(exittest) - engine.block() - self.assertEqual(engine.state, engine.states.EXITING) - - def test_1_Restart(self): - cherrypy.server.start() - engine.start() - - # The db_connection should be running now - self.assertEqual(db_connection.running, True) - grace = db_connection.gracecount - - self.getPage("/") - self.assertBody("Hello World") - self.assertEqual(len(db_connection.threads), 1) - - # Test server restart from this thread - engine.graceful() - self.assertEqual(engine.state, engine.states.STARTED) - self.getPage("/") - self.assertBody("Hello World") - self.assertEqual(db_connection.running, True) - self.assertEqual(db_connection.gracecount, grace + 1) - self.assertEqual(len(db_connection.threads), 1) - - # Test server restart from inside a page handler - self.getPage("/graceful") - self.assertEqual(engine.state, engine.states.STARTED) - self.assertBody("app was (gracefully) restarted succesfully") - self.assertEqual(db_connection.running, True) - self.assertEqual(db_connection.gracecount, grace + 2) - # Since we are requesting synchronously, is only one thread used? - # Note that the "/graceful" request has been flushed. - self.assertEqual(len(db_connection.threads), 0) - - engine.stop() - self.assertEqual(engine.state, engine.states.STOPPED) - self.assertEqual(db_connection.running, False) - self.assertEqual(len(db_connection.threads), 0) - - def test_2_KeyboardInterrupt(self): - # Raise a keyboard interrupt in the HTTP server's main thread. - # We must start the server in this, the main thread - engine.start() - cherrypy.server.start() - - self.persistent = True - try: - # Make the first request and assert there's no "Connection: close". - self.getPage("/") - self.assertStatus('200 OK') - self.assertBody("Hello World") - self.assertNoHeader("Connection") - - cherrypy.server.httpserver.interrupt = KeyboardInterrupt - engine.block() - - self.assertEqual(db_connection.running, False) - self.assertEqual(len(db_connection.threads), 0) - self.assertEqual(engine.state, engine.states.EXITING) - finally: - self.persistent = False - - # Raise a keyboard interrupt in a page handler; on multithreaded - # servers, this should occur in one of the worker threads. - # This should raise a BadStatusLine error, since the worker - # thread will just die without writing a response. - engine.start() - cherrypy.server.start() - - try: - self.getPage("/ctrlc") - except BadStatusLine: - pass - else: - print(self.body) - self.fail("AssertionError: BadStatusLine not raised") - - engine.block() - self.assertEqual(db_connection.running, False) - self.assertEqual(len(db_connection.threads), 0) - - def test_3_Deadlocks(self): - cherrypy.config.update({'response.timeout': 0.2}) - - engine.start() - cherrypy.server.start() - try: - self.assertNotEqual(engine.timeout_monitor.thread, None) - - # Request a "normal" page. - self.assertEqual(engine.timeout_monitor.servings, []) - self.getPage("/") - self.assertBody("Hello World") - # request.close is called async. - while engine.timeout_monitor.servings: - sys.stdout.write(".") - time.sleep(0.01) - - # Request a page that explicitly checks itself for deadlock. - # The deadlock_timeout should be 2 secs. - self.getPage("/block_explicit") - self.assertBody("broken!") - - # Request a page that implicitly breaks deadlock. - # If we deadlock, we want to touch as little code as possible, - # so we won't even call handle_error, just bail ASAP. - self.getPage("/block_implicit") - self.assertStatus(500) - self.assertInBody("raise cherrypy.TimeoutError()") - finally: - engine.exit() - - def test_4_Autoreload(self): - # Start the demo script in a new process - p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) - p.write_conf( - extra='test_case_name: "test_4_Autoreload"') - p.start(imports='cherrypy.test._test_states_demo') - try: - self.getPage("/start") - start = float(self.body) - - # Give the autoreloader time to cache the file time. - time.sleep(2) - - # Touch the file - os.utime(os.path.join(thisdir, "_test_states_demo.py"), None) - - # Give the autoreloader time to re-exec the process - time.sleep(2) - host = cherrypy.server.socket_host - port = cherrypy.server.socket_port - cherrypy._cpserver.wait_for_occupied_port(host, port) - - self.getPage("/start") - self.assert_(float(self.body) > start) - finally: - # Shut down the spawned process - self.getPage("/exit") - p.join() - - def test_5_Start_Error(self): - # If a process errors during start, it should stop the engine - # and exit with a non-zero exit code. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), - wait=True) - p.write_conf( - extra="""starterror: True -test_case_name: "test_5_Start_Error" -""" - ) - p.start(imports='cherrypy.test._test_states_demo') - if p.exit_code == 0: - self.fail("Process failed to return nonzero exit code.") - - -class PluginTests(helper.CPWebCase): - def test_daemonize(self): - if os.name not in ['posix']: - return self.skip("skipped (not on posix) ") - self.HOST = '127.0.0.1' - self.PORT = 8081 - # Spawn the process and wait, when this returns, the original process - # is finished. If it daemonized properly, we should still be able - # to access pages. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), - wait=True, daemonize=True, - socket_host='127.0.0.1', - socket_port=8081) - p.write_conf( - extra='test_case_name: "test_daemonize"') - p.start(imports='cherrypy.test._test_states_demo') - try: - # Just get the pid of the daemonization process. - self.getPage("/pid") - self.assertStatus(200) - page_pid = int(self.body) - self.assertEqual(page_pid, p.get_pid()) - finally: - # Shut down the spawned process - self.getPage("/exit") - p.join() - - # Wait until here to test the exit code because we want to ensure - # that we wait for the daemon to finish running before we fail. - if p.exit_code != 0: - self.fail("Daemonized parent process failed to exit cleanly.") - - -class SignalHandlingTests(helper.CPWebCase): - def test_SIGHUP_tty(self): - # When not daemonized, SIGHUP should shut down the server. - try: - from signal import SIGHUP - except ImportError: - return self.skip("skipped (no SIGHUP) ") - - # Spawn the process. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) - p.write_conf( - extra='test_case_name: "test_SIGHUP_tty"') - p.start(imports='cherrypy.test._test_states_demo') - # Send a SIGHUP - os.kill(p.get_pid(), SIGHUP) - # This might hang if things aren't working right, but meh. - p.join() - - def test_SIGHUP_daemonized(self): - # When daemonized, SIGHUP should restart the server. - try: - from signal import SIGHUP - except ImportError: - return self.skip("skipped (no SIGHUP) ") - - if os.name not in ['posix']: - return self.skip("skipped (not on posix) ") - - # Spawn the process and wait, when this returns, the original process - # is finished. If it daemonized properly, we should still be able - # to access pages. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), - wait=True, daemonize=True) - p.write_conf( - extra='test_case_name: "test_SIGHUP_daemonized"') - p.start(imports='cherrypy.test._test_states_demo') - - pid = p.get_pid() - try: - # Send a SIGHUP - os.kill(pid, SIGHUP) - # Give the server some time to restart - time.sleep(2) - self.getPage("/pid") - self.assertStatus(200) - new_pid = int(self.body) - self.assertNotEqual(new_pid, pid) - finally: - # Shut down the spawned process - self.getPage("/exit") - p.join() - - def test_SIGTERM(self): - # SIGTERM should shut down the server whether daemonized or not. - try: - from signal import SIGTERM - except ImportError: - return self.skip("skipped (no SIGTERM) ") - - try: - from os import kill - except ImportError: - return self.skip("skipped (no os.kill) ") - - # Spawn a normal, undaemonized process. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) - p.write_conf( - extra='test_case_name: "test_SIGTERM"') - p.start(imports='cherrypy.test._test_states_demo') - # Send a SIGTERM - os.kill(p.get_pid(), SIGTERM) - # This might hang if things aren't working right, but meh. - p.join() - - if os.name in ['posix']: - # Spawn a daemonized process and test again. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https'), - wait=True, daemonize=True) - p.write_conf( - extra='test_case_name: "test_SIGTERM_2"') - p.start(imports='cherrypy.test._test_states_demo') - # Send a SIGTERM - os.kill(p.get_pid(), SIGTERM) - # This might hang if things aren't working right, but meh. - p.join() - - def test_signal_handler_unsubscribe(self): - try: - from signal import SIGTERM - except ImportError: - return self.skip("skipped (no SIGTERM) ") - - try: - from os import kill - except ImportError: - return self.skip("skipped (no os.kill) ") - - # Spawn a normal, undaemonized process. - p = helper.CPProcess(ssl=(self.scheme.lower()=='https')) - p.write_conf( - extra="""unsubsig: True -test_case_name: "test_signal_handler_unsubscribe" -""") - p.start(imports='cherrypy.test._test_states_demo') - # Send a SIGTERM - os.kill(p.get_pid(), SIGTERM) - # This might hang if things aren't working right, but meh. - p.join() - - # Assert the old handler ran. - target_line = open(p.error_log, 'rb').readlines()[-10] - if not ntob("I am an old SIGTERM handler.") in target_line: - self.fail("Old SIGTERM handler did not run.\n%r" % target_line) - diff --git a/cherrypy/test/test_static.py b/cherrypy/test/test_static.py deleted file mode 100644 index 871420bd..00000000 --- a/cherrypy/test/test_static.py +++ /dev/null @@ -1,300 +0,0 @@ -from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob -from cherrypy._cpcompat import BytesIO - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -has_space_filepath = os.path.join(curdir, 'static', 'has space.html') -bigfile_filepath = os.path.join(curdir, "static", "bigfile.log") -BIGFILE_SIZE = 1024 * 1024 -import threading - -import cherrypy -from cherrypy.lib import static -from cherrypy.test import helper - - -class StaticTest(helper.CPWebCase): - - def setup_server(): - if not os.path.exists(has_space_filepath): - open(has_space_filepath, 'wb').write(ntob('Hello, world\r\n')) - if not os.path.exists(bigfile_filepath): - open(bigfile_filepath, 'wb').write(ntob("x" * BIGFILE_SIZE)) - - class Root: - - def bigfile(self): - from cherrypy.lib import static - self.f = static.serve_file(bigfile_filepath) - return self.f - bigfile.exposed = True - bigfile._cp_config = {'response.stream': True} - - def tell(self): - if self.f.input.closed: - return '' - return repr(self.f.input.tell()).rstrip('L') - tell.exposed = True - - def fileobj(self): - f = open(os.path.join(curdir, 'style.css'), 'rb') - return static.serve_fileobj(f, content_type='text/css') - fileobj.exposed = True - - def bytesio(self): - f = BytesIO(ntob('Fee\nfie\nfo\nfum')) - return static.serve_fileobj(f, content_type='text/plain') - bytesio.exposed = True - - class Static: - - def index(self): - return 'You want the Baron? You can have the Baron!' - index.exposed = True - - def dynamic(self): - return "This is a DYNAMIC page" - dynamic.exposed = True - - - root = Root() - root.static = Static() - - rootconf = { - '/static': { - 'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.root': curdir, - }, - '/style.css': { - 'tools.staticfile.on': True, - 'tools.staticfile.filename': os.path.join(curdir, 'style.css'), - }, - '/docroot': { - 'tools.staticdir.on': True, - 'tools.staticdir.root': curdir, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.index': 'index.html', - }, - '/error': { - 'tools.staticdir.on': True, - 'request.show_tracebacks': True, - }, - } - rootApp = cherrypy.Application(root) - rootApp.merge(rootconf) - - test_app_conf = { - '/test': { - 'tools.staticdir.index': 'index.html', - 'tools.staticdir.on': True, - 'tools.staticdir.root': curdir, - 'tools.staticdir.dir': 'static', - }, - } - testApp = cherrypy.Application(Static()) - testApp.merge(test_app_conf) - - vhost = cherrypy._cpwsgi.VirtualHost(rootApp, {'virt.net': testApp}) - cherrypy.tree.graft(vhost) - setup_server = staticmethod(setup_server) - - - def teardown_server(): - for f in (has_space_filepath, bigfile_filepath): - if os.path.exists(f): - try: - os.unlink(f) - except: - pass - teardown_server = staticmethod(teardown_server) - - - def testStatic(self): - self.getPage("/static/index.html") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html') - self.assertBody('Hello, world\r\n') - - # Using a staticdir.root value in a subdir... - self.getPage("/docroot/index.html") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html') - self.assertBody('Hello, world\r\n') - - # Check a filename with spaces in it - self.getPage("/static/has%20space.html") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html') - self.assertBody('Hello, world\r\n') - - self.getPage("/style.css") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/css') - # Note: The body should be exactly 'Dummy stylesheet\n', but - # unfortunately some tools such as WinZip sometimes turn \n - # into \r\n on Windows when extracting the CherryPy tarball so - # we just check the content - self.assertMatchesBody('^Dummy stylesheet') - - def test_fallthrough(self): - # Test that NotFound will then try dynamic handlers (see [878]). - self.getPage("/static/dynamic") - self.assertBody("This is a DYNAMIC page") - - # Check a directory via fall-through to dynamic handler. - self.getPage("/static/") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html;charset=utf-8') - self.assertBody('You want the Baron? You can have the Baron!') - - def test_index(self): - # Check a directory via "staticdir.index". - self.getPage("/docroot/") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/html') - self.assertBody('Hello, world\r\n') - # The same page should be returned even if redirected. - self.getPage("/docroot") - self.assertStatus(301) - self.assertHeader('Location', '%s/docroot/' % self.base()) - self.assertMatchesBody("This resource .* " - "%s/docroot/." % (self.base(), self.base())) - - def test_config_errors(self): - # Check that we get an error if no .file or .dir - self.getPage("/error/thing.html") - self.assertErrorPage(500) - self.assertMatchesBody(ntob("TypeError: staticdir\(\) takes at least 2 " - "(positional )?arguments \(0 given\)")) - - def test_security(self): - # Test up-level security - self.getPage("/static/../../test/style.css") - self.assertStatus((400, 403)) - - def test_modif(self): - # Test modified-since on a reasonably-large file - self.getPage("/static/dirback.jpg") - self.assertStatus("200 OK") - lastmod = "" - for k, v in self.headers: - if k == 'Last-Modified': - lastmod = v - ims = ("If-Modified-Since", lastmod) - self.getPage("/static/dirback.jpg", headers=[ims]) - self.assertStatus(304) - self.assertNoHeader("Content-Type") - self.assertNoHeader("Content-Length") - self.assertNoHeader("Content-Disposition") - self.assertBody("") - - def test_755_vhost(self): - self.getPage("/test/", [('Host', 'virt.net')]) - self.assertStatus(200) - self.getPage("/test", [('Host', 'virt.net')]) - self.assertStatus(301) - self.assertHeader('Location', self.scheme + '://virt.net/test/') - - def test_serve_fileobj(self): - self.getPage("/fileobj") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/css;charset=utf-8') - self.assertMatchesBody('^Dummy stylesheet') - - def test_serve_bytesio(self): - self.getPage("/bytesio") - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/plain;charset=utf-8') - self.assertHeader('Content-Length', 14) - self.assertMatchesBody('Fee\nfie\nfo\nfum') - - def test_file_stream(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Make an initial request - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/bigfile", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - - body = ntob('') - remaining = BIGFILE_SIZE - while remaining > 0: - data = response.fp.read(65536) - if not data: - break - body += data - remaining -= len(data) - - if self.scheme == "https": - newconn = HTTPSConnection - else: - newconn = HTTPConnection - s, h, b = helper.webtest.openURL( - ntob("/tell"), headers=[], host=self.HOST, port=self.PORT, - http_conn=newconn) - if not b: - # The file was closed on the server. - tell_position = BIGFILE_SIZE - else: - tell_position = int(b) - - expected = len(body) - if tell_position >= BIGFILE_SIZE: - # We can't exactly control how much content the server asks for. - # Fudge it by only checking the first half of the reads. - if expected < (BIGFILE_SIZE / 2): - self.fail( - "The file should have advanced to position %r, but has " - "already advanced to the end of the file. It may not be " - "streamed as intended, or at the wrong chunk size (64k)" % - expected) - elif tell_position < expected: - self.fail( - "The file should have advanced to position %r, but has " - "only advanced to position %r. It may not be streamed " - "as intended, or at the wrong chunk size (65536)" % - (expected, tell_position)) - - if body != ntob("x" * BIGFILE_SIZE): - self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % - (BIGFILE_SIZE, body[:50], len(body))) - conn.close() - - def test_file_stream_deadlock(self): - if cherrypy.server.protocol_version != "HTTP/1.1": - return self.skip() - - self.PROTOCOL = "HTTP/1.1" - - # Make an initial request but abort early. - self.persistent = True - conn = self.HTTP_CONN - conn.putrequest("GET", "/bigfile", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - response = conn.response_class(conn.sock, method="GET") - response.begin() - self.assertEqual(response.status, 200) - body = response.fp.read(65536) - if body != ntob("x" * len(body)): - self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % - (65536, body[:50], len(body))) - response.close() - conn.close() - - # Make a second request, which should fetch the whole file. - self.persistent = False - self.getPage("/bigfile") - if self.body != ntob("x" * BIGFILE_SIZE): - self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." % - (BIGFILE_SIZE, self.body[:50], len(body))) - diff --git a/cherrypy/test/test_tools.py b/cherrypy/test/test_tools.py deleted file mode 100644 index bc8579f0..00000000 --- a/cherrypy/test/test_tools.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Test the various means of instantiating and invoking tools.""" - -import gzip -import sys -from cherrypy._cpcompat import BytesIO, copyitems, itervalues, IncompleteRead, ntob, ntou, xrange -import time -timeout = 0.2 -import types - -import cherrypy -from cherrypy import tools - - -europoundUnicode = ntou('\x80\xa3') - - -# Client-side code # - -from cherrypy.test import helper - - -class ToolTests(helper.CPWebCase): - def setup_server(): - - # Put check_access in a custom toolbox with its own namespace - myauthtools = cherrypy._cptools.Toolbox("myauth") - - def check_access(default=False): - if not getattr(cherrypy.request, "userid", default): - raise cherrypy.HTTPError(401) - myauthtools.check_access = cherrypy.Tool('before_request_body', check_access) - - def numerify(): - def number_it(body): - for chunk in body: - for k, v in cherrypy.request.numerify_map: - chunk = chunk.replace(k, v) - yield chunk - cherrypy.response.body = number_it(cherrypy.response.body) - - class NumTool(cherrypy.Tool): - def _setup(self): - def makemap(): - m = self._merged_args().get("map", {}) - cherrypy.request.numerify_map = copyitems(m) - cherrypy.request.hooks.attach('on_start_resource', makemap) - - def critical(): - cherrypy.request.error_response = cherrypy.HTTPError(502).set_response - critical.failsafe = True - - cherrypy.request.hooks.attach('on_start_resource', critical) - cherrypy.request.hooks.attach(self._point, self.callable) - - tools.numerify = NumTool('before_finalize', numerify) - - # It's not mandatory to inherit from cherrypy.Tool. - class NadsatTool: - - def __init__(self): - self.ended = {} - self._name = "nadsat" - - def nadsat(self): - def nadsat_it_up(body): - for chunk in body: - chunk = chunk.replace(ntob("good"), ntob("horrorshow")) - chunk = chunk.replace(ntob("piece"), ntob("lomtick")) - yield chunk - cherrypy.response.body = nadsat_it_up(cherrypy.response.body) - nadsat.priority = 0 - - def cleanup(self): - # This runs after the request has been completely written out. - cherrypy.response.body = [ntob("razdrez")] - id = cherrypy.request.params.get("id") - if id: - self.ended[id] = True - cleanup.failsafe = True - - def _setup(self): - cherrypy.request.hooks.attach('before_finalize', self.nadsat) - cherrypy.request.hooks.attach('on_end_request', self.cleanup) - tools.nadsat = NadsatTool() - - def pipe_body(): - cherrypy.request.process_request_body = False - clen = int(cherrypy.request.headers['Content-Length']) - cherrypy.request.body = cherrypy.request.rfile.read(clen) - - # Assert that we can use a callable object instead of a function. - class Rotator(object): - def __call__(self, scale): - r = cherrypy.response - r.collapse_body() - r.body = [chr((ord(x) + scale) % 256) for x in r.body[0]] - cherrypy.tools.rotator = cherrypy.Tool('before_finalize', Rotator()) - - def stream_handler(next_handler, *args, **kwargs): - cherrypy.response.output = o = BytesIO() - try: - response = next_handler(*args, **kwargs) - # Ignore the response and return our accumulated output instead. - return o.getvalue() - finally: - o.close() - cherrypy.tools.streamer = cherrypy._cptools.HandlerWrapperTool(stream_handler) - - class Root: - def index(self): - return "Howdy earth!" - index.exposed = True - - def tarfile(self): - cherrypy.response.output.write(ntob('I am ')) - cherrypy.response.output.write(ntob('a tarfile')) - tarfile.exposed = True - tarfile._cp_config = {'tools.streamer.on': True} - - def euro(self): - hooks = list(cherrypy.request.hooks['before_finalize']) - hooks.sort() - cbnames = [x.callback.__name__ for x in hooks] - assert cbnames == ['gzip'], cbnames - priorities = [x.priority for x in hooks] - assert priorities == [80], priorities - yield ntou("Hello,") - yield ntou("world") - yield europoundUnicode - euro.exposed = True - - # Bare hooks - def pipe(self): - return cherrypy.request.body - pipe.exposed = True - pipe._cp_config = {'hooks.before_request_body': pipe_body} - - # Multiple decorators; include kwargs just for fun. - # Note that rotator must run before gzip. - def decorated_euro(self, *vpath): - yield ntou("Hello,") - yield ntou("world") - yield europoundUnicode - decorated_euro.exposed = True - decorated_euro = tools.gzip(compress_level=6)(decorated_euro) - decorated_euro = tools.rotator(scale=3)(decorated_euro) - - root = Root() - - - class TestType(type): - """Metaclass which automatically exposes all functions in each subclass, - and adds an instance of the subclass as an attribute of root. - """ - def __init__(cls, name, bases, dct): - type.__init__(cls, name, bases, dct) - for value in itervalues(dct): - if isinstance(value, types.FunctionType): - value.exposed = True - setattr(root, name.lower(), cls()) - class Test(object): - __metaclass__ = TestType - - - # METHOD ONE: - # Declare Tools in _cp_config - class Demo(Test): - - _cp_config = {"tools.nadsat.on": True} - - def index(self, id=None): - return "A good piece of cherry pie" - - def ended(self, id): - return repr(tools.nadsat.ended[id]) - - def err(self, id=None): - raise ValueError() - - def errinstream(self, id=None): - yield "nonconfidential" - raise ValueError() - yield "confidential" - - # METHOD TWO: decorator using Tool() - # We support Python 2.3, but the @-deco syntax would look like this: - # @tools.check_access() - def restricted(self): - return "Welcome!" - restricted = myauthtools.check_access()(restricted) - userid = restricted - - def err_in_onstart(self): - return "success!" - - def stream(self, id=None): - for x in xrange(100000000): - yield str(x) - stream._cp_config = {'response.stream': True} - - - conf = { - # METHOD THREE: - # Declare Tools in detached config - '/demo': { - 'tools.numerify.on': True, - 'tools.numerify.map': {ntob("pie"): ntob("3.14159")}, - }, - '/demo/restricted': { - 'request.show_tracebacks': False, - }, - '/demo/userid': { - 'request.show_tracebacks': False, - 'myauth.check_access.default': True, - }, - '/demo/errinstream': { - 'response.stream': True, - }, - '/demo/err_in_onstart': { - # Because this isn't a dict, on_start_resource will error. - 'tools.numerify.map': "pie->3.14159" - }, - # Combined tools - '/euro': { - 'tools.gzip.on': True, - 'tools.encode.on': True, - }, - # Priority specified in config - '/decorated_euro/subpath': { - 'tools.gzip.priority': 10, - }, - # Handler wrappers - '/tarfile': {'tools.streamer.on': True} - } - app = cherrypy.tree.mount(root, config=conf) - app.request_class.namespaces['myauth'] = myauthtools - - if sys.version_info >= (2, 5): - from cherrypy.test import _test_decorators - root.tooldecs = _test_decorators.ToolExamples() - setup_server = staticmethod(setup_server) - - def testHookErrors(self): - self.getPage("/demo/?id=1") - # If body is "razdrez", then on_end_request is being called too early. - self.assertBody("A horrorshow lomtick of cherry 3.14159") - # If this fails, then on_end_request isn't being called at all. - time.sleep(0.1) - self.getPage("/demo/ended/1") - self.assertBody("True") - - valerr = '\n raise ValueError()\nValueError' - self.getPage("/demo/err?id=3") - # If body is "razdrez", then on_end_request is being called too early. - self.assertErrorPage(502, pattern=valerr) - # If this fails, then on_end_request isn't being called at all. - time.sleep(0.1) - self.getPage("/demo/ended/3") - self.assertBody("True") - - # If body is "razdrez", then on_end_request is being called too early. - if (cherrypy.server.protocol_version == "HTTP/1.0" or - getattr(cherrypy.server, "using_apache", False)): - self.getPage("/demo/errinstream?id=5") - # Because this error is raised after the response body has - # started, the status should not change to an error status. - self.assertStatus("200 OK") - self.assertBody("nonconfidential") - else: - # Because this error is raised after the response body has - # started, and because it's chunked output, an error is raised by - # the HTTP client when it encounters incomplete output. - self.assertRaises((ValueError, IncompleteRead), self.getPage, - "/demo/errinstream?id=5") - # If this fails, then on_end_request isn't being called at all. - time.sleep(0.1) - self.getPage("/demo/ended/5") - self.assertBody("True") - - # Test the "__call__" technique (compile-time decorator). - self.getPage("/demo/restricted") - self.assertErrorPage(401) - - # Test compile-time decorator with kwargs from config. - self.getPage("/demo/userid") - self.assertBody("Welcome!") - - def testEndRequestOnDrop(self): - old_timeout = None - try: - httpserver = cherrypy.server.httpserver - old_timeout = httpserver.timeout - except (AttributeError, IndexError): - return self.skip() - - try: - httpserver.timeout = timeout - - # Test that on_end_request is called even if the client drops. - self.persistent = True - try: - conn = self.HTTP_CONN - conn.putrequest("GET", "/demo/stream?id=9", skip_host=True) - conn.putheader("Host", self.HOST) - conn.endheaders() - # Skip the rest of the request and close the conn. This will - # cause the server's active socket to error, which *should* - # result in the request being aborted, and request.close being - # called all the way up the stack (including WSGI middleware), - # eventually calling our on_end_request hook. - finally: - self.persistent = False - time.sleep(timeout * 2) - # Test that the on_end_request hook was called. - self.getPage("/demo/ended/9") - self.assertBody("True") - finally: - if old_timeout is not None: - httpserver.timeout = old_timeout - - def testGuaranteedHooks(self): - # The 'critical' on_start_resource hook is 'failsafe' (guaranteed - # to run even if there are failures in other on_start methods). - # This is NOT true of the other hooks. - # Here, we have set up a failure in NumerifyTool.numerify_map, - # but our 'critical' hook should run and set the error to 502. - self.getPage("/demo/err_in_onstart") - self.assertErrorPage(502) - self.assertInBody("AttributeError: 'str' object has no attribute 'items'") - - def testCombinedTools(self): - expectedResult = (ntou("Hello,world") + europoundUnicode).encode('utf-8') - zbuf = BytesIO() - zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9) - zfile.write(expectedResult) - zfile.close() - - self.getPage("/euro", headers=[("Accept-Encoding", "gzip"), - ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")]) - self.assertInBody(zbuf.getvalue()[:3]) - - zbuf = BytesIO() - zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=6) - zfile.write(expectedResult) - zfile.close() - - self.getPage("/decorated_euro", headers=[("Accept-Encoding", "gzip")]) - self.assertInBody(zbuf.getvalue()[:3]) - - # This returns a different value because gzip's priority was - # lowered in conf, allowing the rotator to run after gzip. - # Of course, we don't want breakage in production apps, - # but it proves the priority was changed. - self.getPage("/decorated_euro/subpath", - headers=[("Accept-Encoding", "gzip")]) - self.assertInBody(''.join([chr((ord(x) + 3) % 256) for x in zbuf.getvalue()])) - - def testBareHooks(self): - content = "bit of a pain in me gulliver" - self.getPage("/pipe", - headers=[("Content-Length", str(len(content))), - ("Content-Type", "text/plain")], - method="POST", body=content) - self.assertBody(content) - - def testHandlerWrapperTool(self): - self.getPage("/tarfile") - self.assertBody("I am a tarfile") - - def testToolWithConfig(self): - if not sys.version_info >= (2, 5): - return self.skip("skipped (Python 2.5+ only)") - - self.getPage('/tooldecs/blah') - self.assertHeader('Content-Type', 'application/data') - - def testWarnToolOn(self): - # get - try: - numon = cherrypy.tools.numerify.on - except AttributeError: - pass - else: - raise AssertionError("Tool.on did not error as it should have.") - - # set - try: - cherrypy.tools.numerify.on = True - except AttributeError: - pass - else: - raise AssertionError("Tool.on did not error as it should have.") - diff --git a/cherrypy/test/test_tutorials.py b/cherrypy/test/test_tutorials.py deleted file mode 100644 index aab27861..00000000 --- a/cherrypy/test/test_tutorials.py +++ /dev/null @@ -1,201 +0,0 @@ -import sys - -import cherrypy -from cherrypy.test import helper - - -class TutorialTest(helper.CPWebCase): - - def setup_server(cls): - - conf = cherrypy.config.copy() - - def load_tut_module(name): - """Import or reload tutorial module as needed.""" - cherrypy.config.reset() - cherrypy.config.update(conf) - - target = "cherrypy.tutorial." + name - if target in sys.modules: - module = reload(sys.modules[target]) - else: - module = __import__(target, globals(), locals(), ['']) - # The above import will probably mount a new app at "". - app = cherrypy.tree.apps[""] - - app.root.load_tut_module = load_tut_module - app.root.sessions = sessions - app.root.traceback_setting = traceback_setting - - cls.supervisor.sync_apps() - load_tut_module.exposed = True - - def sessions(): - cherrypy.config.update({"tools.sessions.on": True}) - sessions.exposed = True - - def traceback_setting(): - return repr(cherrypy.request.show_tracebacks) - traceback_setting.exposed = True - - class Dummy: - pass - root = Dummy() - root.load_tut_module = load_tut_module - cherrypy.tree.mount(root) - setup_server = classmethod(setup_server) - - - def test01HelloWorld(self): - self.getPage("/load_tut_module/tut01_helloworld") - self.getPage("/") - self.assertBody('Hello world!') - - def test02ExposeMethods(self): - self.getPage("/load_tut_module/tut02_expose_methods") - self.getPage("/showMessage") - self.assertBody('Hello world!') - - def test03GetAndPost(self): - self.getPage("/load_tut_module/tut03_get_and_post") - - # Try different GET queries - self.getPage("/greetUser?name=Bob") - self.assertBody("Hey Bob, what's up?") - - self.getPage("/greetUser") - self.assertBody('Please enter your name here.') - - self.getPage("/greetUser?name=") - self.assertBody('No, really, enter your name here.') - - # Try the same with POST - self.getPage("/greetUser", method="POST", body="name=Bob") - self.assertBody("Hey Bob, what's up?") - - self.getPage("/greetUser", method="POST", body="name=") - self.assertBody('No, really, enter your name here.') - - def test04ComplexSite(self): - self.getPage("/load_tut_module/tut04_complex_site") - msg = ''' -

Here are some extra useful links:

- - - -

[Return to links page]

''' - self.getPage("/links/extra/") - self.assertBody(msg) - - def test05DerivedObjects(self): - self.getPage("/load_tut_module/tut05_derived_objects") - msg = ''' - - - Another Page - - -

Another Page

- -

- And this is the amazing second page! -

- - - - ''' - self.getPage("/another/") - self.assertBody(msg) - - def test06DefaultMethod(self): - self.getPage("/load_tut_module/tut06_default_method") - self.getPage('/hendrik') - self.assertBody('Hendrik Mans, CherryPy co-developer & crazy German ' - '(back)') - - def test07Sessions(self): - self.getPage("/load_tut_module/tut07_sessions") - self.getPage("/sessions") - - self.getPage('/') - self.assertBody("\n During your current session, you've viewed this" - "\n page 1 times! Your life is a patio of fun!" - "\n ") - - self.getPage('/', self.cookies) - self.assertBody("\n During your current session, you've viewed this" - "\n page 2 times! Your life is a patio of fun!" - "\n ") - - def test08GeneratorsAndYield(self): - self.getPage("/load_tut_module/tut08_generators_and_yield") - self.getPage('/') - self.assertBody('

Generators rule!

' - '

List of users:

' - 'Remi
Carlos
Hendrik
Lorenzo Lamas
' - '') - - def test09Files(self): - self.getPage("/load_tut_module/tut09_files") - - # Test upload - filesize = 5 - h = [("Content-type", "multipart/form-data; boundary=x"), - ("Content-Length", str(105 + filesize))] - b = '--x\n' + \ - 'Content-Disposition: form-data; name="myFile"; filename="hello.txt"\r\n' + \ - 'Content-Type: text/plain\r\n' + \ - '\r\n' + \ - 'a' * filesize + '\n' + \ - '--x--\n' - self.getPage('/upload', h, "POST", b) - self.assertBody(''' - - myFile length: %d
- myFile filename: hello.txt
- myFile mime-type: text/plain - - ''' % filesize) - - # Test download - self.getPage('/download') - self.assertStatus("200 OK") - self.assertHeader("Content-Type", "application/x-download") - self.assertHeader("Content-Disposition", - # Make sure the filename is quoted. - 'attachment; filename="pdf_file.pdf"') - self.assertEqual(len(self.body), 85698) - - def test10HTTPErrors(self): - self.getPage("/load_tut_module/tut10_http_errors") - - self.getPage("/") - self.assertInBody("""""") - self.assertInBody("""""") - self.assertInBody("""""") - self.assertInBody("""""") - self.assertInBody("""""") - - self.getPage("/traceback_setting") - setting = self.body - self.getPage("/toggleTracebacks") - self.assertStatus((302, 303)) - self.getPage("/traceback_setting") - self.assertBody(str(not eval(setting))) - - self.getPage("/error?code=500") - self.assertStatus(500) - self.assertInBody("The server encountered an unexpected condition " - "which prevented it from fulfilling the request.") - - self.getPage("/error?code=403") - self.assertStatus(403) - self.assertInBody("

You can't do that!

") - - self.getPage("/messageArg") - self.assertStatus(500) - self.assertInBody("If you construct an HTTPError with a 'message'") - diff --git a/cherrypy/test/test_virtualhost.py b/cherrypy/test/test_virtualhost.py deleted file mode 100644 index d6eed0ea..00000000 --- a/cherrypy/test/test_virtualhost.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - -import cherrypy -from cherrypy.test import helper - - -class VirtualHostTest(helper.CPWebCase): - - def setup_server(): - class Root: - def index(self): - return "Hello, world" - index.exposed = True - - def dom4(self): - return "Under construction" - dom4.exposed = True - - def method(self, value): - return "You sent %s" % repr(value) - method.exposed = True - - class VHost: - def __init__(self, sitename): - self.sitename = sitename - - def index(self): - return "Welcome to %s" % self.sitename - index.exposed = True - - def vmethod(self, value): - return "You sent %s" % repr(value) - vmethod.exposed = True - - def url(self): - return cherrypy.url("nextpage") - url.exposed = True - - # Test static as a handler (section must NOT include vhost prefix) - static = cherrypy.tools.staticdir.handler(section='/static', dir=curdir) - - root = Root() - root.mydom2 = VHost("Domain 2") - root.mydom3 = VHost("Domain 3") - hostmap = {'www.mydom2.com': '/mydom2', - 'www.mydom3.com': '/mydom3', - 'www.mydom4.com': '/dom4', - } - cherrypy.tree.mount(root, config={ - '/': {'request.dispatch': cherrypy.dispatch.VirtualHost(**hostmap)}, - # Test static in config (section must include vhost prefix) - '/mydom2/static2': {'tools.staticdir.on': True, - 'tools.staticdir.root': curdir, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.index': 'index.html', - }, - }) - setup_server = staticmethod(setup_server) - - def testVirtualHost(self): - self.getPage("/", [('Host', 'www.mydom1.com')]) - self.assertBody('Hello, world') - self.getPage("/mydom2/", [('Host', 'www.mydom1.com')]) - self.assertBody('Welcome to Domain 2') - - self.getPage("/", [('Host', 'www.mydom2.com')]) - self.assertBody('Welcome to Domain 2') - self.getPage("/", [('Host', 'www.mydom3.com')]) - self.assertBody('Welcome to Domain 3') - self.getPage("/", [('Host', 'www.mydom4.com')]) - self.assertBody('Under construction') - - # Test GET, POST, and positional params - self.getPage("/method?value=root") - self.assertBody("You sent u'root'") - self.getPage("/vmethod?value=dom2+GET", [('Host', 'www.mydom2.com')]) - self.assertBody("You sent u'dom2 GET'") - self.getPage("/vmethod", [('Host', 'www.mydom3.com')], method="POST", - body="value=dom3+POST") - self.assertBody("You sent u'dom3 POST'") - self.getPage("/vmethod/pos", [('Host', 'www.mydom3.com')]) - self.assertBody("You sent 'pos'") - - # Test that cherrypy.url uses the browser url, not the virtual url - self.getPage("/url", [('Host', 'www.mydom2.com')]) - self.assertBody("%s://www.mydom2.com/nextpage" % self.scheme) - - def test_VHost_plus_Static(self): - # Test static as a handler - self.getPage("/static/style.css", [('Host', 'www.mydom2.com')]) - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'text/css;charset=utf-8') - - # Test static in config - self.getPage("/static2/dirback.jpg", [('Host', 'www.mydom2.com')]) - self.assertStatus('200 OK') - self.assertHeader('Content-Type', 'image/jpeg') - - # Test static config with "index" arg - self.getPage("/static2/", [('Host', 'www.mydom2.com')]) - self.assertStatus('200 OK') - self.assertBody('Hello, world\r\n') - # Since tools.trailing_slash is on by default, this should redirect - self.getPage("/static2", [('Host', 'www.mydom2.com')]) - self.assertStatus(301) - diff --git a/cherrypy/test/test_wsgi_ns.py b/cherrypy/test/test_wsgi_ns.py deleted file mode 100644 index d57013c3..00000000 --- a/cherrypy/test/test_wsgi_ns.py +++ /dev/null @@ -1,80 +0,0 @@ -import cherrypy -from cherrypy.test import helper - - -class WSGI_Namespace_Test(helper.CPWebCase): - - def setup_server(): - - class WSGIResponse(object): - - def __init__(self, appresults): - self.appresults = appresults - self.iter = iter(appresults) - - def __iter__(self): - return self - - def next(self): - return self.iter.next() - - def close(self): - if hasattr(self.appresults, "close"): - self.appresults.close() - - - class ChangeCase(object): - - def __init__(self, app, to=None): - self.app = app - self.to = to - - def __call__(self, environ, start_response): - res = self.app(environ, start_response) - class CaseResults(WSGIResponse): - def next(this): - return getattr(this.iter.next(), self.to)() - return CaseResults(res) - - class Replacer(object): - - def __init__(self, app, map={}): - self.app = app - self.map = map - - def __call__(self, environ, start_response): - res = self.app(environ, start_response) - class ReplaceResults(WSGIResponse): - def next(this): - line = this.iter.next() - for k, v in self.map.iteritems(): - line = line.replace(k, v) - return line - return ReplaceResults(res) - - class Root(object): - - def index(self): - return "HellO WoRlD!" - index.exposed = True - - - root_conf = {'wsgi.pipeline': [('replace', Replacer)], - 'wsgi.replace.map': {'L': 'X', 'l': 'r'}, - } - - app = cherrypy.Application(Root()) - app.wsgiapp.pipeline.append(('changecase', ChangeCase)) - app.wsgiapp.config['changecase'] = {'to': 'upper'} - cherrypy.tree.mount(app, config={'/': root_conf}) - setup_server = staticmethod(setup_server) - - - def test_pipeline(self): - if not cherrypy.server.httpserver: - return self.skip() - - self.getPage("/") - # If body is "HEXXO WORXD!", the middleware was applied out of order. - self.assertBody("HERRO WORRD!") - diff --git a/cherrypy/test/test_wsgi_vhost.py b/cherrypy/test/test_wsgi_vhost.py deleted file mode 100644 index abb1a917..00000000 --- a/cherrypy/test/test_wsgi_vhost.py +++ /dev/null @@ -1,36 +0,0 @@ -import cherrypy -from cherrypy.test import helper - - -class WSGI_VirtualHost_Test(helper.CPWebCase): - - def setup_server(): - - class ClassOfRoot(object): - - def __init__(self, name): - self.name = name - - def index(self): - return "Welcome to the %s website!" % self.name - index.exposed = True - - - default = cherrypy.Application(None) - - domains = {} - for year in range(1997, 2008): - app = cherrypy.Application(ClassOfRoot('Class of %s' % year)) - domains['www.classof%s.example' % year] = app - - cherrypy.tree.graft(cherrypy._cpwsgi.VirtualHost(default, domains)) - setup_server = staticmethod(setup_server) - - def test_welcome(self): - if not cherrypy.server.using_wsgi: - return self.skip("skipped (not using WSGI)... ") - - for year in range(1997, 2008): - self.getPage("/", headers=[('Host', 'www.classof%s.example' % year)]) - self.assertBody("Welcome to the Class of %s website!" % year) - diff --git a/cherrypy/test/test_wsgiapps.py b/cherrypy/test/test_wsgiapps.py deleted file mode 100644 index fa5420c5..00000000 --- a/cherrypy/test/test_wsgiapps.py +++ /dev/null @@ -1,111 +0,0 @@ -from cherrypy.test import helper - - -class WSGIGraftTests(helper.CPWebCase): - - def setup_server(): - import os - curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - - import cherrypy - - def test_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - start_response(status, response_headers) - output = ['Hello, world!\n', - 'This is a wsgi app running within CherryPy!\n\n'] - keys = list(environ.keys()) - keys.sort() - for k in keys: - output.append('%s: %s\n' % (k,environ[k])) - return output - - def test_empty_string_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - start_response(status, response_headers) - return ['Hello', '', ' ', '', 'world'] - - - class WSGIResponse(object): - - def __init__(self, appresults): - self.appresults = appresults - self.iter = iter(appresults) - - def __iter__(self): - return self - - def next(self): - return self.iter.next() - - def close(self): - if hasattr(self.appresults, "close"): - self.appresults.close() - - - class ReversingMiddleware(object): - - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - results = app(environ, start_response) - class Reverser(WSGIResponse): - def next(this): - line = list(this.iter.next()) - line.reverse() - return "".join(line) - return Reverser(results) - - class Root: - def index(self): - return "I'm a regular CherryPy page handler!" - index.exposed = True - - - cherrypy.tree.mount(Root()) - - cherrypy.tree.graft(test_app, '/hosted/app1') - cherrypy.tree.graft(test_empty_string_app, '/hosted/app3') - - # Set script_name explicitly to None to signal CP that it should - # be pulled from the WSGI environ each time. - app = cherrypy.Application(Root(), script_name=None) - cherrypy.tree.graft(ReversingMiddleware(app), '/hosted/app2') - setup_server = staticmethod(setup_server) - - wsgi_output = '''Hello, world! -This is a wsgi app running within CherryPy!''' - - def test_01_standard_app(self): - self.getPage("/") - self.assertBody("I'm a regular CherryPy page handler!") - - def test_04_pure_wsgi(self): - import cherrypy - if not cherrypy.server.using_wsgi: - return self.skip("skipped (not using WSGI)... ") - self.getPage("/hosted/app1") - self.assertHeader("Content-Type", "text/plain") - self.assertInBody(self.wsgi_output) - - def test_05_wrapped_cp_app(self): - import cherrypy - if not cherrypy.server.using_wsgi: - return self.skip("skipped (not using WSGI)... ") - self.getPage("/hosted/app2/") - body = list("I'm a regular CherryPy page handler!") - body.reverse() - body = "".join(body) - self.assertInBody(body) - - def test_06_empty_string_app(self): - import cherrypy - if not cherrypy.server.using_wsgi: - return self.skip("skipped (not using WSGI)... ") - self.getPage("/hosted/app3") - self.assertHeader("Content-Type", "text/plain") - self.assertInBody('Hello world') - diff --git a/cherrypy/test/test_xmlrpc.py b/cherrypy/test/test_xmlrpc.py deleted file mode 100644 index c4bf61e0..00000000 --- a/cherrypy/test/test_xmlrpc.py +++ /dev/null @@ -1,172 +0,0 @@ -import sys -from xmlrpclib import DateTime, Fault, ServerProxy, SafeTransport - -class HTTPSTransport(SafeTransport): - """Subclass of SafeTransport to fix sock.recv errors (by using file).""" - - def request(self, host, handler, request_body, verbose=0): - # issue XML-RPC request - h = self.make_connection(host) - if verbose: - h.set_debuglevel(1) - - self.send_request(h, handler, request_body) - self.send_host(h, host) - self.send_user_agent(h) - self.send_content(h, request_body) - - errcode, errmsg, headers = h.getreply() - if errcode != 200: - raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg, - headers) - - self.verbose = verbose - - # Here's where we differ from the superclass. It says: - # try: - # sock = h._conn.sock - # except AttributeError: - # sock = None - # return self._parse_response(h.getfile(), sock) - - return self.parse_response(h.getfile()) - -import cherrypy - - -def setup_server(): - from cherrypy import _cptools - - class Root: - def index(self): - return "I'm a standard index!" - index.exposed = True - - - class XmlRpc(_cptools.XMLRPCController): - - def foo(self): - return "Hello world!" - foo.exposed = True - - def return_single_item_list(self): - return [42] - return_single_item_list.exposed = True - - def return_string(self): - return "here is a string" - return_string.exposed = True - - def return_tuple(self): - return ('here', 'is', 1, 'tuple') - return_tuple.exposed = True - - def return_dict(self): - return dict(a=1, b=2, c=3) - return_dict.exposed = True - - def return_composite(self): - return dict(a=1,z=26), 'hi', ['welcome', 'friend'] - return_composite.exposed = True - - def return_int(self): - return 42 - return_int.exposed = True - - def return_float(self): - return 3.14 - return_float.exposed = True - - def return_datetime(self): - return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) - return_datetime.exposed = True - - def return_boolean(self): - return True - return_boolean.exposed = True - - def test_argument_passing(self, num): - return num * 2 - test_argument_passing.exposed = True - - def test_returning_Fault(self): - return Fault(1, "custom Fault response") - test_returning_Fault.exposed = True - - root = Root() - root.xmlrpc = XmlRpc() - cherrypy.tree.mount(root, config={'/': { - 'request.dispatch': cherrypy.dispatch.XMLRPCDispatcher(), - 'tools.xmlrpc.allow_none': 0, - }}) - - -from cherrypy.test import helper - -class XmlRpcTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - def testXmlRpc(self): - - scheme = "http" - try: - scheme = self.harness.scheme - except AttributeError: - pass - - if scheme == "https": - url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url, transport=HTTPSTransport()) - else: - url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url) - - # begin the tests ... - self.getPage("/xmlrpc/foo") - self.assertBody("Hello world!") - - self.assertEqual(proxy.return_single_item_list(), [42]) - self.assertNotEqual(proxy.return_single_item_list(), 'one bazillion') - self.assertEqual(proxy.return_string(), "here is a string") - self.assertEqual(proxy.return_tuple(), list(('here', 'is', 1, 'tuple'))) - self.assertEqual(proxy.return_dict(), {'a': 1, 'c': 3, 'b': 2}) - self.assertEqual(proxy.return_composite(), - [{'a': 1, 'z': 26}, 'hi', ['welcome', 'friend']]) - self.assertEqual(proxy.return_int(), 42) - self.assertEqual(proxy.return_float(), 3.14) - self.assertEqual(proxy.return_datetime(), - DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) - self.assertEqual(proxy.return_boolean(), True) - self.assertEqual(proxy.test_argument_passing(22), 22 * 2) - - # Test an error in the page handler (should raise an xmlrpclib.Fault) - try: - proxy.test_argument_passing({}) - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, ("unsupported operand type(s) " - "for *: 'dict' and 'int'")) - else: - self.fail("Expected xmlrpclib.Fault") - - # http://www.cherrypy.org/ticket/533 - # if a method is not found, an xmlrpclib.Fault should be raised - try: - proxy.non_method() - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, 'method "non_method" is not supported') - else: - self.fail("Expected xmlrpclib.Fault") - - # Test returning a Fault from the page handler. - try: - proxy.test_returning_Fault() - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, ("custom Fault response")) - else: - self.fail("Expected xmlrpclib.Fault") - diff --git a/cherrypy/test/webtest.py b/cherrypy/test/webtest.py deleted file mode 100644 index 969eab0e..00000000 --- a/cherrypy/test/webtest.py +++ /dev/null @@ -1,535 +0,0 @@ -"""Extensions to unittest for web frameworks. - -Use the WebCase.getPage method to request a page from your HTTP server. - -Framework Integration -===================== - -If you have control over your server process, you can handle errors -in the server-side of the HTTP conversation a bit better. You must run -both the client (your WebCase tests) and the server in the same process -(but in separate threads, obviously). - -When an error occurs in the framework, call server_error. It will print -the traceback to stdout, and keep any assertions you have from running -(the assumption is that, if the server errors, the page output will not -be of further significance to your tests). -""" - -import os -import pprint -import re -import socket -import sys -import time -import traceback -import types - -from unittest import * -from unittest import _TextTestResult - -from cherrypy._cpcompat import basestring, HTTPConnection, HTTPSConnection, unicodestr - - - -def interface(host): - """Return an IP address for a client connection given the server host. - - If the server is listening on '0.0.0.0' (INADDR_ANY) - or '::' (IN6ADDR_ANY), this will return the proper localhost.""" - if host == '0.0.0.0': - # INADDR_ANY, which should respond on localhost. - return "127.0.0.1" - if host == '::': - # IN6ADDR_ANY, which should respond on localhost. - return "::1" - return host - - -class TerseTestResult(_TextTestResult): - - def printErrors(self): - # Overridden to avoid unnecessary empty line - if self.errors or self.failures: - if self.dots or self.showAll: - self.stream.writeln() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - - -class TerseTestRunner(TextTestRunner): - """A test runner class that displays results in textual form.""" - - def _makeResult(self): - return TerseTestResult(self.stream, self.descriptions, self.verbosity) - - def run(self, test): - "Run the given test case or test suite." - # Overridden to remove unnecessary empty lines and separators - result = self._makeResult() - test(result) - result.printErrors() - if not result.wasSuccessful(): - self.stream.write("FAILED (") - failed, errored = list(map(len, (result.failures, result.errors))) - if failed: - self.stream.write("failures=%d" % failed) - if errored: - if failed: self.stream.write(", ") - self.stream.write("errors=%d" % errored) - self.stream.writeln(")") - return result - - -class ReloadingTestLoader(TestLoader): - - def loadTestsFromName(self, name, module=None): - """Return a suite of all tests cases given a string specifier. - - The name may resolve either to a module, a test case class, a - test method within a test case class, or a callable object which - returns a TestCase or TestSuite instance. - - The method optionally resolves the names relative to a given module. - """ - parts = name.split('.') - unused_parts = [] - if module is None: - if not parts: - raise ValueError("incomplete test name: %s" % name) - else: - parts_copy = parts[:] - while parts_copy: - target = ".".join(parts_copy) - if target in sys.modules: - module = reload(sys.modules[target]) - parts = unused_parts - break - else: - try: - module = __import__(target) - parts = unused_parts - break - except ImportError: - unused_parts.insert(0,parts_copy[-1]) - del parts_copy[-1] - if not parts_copy: - raise - parts = parts[1:] - obj = module - for part in parts: - obj = getattr(obj, part) - - if type(obj) == types.ModuleType: - return self.loadTestsFromModule(obj) - elif (isinstance(obj, (type, types.ClassType)) and - issubclass(obj, TestCase)): - return self.loadTestsFromTestCase(obj) - elif type(obj) == types.UnboundMethodType: - return obj.im_class(obj.__name__) - elif hasattr(obj, '__call__'): - test = obj() - if not isinstance(test, TestCase) and \ - not isinstance(test, TestSuite): - raise ValueError("calling %s returned %s, " - "not a test" % (obj,test)) - return test - else: - raise ValueError("do not know how to make test from: %s" % obj) - - -try: - # Jython support - if sys.platform[:4] == 'java': - def getchar(): - # Hopefully this is enough - return sys.stdin.read(1) - else: - # On Windows, msvcrt.getch reads a single char without output. - import msvcrt - def getchar(): - return msvcrt.getch() -except ImportError: - # Unix getchr - import tty, termios - def getchar(): - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - -class WebCase(TestCase): - HOST = "127.0.0.1" - PORT = 8000 - HTTP_CONN = HTTPConnection - PROTOCOL = "HTTP/1.1" - - scheme = "http" - url = None - - status = None - headers = None - body = None - - encoding = 'utf-8' - - time = None - - def get_conn(self, auto_open=False): - """Return a connection to our HTTP server.""" - if self.scheme == "https": - cls = HTTPSConnection - else: - cls = HTTPConnection - conn = cls(self.interface(), self.PORT) - # Automatically re-connect? - conn.auto_open = auto_open - conn.connect() - return conn - - def set_persistent(self, on=True, auto_open=False): - """Make our HTTP_CONN persistent (or not). - - If the 'on' argument is True (the default), then self.HTTP_CONN - will be set to an instance of HTTPConnection (or HTTPS - if self.scheme is "https"). This will then persist across requests. - - We only allow for a single open connection, so if you call this - and we currently have an open connection, it will be closed. - """ - try: - self.HTTP_CONN.close() - except (TypeError, AttributeError): - pass - - if on: - self.HTTP_CONN = self.get_conn(auto_open=auto_open) - else: - if self.scheme == "https": - self.HTTP_CONN = HTTPSConnection - else: - self.HTTP_CONN = HTTPConnection - - def _get_persistent(self): - return hasattr(self.HTTP_CONN, "__class__") - def _set_persistent(self, on): - self.set_persistent(on) - persistent = property(_get_persistent, _set_persistent) - - def interface(self): - """Return an IP address for a client connection. - - If the server is listening on '0.0.0.0' (INADDR_ANY) - or '::' (IN6ADDR_ANY), this will return the proper localhost.""" - return interface(self.HOST) - - def getPage(self, url, headers=None, method="GET", body=None, protocol=None): - """Open the url with debugging support. Return status, headers, body.""" - ServerError.on = False - - if isinstance(url, unicodestr): - url = url.encode('utf-8') - if isinstance(body, unicodestr): - body = body.encode('utf-8') - - self.url = url - self.time = None - start = time.time() - result = openURL(url, headers, method, body, self.HOST, self.PORT, - self.HTTP_CONN, protocol or self.PROTOCOL) - self.time = time.time() - start - self.status, self.headers, self.body = result - - # Build a list of request cookies from the previous response cookies. - self.cookies = [('Cookie', v) for k, v in self.headers - if k.lower() == 'set-cookie'] - - if ServerError.on: - raise ServerError() - return result - - interactive = True - console_height = 30 - - def _handlewebError(self, msg): - print("") - print(" ERROR: %s" % msg) - - if not self.interactive: - raise self.failureException(msg) - - p = " Show: [B]ody [H]eaders [S]tatus [U]RL; [I]gnore, [R]aise, or sys.e[X]it >> " - sys.stdout.write(p) - sys.stdout.flush() - while True: - i = getchar().upper() - if i not in "BHSUIRX": - continue - print(i.upper()) # Also prints new line - if i == "B": - for x, line in enumerate(self.body.splitlines()): - if (x + 1) % self.console_height == 0: - # The \r and comma should make the next line overwrite - sys.stdout.write("<-- More -->\r") - m = getchar().lower() - # Erase our "More" prompt - sys.stdout.write(" \r") - if m == "q": - break - print(line) - elif i == "H": - pprint.pprint(self.headers) - elif i == "S": - print(self.status) - elif i == "U": - print(self.url) - elif i == "I": - # return without raising the normal exception - return - elif i == "R": - raise self.failureException(msg) - elif i == "X": - self.exit() - sys.stdout.write(p) - sys.stdout.flush() - - def exit(self): - sys.exit() - - def assertStatus(self, status, msg=None): - """Fail if self.status != status.""" - if isinstance(status, basestring): - if not self.status == status: - if msg is None: - msg = 'Status (%r) != %r' % (self.status, status) - self._handlewebError(msg) - elif isinstance(status, int): - code = int(self.status[:3]) - if code != status: - if msg is None: - msg = 'Status (%r) != %r' % (self.status, status) - self._handlewebError(msg) - else: - # status is a tuple or list. - match = False - for s in status: - if isinstance(s, basestring): - if self.status == s: - match = True - break - elif int(self.status[:3]) == s: - match = True - break - if not match: - if msg is None: - msg = 'Status (%r) not in %r' % (self.status, status) - self._handlewebError(msg) - - def assertHeader(self, key, value=None, msg=None): - """Fail if (key, [value]) not in self.headers.""" - lowkey = key.lower() - for k, v in self.headers: - if k.lower() == lowkey: - if value is None or str(value) == v: - return v - - if msg is None: - if value is None: - msg = '%r not in headers' % key - else: - msg = '%r:%r not in headers' % (key, value) - self._handlewebError(msg) - - def assertHeaderItemValue(self, key, value, msg=None): - """Fail if the header does not contain the specified value""" - actual_value = self.assertHeader(key, msg=msg) - header_values = map(str.strip, actual_value.split(',')) - if value in header_values: - return value - - if msg is None: - msg = "%r not in %r" % (value, header_values) - self._handlewebError(msg) - - def assertNoHeader(self, key, msg=None): - """Fail if key in self.headers.""" - lowkey = key.lower() - matches = [k for k, v in self.headers if k.lower() == lowkey] - if matches: - if msg is None: - msg = '%r in headers' % key - self._handlewebError(msg) - - def assertBody(self, value, msg=None): - """Fail if value != self.body.""" - if value != self.body: - if msg is None: - msg = 'expected body:\n%r\n\nactual body:\n%r' % (value, self.body) - self._handlewebError(msg) - - def assertInBody(self, value, msg=None): - """Fail if value not in self.body.""" - if value not in self.body: - if msg is None: - msg = '%r not in body: %s' % (value, self.body) - self._handlewebError(msg) - - def assertNotInBody(self, value, msg=None): - """Fail if value in self.body.""" - if value in self.body: - if msg is None: - msg = '%r found in body' % value - self._handlewebError(msg) - - def assertMatchesBody(self, pattern, msg=None, flags=0): - """Fail if value (a regex pattern) is not in self.body.""" - if re.search(pattern, self.body, flags) is None: - if msg is None: - msg = 'No match for %r in body' % pattern - self._handlewebError(msg) - - -methods_with_bodies = ("POST", "PUT") - -def cleanHeaders(headers, method, body, host, port): - """Return request headers, with required headers added (if missing).""" - if headers is None: - headers = [] - - # Add the required Host request header if not present. - # [This specifies the host:port of the server, not the client.] - found = False - for k, v in headers: - if k.lower() == 'host': - found = True - break - if not found: - if port == 80: - headers.append(("Host", host)) - else: - headers.append(("Host", "%s:%s" % (host, port))) - - if method in methods_with_bodies: - # Stick in default type and length headers if not present - found = False - for k, v in headers: - if k.lower() == 'content-type': - found = True - break - if not found: - headers.append(("Content-Type", "application/x-www-form-urlencoded")) - headers.append(("Content-Length", str(len(body or "")))) - - return headers - - -def shb(response): - """Return status, headers, body the way we like from a response.""" - h = [] - key, value = None, None - for line in response.msg.headers: - if line: - if line[0] in " \t": - value += line.strip() - else: - if key and value: - h.append((key, value)) - key, value = line.split(":", 1) - key = key.strip() - value = value.strip() - if key and value: - h.append((key, value)) - - return "%s %s" % (response.status, response.reason), h, response.read() - - -def openURL(url, headers=None, method="GET", body=None, - host="127.0.0.1", port=8000, http_conn=HTTPConnection, - protocol="HTTP/1.1"): - """Open the given HTTP resource and return status, headers, and body.""" - - headers = cleanHeaders(headers, method, body, host, port) - - # Trying 10 times is simply in case of socket errors. - # Normal case--it should run once. - for trial in range(10): - try: - # Allow http_conn to be a class or an instance - if hasattr(http_conn, "host"): - conn = http_conn - else: - conn = http_conn(interface(host), port) - - conn._http_vsn_str = protocol - conn._http_vsn = int("".join([x for x in protocol if x.isdigit()])) - - # skip_accept_encoding argument added in python version 2.4 - if sys.version_info < (2, 4): - def putheader(self, header, value): - if header == 'Accept-Encoding' and value == 'identity': - return - self.__class__.putheader(self, header, value) - import new - conn.putheader = new.instancemethod(putheader, conn, conn.__class__) - conn.putrequest(method.upper(), url, skip_host=True) - else: - conn.putrequest(method.upper(), url, skip_host=True, - skip_accept_encoding=True) - - for key, value in headers: - conn.putheader(key, value) - conn.endheaders() - - if body is not None: - conn.send(body) - - # Handle response - response = conn.getresponse() - - s, h, b = shb(response) - - if not hasattr(http_conn, "host"): - # We made our own conn instance. Close it. - conn.close() - - return s, h, b - except socket.error: - time.sleep(0.5) - raise - - -# Add any exceptions which your web framework handles -# normally (that you don't want server_error to trap). -ignored_exceptions = [] - -# You'll want set this to True when you can't guarantee -# that each response will immediately follow each request; -# for example, when handling requests via multiple threads. -ignore_all = False - -class ServerError(Exception): - on = False - - -def server_error(exc=None): - """Server debug hook. Return True if exception handled, False if ignored. - - You probably want to wrap this, so you can still handle an error using - your framework when it's ignored. - """ - if exc is None: - exc = sys.exc_info() - - if ignore_all or exc[0] in ignored_exceptions: - return False - else: - ServerError.on = True - print("") - print("".join(traceback.format_exception(*exc))) - return True - diff --git a/cherrypy/wsgiserver/__init__.py b/cherrypy/wsgiserver/__init__.py index 55d1dd90..ee6190fe 100644 --- a/cherrypy/wsgiserver/__init__.py +++ b/cherrypy/wsgiserver/__init__.py @@ -1,2219 +1,14 @@ -"""A high-speed, production ready, thread pooled, generic HTTP server. +__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', + 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', + 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', + 'WorkerThread', 'ThreadPool', 'SSLAdapter', + 'CherryPyWSGIServer', + 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', + 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class'] -Simplest example on how to use this module directly -(without using CherryPy's application machinery):: - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - server.start() - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher:: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. - -This won't call the CherryPy engine (application side) at all, only the -HTTP server, which is independent from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue:: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop:: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - read_headers(req.rfile, req.inheaders) - req.respond() - -> response = app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - -CRLF = '\r\n' -import os -import Queue -import re -quoted_slash = re.compile("(?i)%2F") -import rfc822 -import socket import sys -if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 -try: - import cStringIO as StringIO -except ImportError: - import StringIO -DEFAULT_BUFFER_SIZE = -1 - -_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) - -import threading -import time -import traceback -def format_exc(limit=None): - """Like print_exc() but return a string. Backport for Python 2.3.""" - try: - etype, value, tb = sys.exc_info() - return ''.join(traceback.format_exception(etype, value, tb, limit)) - finally: - etype = value = tb = None - - -from urllib import unquote -from urlparse import urlparse -import warnings - -import errno - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return dict.fromkeys(nums).keys() - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", - ) -socket_errors_to_ignore.append("timed out") -socket_errors_to_ignore.append("The read operation timed out") - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate'] - - -import logging -if not hasattr(logging, 'statistics'): logging.statistics = {} - - -def read_headers(rfile, hdict=None): - """Read headers from the given stream into the given header dict. - - If hdict is None, a new header dict is created. Returns the populated - header dict. - - Headers which are repeated are folded together using a comma if their - specification so dictates. - - This function raises ValueError when the read bytes violate the HTTP spec. - You should probably return "400 Bad Request" if this happens. - """ - if hdict is None: - hdict = {} - - while True: - line = rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - if line[0] in ' \t': - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(":", 1) - except ValueError: - raise ValueError("Illegal header line.") - # TODO: what about TE and WWW-Authenticate? - k = k.strip().title() - v = v.strip() - hname = k - - if k in comma_separated_headers: - existing = hdict.get(hname) - if existing: - v = ", ".join((existing, v)) - hdict[hname] = v - - return hdict - - -class MaxSizeExceeded(Exception): - pass - -class SizeCheckWrapper(object): - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See http://www.cherrypy.org/ticket/421 - if len(data) < 256 or data[-1:] == "\n": - return ''.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class KnownLengthRFile(object): - """Wraps a file-like object, returning an empty string when exhausted.""" - - def __init__(self, rfile, content_length): - self.rfile = rfile - self.remaining = content_length - - def read(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.read(size) - self.remaining -= len(data) - return data - - def readline(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.readline(size) - self.remaining -= len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.remaining -= len(data) - return data - - -class ChunkedRFile(object): - """Wraps a file-like object, returning an empty string when exhausted. - - This class is intended to provide a conforming wsgi.input value for - request entities that have been encoded with the 'chunked' transfer - encoding. - """ - - def __init__(self, rfile, maxlen, bufsize=8192): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - self.buffer = '' - self.bufsize = bufsize - self.closed = False - - def _fetch(self): - if self.closed: - return - - line = self.rfile.readline() - self.bytes_read += len(line) - - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) - - line = line.strip().split(";", 1) - - try: - chunk_size = line.pop(0) - chunk_size = int(chunk_size, 16) - except ValueError: - raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) - - if chunk_size <= 0: - self.closed = True - return - -## if line: chunk_extension = line[0] - - if self.maxlen and self.bytes_read + chunk_size > self.maxlen: - raise IOError("Request Entity Too Large") - - chunk = self.rfile.read(chunk_size) - self.bytes_read += len(chunk) - self.buffer += chunk - - crlf = self.rfile.read(2) - if crlf != CRLF: - raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") - - def read(self, size=None): - data = '' - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - if size: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - data += self.buffer - - def readline(self, size=None): - data = '' - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - newline_pos = self.buffer.find('\n') - if size: - if newline_pos == -1: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - remaining = min(size - len(data), newline_pos) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - if newline_pos == -1: - data += self.buffer - else: - data += self.buffer[:newline_pos] - self.buffer = self.buffer[newline_pos:] - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def read_trailer_lines(self): - if not self.closed: - raise ValueError( - "Cannot read trailers until the request body has been read.") - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - self.bytes_read += len(line) - if self.maxlen and self.bytes_read > self.maxlen: - raise IOError("Request Entity Too Large") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - yield line - - def close(self): - self.rfile.close() - - def __iter__(self): - # Shamelessly stolen from StringIO - total = 0 - line = self.readline(sizehint) - while line: - yield line - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - - -class HTTPRequest(object): - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - """ - - server = None - """The HTTPServer object which is receiving this request.""" - - conn = None - """The HTTPConnection object on which this request connected.""" - - inheaders = {} - """A dict of request headers.""" - - outheaders = [] - """A list of header tuples to write in the response.""" - - ready = False - """When True, the request has been parsed and is ready to begin generating - the response. When False, signals the calling Connection that the response - should not be generated and the connection should close.""" - - close_connection = False - """Signals the calling Connection that the request should close. This does - not imply an error! The client and/or server may each request that the - connection be closed.""" - - chunked_write = False - """If True, output will be encoded with the "chunked" transfer-coding. - - This value is set automatically inside send_headers.""" - - def __init__(self, server, conn): - self.server= server - self.conn = conn - - self.ready = False - self.started_request = False - self.scheme = "http" - if self.server.ssl_adapter is not None: - self.scheme = "https" - # Use the lowest-common protocol in case read_request_line errors. - self.response_protocol = 'HTTP/1.0' - self.inheaders = {} - - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = self.__class__.close_connection - self.chunked_read = False - self.chunked_write = self.__class__.chunked_write - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) - try: - self.read_request_line() - except MaxSizeExceeded: - self.simple_response("414 Request-URI Too Long", - "The Request-URI sent with the request exceeds the maximum " - "allowed bytes.") - return - - try: - success = self.read_request_headers() - except MaxSizeExceeded: - self.simple_response("413 Request Entity Too Large", - "The headers sent with the request exceed the maximum " - "allowed bytes.") - return - else: - if not success: - return - - self.ready = True - - def read_request_line(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - - # Set started_request to True so communicate() knows to send 408 - # from here on out. - self.started_request = True - if not request_line: - # Force self.ready = False so the connection will close. - self.ready = False - return - - if request_line == CRLF: - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - self.ready = False - return - - if not request_line.endswith(CRLF): - self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") - return - - try: - method, uri, req_protocol = request_line.strip().split(" ", 2) - rp = int(req_protocol[5]), int(req_protocol[7]) - except (ValueError, IndexError): - self.simple_response("400 Bad Request", "Malformed Request-Line") - return - - self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if '#' in path: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return - - if scheme: - self.scheme = scheme - - qs = '' - if '?' in path: - path, qs = path.split('?', 1) - - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [unquote(x) for x in quoted_slash.split(path)] - except ValueError, ex: - self.simple_response("400 Bad Request", ex.args[0]) - return - path = "%2F".join(atoms) - self.path = path - - # Note that, like wsgiref and most other HTTP servers, - # we "% HEX HEX"-unquote the path but not the query string. - self.qs = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - sp = int(self.server.protocol[5]), int(self.server.protocol[7]) - - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return - self.request_protocol = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" - - # then all the http headers - try: - read_headers(self.rfile, self.inheaders) - except ValueError, ex: - self.simple_response("400 Bad Request", ex.args[0]) - return False - - mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: - self.simple_response("413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return False - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if self.inheaders.get("Connection", "") == "close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if self.inheaders.get("Connection", "") != "Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = self.inheaders.get("Transfer-Encoding") - if te: - te = [x.strip().lower() for x in te.split(",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == "chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return False - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if self.inheaders.get("Expect", "") == "100-continue": - # Don't use simple_response here, because it emits headers - # we don't want. See http://www.cherrypy.org/ticket/951 - msg = self.server.protocol + " 100 Continue\r\n\r\n" - try: - self.conn.wfile.sendall(msg) - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - raise - return True - - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == "*": - return None, None, uri - - i = uri.find('://') - if i > 0 and '?' not in uri[:i]: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] - scheme, remainder = uri[:i].lower(), uri[i + 3:] - authority, path = remainder.split("/", 1) - return scheme, authority, path - - if uri.startswith('/'): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - - def respond(self): - """Call the gateway and write its iterable output.""" - mrbs = self.server.max_request_body_size - if self.chunked_read: - self.rfile = ChunkedRFile(self.conn.rfile, mrbs) - else: - cl = int(self.inheaders.get("Content-Length", 0)) - if mrbs and mrbs < cl: - if not self.sent_headers: - self.simple_response("413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return - self.rfile = KnownLengthRFile(self.conn.rfile, cl) - - self.server.gateway(self).respond() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.conn.wfile.sendall("0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = [self.server.protocol + " " + - status + CRLF, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n"] - - if status[:3] in ("413", "414"): - # Request Entity Too Large / Request-URI Too Long - self.close_connection = True - if self.response_protocol == 'HTTP/1.1': - # This will not be true for 414, since read_request_line - # usually raises 414 before reading the whole line, and we - # therefore cannot know the proper response_protocol. - buf.append("Connection: close\r\n") - else: - # HTTP/1.0 had no 413/414 status nor Connection header. - # Emit 400 instead and trust the message body is enough. - status = "400 Bad Request" - - buf.append(CRLF) - if msg: - if isinstance(msg, unicode): - msg = msg.encode("ISO-8859-1") - buf.append(msg) - - try: - self.conn.wfile.sendall("".join(buf)) - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - raise - - def write(self, chunk): - """Write unbuffered data to the client.""" - if self.chunked_write and chunk: - buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF] - self.conn.wfile.sendall("".join(buf)) - else: - self.conn.wfile.sendall(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers. - - You must set self.status, and self.outheaders before calling this. - """ - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif "content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != 'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append(("Transfer-Encoding", "chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if "connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append(("Connection", "close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append(("Connection", "Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - remaining = getattr(self.rfile, 'remaining', 0) - if remaining > 0: - self.rfile.read(remaining) - - if "date" not in hkeys: - self.outheaders.append(("Date", rfc822.formatdate())) - - if "server" not in hkeys: - self.outheaders.append(("Server", self.server.server_name)) - - buf = [self.server.protocol + " " + self.status + CRLF] - for k, v in self.outheaders: - buf.append(k + ": " + v + CRLF) - buf.append(CRLF) - self.conn.wfile.sendall("".join(buf)) - - -class NoSSLError(Exception): - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -class CP_fileobject(socket._fileobject): - """Faux file object attached to a socket object.""" - - def __init__(self, *args, **kwargs): - self.bytes_read = 0 - self.bytes_written = 0 - socket._fileobject.__init__(self, *args, **kwargs) - - def sendall(self, data): - """Sendall for non-blocking sockets.""" - while data: - try: - bytes_sent = self.send(data) - data = data[bytes_sent:] - except socket.error, e: - if e.args[0] not in socket_errors_nonblocking: - raise - - def send(self, data): - bytes_sent = self._sock.send(data) - self.bytes_written += bytes_sent - return bytes_sent - - def flush(self): - if self._wbuf: - buffer = "".join(self._wbuf) - self._wbuf = [] - self.sendall(buffer) - - def recv(self, size): - while True: - try: - data = self._sock.recv(size) - self.bytes_read += len(data) - return data - except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): - raise - - if not _fileobject_uses_str_type: - def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - if size < 0: - # Read until EOF - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(rbufsize) - if not data: - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - data = self.recv(left) - if not data: - break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - def readline(self, size=-1): - buf = self._rbuf - buf.seek(0, 2) # seek end - if buf.tell() > 0: - # check if we already have it in our buffer - buf.seek(0) - bline = buf.readline(size) - if bline.endswith('\n') or len(bline) == size: - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return bline - del bline - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - buf.seek(0) - buffers = [buf.read()] - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - data = None - recv = self.recv - while data != "\n": - data = recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - - buf.seek(0, 2) # seek end - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - nl = data.find('\n') - if nl >= 0: - nl += 1 - buf.write(data[:nl]) - self._rbuf.write(data[nl:]) - del data - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or \n or EOF seen, whichever comes first - buf.seek(0, 2) # seek end - buf_len = buf.tell() - if buf_len >= size: - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO.StringIO() - self._rbuf.write(buf.read()) - return rv - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. - while True: - data = self.recv(self._rbufsize) - if not data: - break - left = size - buf_len - # did we just receive a newline? - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - # save the excess data to _rbuf - self._rbuf.write(data[nl:]) - if buf_len: - buf.write(data[:nl]) - break - else: - # Shortcut. Avoid data copy through buf when returning - # a substring of our first recv(). - return data[:nl] - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid data copy through buf when - # returning exactly all of our first recv(). - return data - if n >= left: - buf.write(data[:left]) - self._rbuf.write(data[left:]) - break - buf.write(data) - buf_len += n - #assert buf_len == buf.tell() - return buf.getvalue() - else: - def read(self, size=-1): - if size < 0: - # Read until EOF - buffers = [self._rbuf] - self._rbuf = "" - if self._rbufsize <= 1: - recv_size = self.default_bufsize - else: - recv_size = self._rbufsize - - while True: - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - return "".join(buffers) - else: - # Read until size bytes or EOF seen, whichever comes first - data = self._rbuf - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - left = size - buf_len - recv_size = max(self._rbufsize, left) - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - def readline(self, size=-1): - data = self._rbuf - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - assert data == "" - buffers = [] - while data != "\n": - data = self.recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - return "".join(buffers) - else: - # Read until size bytes or \n or EOF seen, whichever comes first - nl = data.find('\n', 0, size) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - left = size - buf_len - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - -class HTTPConnection(object): - """An HTTP connection (active socket). - - server: the Server object which received this connection. - socket: the raw socket object (usually TCP) for this connection. - makefile: a fileobject class for reading from the socket. - """ - - remote_addr = None - remote_port = None - ssl_env = None - rbufsize = DEFAULT_BUFFER_SIZE - wbufsize = DEFAULT_BUFFER_SIZE - RequestHandlerClass = HTTPRequest - - def __init__(self, server, sock, makefile=CP_fileobject): - self.server = server - self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) - self.requests_seen = 0 - - def communicate(self): - """Read each request and respond appropriately.""" - request_seen = False - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return - - request_seen = True - req.respond() - if req.close_connection: - return - except socket.error, e: - errnum = e.args[0] - # sadly SSL sockets return a different (longer) time out string - if errnum == 'timed out' or errnum == 'The read operation timed out': - # Don't error if we're between requests; only error - # if 1) no request has been started at all, or 2) we're - # in the middle of a request. - # See http://www.cherrypy.org/ticket/853 - if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response("408 Request Timeout") - except FatalSSLAlert: - # Close the connection. - return - elif errnum not in socket_errors_to_ignore: - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error", - format_exc()) - except FatalSSLAlert: - # Close the connection. - return - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - self.wfile = CP_fileobject(self.socket._sock, "wb", self.wbufsize) - req.simple_response("400 Bad Request", - "The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - self.linger = True - except Exception: - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error", format_exc()) - except FatalSSLAlert: - # Close the connection. - return - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel socket - # when you call socket.close(). We do so manually here because we - # want this server to send a FIN TCP segment immediately. Note this - # must be called *before* calling socket.close(), because the latter - # drops its reference to the kernel socket. - if hasattr(self.socket, '_sock'): - self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -_SHUTDOWNREQUEST = None - -class WorkerThread(threading.Thread): - """Thread which continuously polls a Queue for Connection objects. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - """The current connection pulled off the Queue, or None.""" - - server = None - """The HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it.""" - - ready = False - """A simple flag for the calling server to know when this thread - has begun polling the Queue.""" - - - def __init__(self, server): - self.ready = False - self.server = server - - self.requests_seen = 0 - self.bytes_read = 0 - self.bytes_written = 0 - self.start_time = None - self.work_time = 0 - self.stats = { - 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and 0 or self.conn.requests_seen), - 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and 0 or self.conn.rfile.bytes_read), - 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and 0 or self.conn.wfile.bytes_written), - 'Work Time': lambda s: self.work_time + ((self.start_time is None) and 0 or time.time() - self.start_time), - 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), - } - threading.Thread.__init__(self) - - def run(self): - self.server.stats['Worker Threads'][self.getName()] = self.stats - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - if self.server.stats['Enabled']: - self.start_time = time.time() - try: - conn.communicate() - finally: - conn.close() - if self.server.stats['Enabled']: - self.requests_seen += self.conn.requests_seen - self.bytes_read += self.conn.rfile.bytes_read - self.bytes_written += self.conn.wfile.bytes_written - self.work_time += time.time() - self.start_time - self.start_time = None - self.conn = None - except (KeyboardInterrupt, SystemExit), exc: - self.server.interrupt = exc - - -class ThreadPool(object): - """A Request Queue for the CherryPyWSGIServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = Queue.Queue() - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP Server " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - for i in range(amount): - if self.max > 0 and len(self._threads) >= self.max: - break - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - self._threads.append(worker) - worker.start() - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - if amount > 0: - for i in range(min(amount, len(self._threads) - self.min)): - # Put a number of shutdown requests on the queue equal - # to 'amount'. Once each of those is processed by a worker, - # that worker will terminate and be culled from our list - # in self.put. - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See http://www.cherrypy.org/ticket/691. - KeyboardInterrupt), exc1: - pass - - def _get_qsize(self): - return self._queue.qsize() - qsize = property(_get_qsize) - - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() +if sys.version_info < (3, 0): + from wsgiserver2 import * else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class SSLAdapter(object): - """Base class for SSL driver library adapters. - - Required methods: - - * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` - """ - - def __init__(self, certificate, private_key, certificate_chain=None): - self.certificate = certificate - self.private_key = private_key - self.certificate_chain = certificate_chain - - def wrap(self, sock): - raise NotImplemented - - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented - - -class HTTPServer(object): - """An HTTP server.""" - - _bind_addr = "127.0.0.1" - _interrupt = None - - gateway = None - """A Gateway instance.""" - - minthreads = None - """The minimum number of worker threads to create (default 10).""" - - maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit).""" - - server_name = None - """The name of the server; defaults to socket.gethostname().""" - - protocol = "HTTP/1.1" - """The version string to write in the Status-Line of all HTTP responses. - - For example, "HTTP/1.1" is the default. This also limits the supported - features used in the response.""" - - request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" - - shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit.""" - - timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - version = "CherryPy/3.2.0" - """A version string for the HTTPServer.""" - - software = None - """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. - - If None, this defaults to ``'%s Server' % self.version``.""" - - ready = False - """An internal flag which marks whether the socket is accepting connections.""" - - max_request_header_size = 0 - """The maximum size, in bytes, for request headers, or 0 for no limit.""" - - max_request_body_size = 0 - """The maximum size, in bytes, for request bodies, or 0 for no limit.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - ConnectionClass = HTTPConnection - """The class to use for handling HTTP connections.""" - - ssl_adapter = None - """An instance of SSLAdapter (or a subclass). - - You must have the corresponding SSL driver library installed.""" - - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, - server_name=None): - self.bind_addr = bind_addr - self.gateway = gateway - - self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) - - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.clear_stats() - - def clear_stats(self): - self._start_time = None - self._run_time = 0 - self.stats = { - 'Enabled': False, - 'Bind Address': lambda s: repr(self.bind_addr), - 'Run time': lambda s: (not s['Enabled']) and 0 or self.runtime(), - 'Accepts': 0, - 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), - 'Queue': lambda s: getattr(self.requests, "qsize", None), - 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), - 'Threads Idle': lambda s: getattr(self.requests, "idle", None), - 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and 0 or sum([w['Requests'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and 0 or sum([w['Bytes Read'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and 0 or sum([w['Bytes Written'](w) for w - in s['Worker Threads'].values()], 0), - 'Work Time': lambda s: (not s['Enabled']) and 0 or sum([w['Work Time'](w) for w - in s['Worker Threads'].values()], 0), - 'Read Throughput': lambda s: (not s['Enabled']) and 0 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Write Throughput': lambda s: (not s['Enabled']) and 0 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Worker Threads': {}, - } - logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats - - def runtime(self): - if self._start_time is None: - return self._run_time - else: - return self._run_time + (time.time() - self._start_time) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property(_get_bind_addr, _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - if self.software is None: - self.software = "%s Server" % self.version - - # SSL backward compatibility - if (self.ssl_adapter is None and - getattr(self, 'ssl_certificate', None) and - getattr(self, 'ssl_private_key', None)): - warnings.warn( - "SSL attributes are deprecated in CherryPy 3.2, and will " - "be removed in CherryPy 3.3. Use an ssl_adapter attribute " - "instead.", - DeprecationWarning - ) - try: - from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter - except ImportError: - pass - else: - self.ssl_adapter = pyOpenSSLAdapter( - self.ssl_certificate, self.ssl_private_key, - getattr(self, 'ssl_certificate_chain', None)) - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: os.unlink(self.bind_addr) - except: pass - - # So everyone can access the socket... - try: os.chmod(self.bind_addr, 0777) - except: pass - - info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) - except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, "", self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error: - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error(msg) - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - self._start_time = time.time() - while self.ready: - self.tick() - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See http://www.cherrypy.org/ticket/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): - try: - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return - - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - makefile = CP_fileobject - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except NoSSLError: - msg = ("The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - buf = ["%s 400 Bad Request\r\n" % self.protocol, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n\r\n", - msg] - - wfile = CP_fileobject(s, "wb", DEFAULT_BUFFER_SIZE) - try: - wfile.sendall("".join(buf)) - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - raise - return - if not s: - return - makefile = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, makefile) - - if not isinstance(self.bind_addr, basestring): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env - - self.requests.put(conn) - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error, x: - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See http://www.cherrypy.org/ticket/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See http://www.cherrypy.org/ticket/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See http://www.cherrypy.org/ticket/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - if self._start_time is not None: - self._run_time += (time.time() - self._start_time) - self._start_time = None - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error, x: - if x.args[0] not in socket_errors_to_ignore: - # Changed to use error code and not message - # See http://www.cherrypy.org/ticket/860. - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - -class Gateway(object): - - def __init__(self, req): - self.req = req - - def respond(self): - raise NotImplemented - - -# These may either be wsgiserver.SSLAdapter subclasses or the string names -# of such classes (in which case they will be lazily loaded). -ssl_adapters = { - 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', - } - -def get_ssl_adapter_class(name='pyopenssl'): - adapter = ssl_adapters[name.lower()] - if isinstance(adapter, basestring): - last_dot = adapter.rfind(".") - attr_name = adapter[last_dot + 1:] - mod_path = adapter[:last_dot] - - try: - mod = sys.modules[mod_path] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(mod_path, globals(), locals(), ['']) - - # Let an AttributeError propagate outward. - try: - adapter = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - return adapter - -# -------------------------------- WSGI Stuff -------------------------------- # - - -class CherryPyWSGIServer(HTTPServer): - - wsgi_version = (1, 0) - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): - self.requests = ThreadPool(self, min=numthreads or 1, max=max) - self.wsgi_app = wsgi_app - self.gateway = wsgi_gateways[self.wsgi_version] - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - self.clear_stats() - - def _get_numthreads(self): - return self.requests.min - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - -class WSGIGateway(Gateway): - - def __init__(self, req): - self.req = req - self.started_response = False - self.env = self.get_environ() - self.remaining_bytes_out = None - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented - - def respond(self): - response = self.req.server.wsgi_app(self.env, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - if isinstance(chunk, unicode): - chunk = chunk.encode('ISO-8859-1') - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - def start_response(self, status, headers, exc_info = None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - self.started_response = True - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.req.sent_headers: - try: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None - - self.req.status = status - for k, v in headers: - if not isinstance(k, str): - raise TypeError("WSGI response header key %r is not a byte string." % k) - if not isinstance(v, str): - raise TypeError("WSGI response header value %r is not a byte string." % v) - if k.lower() == 'content-length': - self.remaining_bytes_out = int(v) - self.req.outheaders.extend(headers) - - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - chunklen = len(chunk) - rbo = self.remaining_bytes_out - if rbo is not None and chunklen > rbo: - if not self.req.sent_headers: - # Whew. We can send a 500 to the client. - self.req.simple_response("500 Internal Server Error", - "The requested resource returned more bytes than the " - "declared Content-Length.") - else: - # Dang. We have probably already sent data. Truncate the chunk - # to fit (so the client doesn't hang) and raise an error later. - chunk = chunk[:rbo] - - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() - - self.req.write(chunk) - - if rbo is not None: - rbo -= chunklen - if rbo < 0: - raise ValueError( - "Response body exceeds the declared Content-Length.") - - -class WSGIGateway_10(WSGIGateway): - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env = { - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, - 'PATH_INFO': req.path, - 'QUERY_STRING': req.qs, - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), - 'REQUEST_METHOD': req.method, - 'REQUEST_URI': req.uri, - 'SCRIPT_NAME': '', - 'SERVER_NAME': req.server.server_name, - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - 'SERVER_PROTOCOL': req.request_protocol, - 'SERVER_SOFTWARE': req.server.software, - 'wsgi.errors': sys.stderr, - 'wsgi.input': req.rfile, - 'wsgi.multiprocess': False, - 'wsgi.multithread': True, - 'wsgi.run_once': False, - 'wsgi.url_scheme': req.scheme, - 'wsgi.version': (1, 0), - } - - if isinstance(req.server.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - env["SERVER_PORT"] = "" - else: - env["SERVER_PORT"] = str(req.server.bind_addr[1]) - - # Request headers - for k, v in req.inheaders.iteritems(): - env["HTTP_" + k.upper().replace("-", "_")] = v - - # CONTENT_TYPE/CONTENT_LENGTH - ct = env.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - env["CONTENT_TYPE"] = ct - cl = env.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - env["CONTENT_LENGTH"] = cl - - if req.conn.ssl_env: - env.update(req.conn.ssl_env) - - return env - - -class WSGIGateway_u0(WSGIGateway_10): - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env_10 = WSGIGateway_10.get_environ(self) - env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) - env[u'wsgi.version'] = ('u', 0) - - # Request-URI - env.setdefault(u'wsgi.url_encoding', u'utf-8') - try: - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - except UnicodeDecodeError: - # Fall back to latin 1 so apps can transcode if needed. - env[u'wsgi.url_encoding'] = u'ISO-8859-1' - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - - for k, v in sorted(env.items()): - if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'): - env[k] = v.decode('ISO-8859-1') - - return env - -wsgi_gateways = { - (1, 0): WSGIGateway_10, - ('u', 0): WSGIGateway_u0, -} - -class WSGIPathInfoDispatcher(object): - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = apps.items() - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort(cmp=lambda x,y: cmp(len(x[0]), len(y[0]))) - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return [''] - + # Le sigh. Boo for backward-incompatible syntax. + exec('from .wsgiserver3 import *') diff --git a/cherrypy/wsgiserver/ssl_builtin.py b/cherrypy/wsgiserver/ssl_builtin.py index 64c0eeb0..03bf05de 100644 --- a/cherrypy/wsgiserver/ssl_builtin.py +++ b/cherrypy/wsgiserver/ssl_builtin.py @@ -11,6 +11,16 @@ try: except ImportError: ssl = None +try: + from _pyio import DEFAULT_BUFFER_SIZE +except ImportError: + try: + from io import DEFAULT_BUFFER_SIZE + except ImportError: + DEFAULT_BUFFER_SIZE = -1 + +import sys + from cherrypy import wsgiserver @@ -40,7 +50,8 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): s = ssl.wrap_socket(sock, do_handshake_on_connect=True, server_side=True, certfile=self.certificate, keyfile=self.private_key, ssl_version=ssl.PROTOCOL_SSLv23) - except ssl.SSLError, e: + except ssl.SSLError: + e = sys.exc_info()[1] if e.errno == ssl.SSL_ERROR_EOF: # This is almost certainly due to the cherrypy engine # 'pinging' the socket to assert it's connectable; @@ -50,6 +61,10 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): if e.args[1].endswith('http request'): # The client is speaking HTTP to an HTTPS server. raise wsgiserver.NoSSLError + elif e.args[1].endswith('unknown protocol'): + # The client is speaking some non-HTTP protocol. + # Drop the conn. + return None, {} raise return s, self.get_environ(s) @@ -67,6 +82,10 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): } return ssl_environ - def makefile(self, sock, mode='r', bufsize=-1): - return wsgiserver.CP_fileobject(sock, mode, bufsize) + if sys.version_info >= (3, 0): + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + return wsgiserver.CP_makefile(sock, mode, bufsize) + else: + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + return wsgiserver.CP_fileobject(sock, mode, bufsize) diff --git a/cherrypy/wsgiserver/wsgiserver2.py b/cherrypy/wsgiserver/wsgiserver2.py new file mode 100644 index 00000000..b6bd4997 --- /dev/null +++ b/cherrypy/wsgiserver/wsgiserver2.py @@ -0,0 +1,2322 @@ +"""A high-speed, production ready, thread pooled, generic HTTP server. + +Simplest example on how to use this module directly +(without using CherryPy's application machinery):: + + from cherrypy import wsgiserver + + def my_crazy_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type','text/plain')] + start_response(status, response_headers) + return ['Hello world!'] + + server = wsgiserver.CherryPyWSGIServer( + ('0.0.0.0', 8070), my_crazy_app, + server_name='www.cherrypy.example') + server.start() + +The CherryPy WSGI server can serve as many WSGI applications +as you want in one instance by using a WSGIPathInfoDispatcher:: + + d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) + server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) + +Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. + +This won't call the CherryPy engine (application side) at all, only the +HTTP server, which is independent from the rest of CherryPy. Don't +let the name "CherryPyWSGIServer" throw you; the name merely reflects +its origin, not its coupling. + +For those of you wanting to understand internals of this module, here's the +basic call flow. The server's listening thread runs a very tight loop, +sticking incoming connections onto a Queue:: + + server = CherryPyWSGIServer(...) + server.start() + while True: + tick() + # This blocks until a request comes in: + child = socket.accept() + conn = HTTPConnection(child, ...) + server.requests.put(conn) + +Worker threads are kept in a pool and poll the Queue, popping off and then +handling each connection in turn. Each connection can consist of an arbitrary +number of requests and their responses, so we run a nested loop:: + + while True: + conn = server.requests.get() + conn.communicate() + -> while True: + req = HTTPRequest(...) + req.parse_request() + -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" + req.rfile.readline() + read_headers(req.rfile, req.inheaders) + req.respond() + -> response = app(...) + try: + for chunk in response: + if chunk: + req.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + if req.close_connection: + return +""" + +__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', + 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', + 'CP_fileobject', + 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', + 'WorkerThread', 'ThreadPool', 'SSLAdapter', + 'CherryPyWSGIServer', + 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', + 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class'] + +import os +try: + import queue +except: + import Queue as queue +import re +import rfc822 +import socket +import sys +if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 +try: + import cStringIO as StringIO +except ImportError: + import StringIO +DEFAULT_BUFFER_SIZE = -1 + +_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) + +import threading +import time +import traceback +def format_exc(limit=None): + """Like print_exc() but return a string. Backport for Python 2.3.""" + try: + etype, value, tb = sys.exc_info() + return ''.join(traceback.format_exception(etype, value, tb, limit)) + finally: + etype = value = tb = None + + +from urllib import unquote +from urlparse import urlparse +import warnings + +if sys.version_info >= (3, 0): + bytestr = bytes + unicodestr = str + basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 3, the native string type is unicode + return n.encode(encoding) +else: + bytestr = str + unicodestr = unicode + basestring = basestring + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 2, the native string type is bytes. Assume it's already + # in the given encoding, which for ISO-8859-1 is almost always what + # was intended. + return n + +LF = ntob('\n') +CRLF = ntob('\r\n') +TAB = ntob('\t') +SPACE = ntob(' ') +COLON = ntob(':') +SEMICOLON = ntob(';') +EMPTY = ntob('') +NUMBER_SIGN = ntob('#') +QUESTION_MARK = ntob('?') +ASTERISK = ntob('*') +FORWARD_SLASH = ntob('/') +quoted_slash = re.compile(ntob("(?i)%2F")) + +import errno + +def plat_specific_errors(*errnames): + """Return error numbers for all errors in errnames on this platform. + + The 'errno' module contains different global constants depending on + the specific platform (OS). This function will return the list of + numeric values for a given list of potential names. + """ + errno_names = dir(errno) + nums = [getattr(errno, k) for k in errnames if k in errno_names] + # de-dupe the list + return list(dict.fromkeys(nums).keys()) + +socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") + +socket_errors_to_ignore = plat_specific_errors( + "EPIPE", + "EBADF", "WSAEBADF", + "ENOTSOCK", "WSAENOTSOCK", + "ETIMEDOUT", "WSAETIMEDOUT", + "ECONNREFUSED", "WSAECONNREFUSED", + "ECONNRESET", "WSAECONNRESET", + "ECONNABORTED", "WSAECONNABORTED", + "ENETRESET", "WSAENETRESET", + "EHOSTDOWN", "EHOSTUNREACH", + ) +socket_errors_to_ignore.append("timed out") +socket_errors_to_ignore.append("The read operation timed out") + +socket_errors_nonblocking = plat_specific_errors( + 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') + +comma_separated_headers = [ntob(h) for h in + ['Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', + 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', + 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', + 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', + 'WWW-Authenticate']] + + +import logging +if not hasattr(logging, 'statistics'): logging.statistics = {} + + +def read_headers(rfile, hdict=None): + """Read headers from the given stream into the given header dict. + + If hdict is None, a new header dict is created. Returns the populated + header dict. + + Headers which are repeated are folded together using a comma if their + specification so dictates. + + This function raises ValueError when the read bytes violate the HTTP spec. + You should probably return "400 Bad Request" if this happens. + """ + if hdict is None: + hdict = {} + + while True: + line = rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + if line[0] in (SPACE, TAB): + # It's a continuation line. + v = line.strip() + else: + try: + k, v = line.split(COLON, 1) + except ValueError: + raise ValueError("Illegal header line.") + # TODO: what about TE and WWW-Authenticate? + k = k.strip().title() + v = v.strip() + hname = k + + if k in comma_separated_headers: + existing = hdict.get(hname) + if existing: + v = ", ".join((existing, v)) + hdict[hname] = v + + return hdict + + +class MaxSizeExceeded(Exception): + pass + +class SizeCheckWrapper(object): + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" + + def __init__(self, rfile, maxlen): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + + def _check_length(self): + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded() + + def read(self, size=None): + data = self.rfile.read(size) + self.bytes_read += len(data) + self._check_length() + return data + + def readline(self, size=None): + if size is not None: + data = self.rfile.readline(size) + self.bytes_read += len(data) + self._check_length() + return data + + # User didn't specify a size ... + # We read the line in chunks to make sure it's not a 100MB line ! + res = [] + while True: + data = self.rfile.readline(256) + self.bytes_read += len(data) + self._check_length() + res.append(data) + # See http://www.cherrypy.org/ticket/421 + if len(data) < 256 or data[-1:] == "\n": + return EMPTY.join(res) + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline() + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline() + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.bytes_read += len(data) + self._check_length() + return data + + def next(self): + data = self.rfile.next() + self.bytes_read += len(data) + self._check_length() + return data + + +class KnownLengthRFile(object): + """Wraps a file-like object, returning an empty string when exhausted.""" + + def __init__(self, rfile, content_length): + self.rfile = rfile + self.remaining = content_length + + def read(self, size=None): + if self.remaining == 0: + return '' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.read(size) + self.remaining -= len(data) + return data + + def readline(self, size=None): + if self.remaining == 0: + return '' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.readline(size) + self.remaining -= len(data) + return data + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.remaining -= len(data) + return data + + +class ChunkedRFile(object): + """Wraps a file-like object, returning an empty string when exhausted. + + This class is intended to provide a conforming wsgi.input value for + request entities that have been encoded with the 'chunked' transfer + encoding. + """ + + def __init__(self, rfile, maxlen, bufsize=8192): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + self.buffer = EMPTY + self.bufsize = bufsize + self.closed = False + + def _fetch(self): + if self.closed: + return + + line = self.rfile.readline() + self.bytes_read += len(line) + + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) + + line = line.strip().split(SEMICOLON, 1) + + try: + chunk_size = line.pop(0) + chunk_size = int(chunk_size, 16) + except ValueError: + raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) + + if chunk_size <= 0: + self.closed = True + return + +## if line: chunk_extension = line[0] + + if self.maxlen and self.bytes_read + chunk_size > self.maxlen: + raise IOError("Request Entity Too Large") + + chunk = self.rfile.read(chunk_size) + self.bytes_read += len(chunk) + self.buffer += chunk + + crlf = self.rfile.read(2) + if crlf != CRLF: + raise ValueError( + "Bad chunked transfer coding (expected '\\r\\n', " + "got " + repr(crlf) + ")") + + def read(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + if size: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + data += self.buffer + + def readline(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + newline_pos = self.buffer.find(LF) + if size: + if newline_pos == -1: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + remaining = min(size - len(data), newline_pos) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + if newline_pos == -1: + data += self.buffer + else: + data += self.buffer[:newline_pos] + self.buffer = self.buffer[newline_pos:] + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def read_trailer_lines(self): + if not self.closed: + raise ValueError( + "Cannot read trailers until the request body has been read.") + + while True: + line = self.rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + self.bytes_read += len(line) + if self.maxlen and self.bytes_read > self.maxlen: + raise IOError("Request Entity Too Large") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + yield line + + def close(self): + self.rfile.close() + + def __iter__(self): + # Shamelessly stolen from StringIO + total = 0 + line = self.readline(sizehint) + while line: + yield line + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + + +class HTTPRequest(object): + """An HTTP Request (and response). + + A single HTTP connection may consist of multiple request/response pairs. + """ + + server = None + """The HTTPServer object which is receiving this request.""" + + conn = None + """The HTTPConnection object on which this request connected.""" + + inheaders = {} + """A dict of request headers.""" + + outheaders = [] + """A list of header tuples to write in the response.""" + + ready = False + """When True, the request has been parsed and is ready to begin generating + the response. When False, signals the calling Connection that the response + should not be generated and the connection should close.""" + + close_connection = False + """Signals the calling Connection that the request should close. This does + not imply an error! The client and/or server may each request that the + connection be closed.""" + + chunked_write = False + """If True, output will be encoded with the "chunked" transfer-coding. + + This value is set automatically inside send_headers.""" + + def __init__(self, server, conn): + self.server= server + self.conn = conn + + self.ready = False + self.started_request = False + self.scheme = ntob("http") + if self.server.ssl_adapter is not None: + self.scheme = ntob("https") + # Use the lowest-common protocol in case read_request_line errors. + self.response_protocol = 'HTTP/1.0' + self.inheaders = {} + + self.status = "" + self.outheaders = [] + self.sent_headers = False + self.close_connection = self.__class__.close_connection + self.chunked_read = False + self.chunked_write = self.__class__.chunked_write + + def parse_request(self): + """Parse the next HTTP request start-line and message-headers.""" + self.rfile = SizeCheckWrapper(self.conn.rfile, + self.server.max_request_header_size) + try: + success = self.read_request_line() + except MaxSizeExceeded: + self.simple_response("414 Request-URI Too Long", + "The Request-URI sent with the request exceeds the maximum " + "allowed bytes.") + return + else: + if not success: + return + + try: + success = self.read_request_headers() + except MaxSizeExceeded: + self.simple_response("413 Request Entity Too Large", + "The headers sent with the request exceed the maximum " + "allowed bytes.") + return + else: + if not success: + return + + self.ready = True + + def read_request_line(self): + # HTTP/1.1 connections are persistent by default. If a client + # requests a page, then idles (leaves the connection open), + # then rfile.readline() will raise socket.error("timed out"). + # Note that it does this based on the value given to settimeout(), + # and doesn't need the client to request or acknowledge the close + # (although your TCP stack might suffer for it: cf Apache's history + # with FIN_WAIT_2). + request_line = self.rfile.readline() + + # Set started_request to True so communicate() knows to send 408 + # from here on out. + self.started_request = True + if not request_line: + return False + + if request_line == CRLF: + # RFC 2616 sec 4.1: "...if the server is reading the protocol + # stream at the beginning of a message and receives a CRLF + # first, it should ignore the CRLF." + # But only ignore one leading line! else we enable a DoS. + request_line = self.rfile.readline() + if not request_line: + return False + + if not request_line.endswith(CRLF): + self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") + return False + + try: + method, uri, req_protocol = request_line.strip().split(SPACE, 2) + rp = int(req_protocol[5]), int(req_protocol[7]) + except (ValueError, IndexError): + self.simple_response("400 Bad Request", "Malformed Request-Line") + return False + + self.uri = uri + self.method = method + + # uri may be an abs_path (including "http://host.domain.tld"); + scheme, authority, path = self.parse_request_uri(uri) + if NUMBER_SIGN in path: + self.simple_response("400 Bad Request", + "Illegal #fragment in Request-URI.") + return False + + if scheme: + self.scheme = scheme + + qs = EMPTY + if QUESTION_MARK in path: + path, qs = path.split(QUESTION_MARK, 1) + + # Unquote the path+params (e.g. "/this%20path" -> "/this path"). + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". + try: + atoms = [unquote(x) for x in quoted_slash.split(path)] + except ValueError: + ex = sys.exc_info()[1] + self.simple_response("400 Bad Request", ex.args[0]) + return False + path = "%2F".join(atoms) + self.path = path + + # Note that, like wsgiref and most other HTTP servers, + # we "% HEX HEX"-unquote the path but not the query string. + self.qs = qs + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + sp = int(self.server.protocol[5]), int(self.server.protocol[7]) + + if sp[0] != rp[0]: + self.simple_response("505 HTTP Version Not Supported") + return False + + self.request_protocol = req_protocol + self.response_protocol = "HTTP/%s.%s" % min(rp, sp) + + return True + + def read_request_headers(self): + """Read self.rfile into self.inheaders. Return success.""" + + # then all the http headers + try: + read_headers(self.rfile, self.inheaders) + except ValueError: + ex = sys.exc_info()[1] + self.simple_response("400 Bad Request", ex.args[0]) + return False + + mrbs = self.server.max_request_body_size + if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return False + + # Persistent connection support + if self.response_protocol == "HTTP/1.1": + # Both server and client are HTTP/1.1 + if self.inheaders.get("Connection", "") == "close": + self.close_connection = True + else: + # Either the server or client (or both) are HTTP/1.0 + if self.inheaders.get("Connection", "") != "Keep-Alive": + self.close_connection = True + + # Transfer-Encoding support + te = None + if self.response_protocol == "HTTP/1.1": + te = self.inheaders.get("Transfer-Encoding") + if te: + te = [x.strip().lower() for x in te.split(",") if x.strip()] + + self.chunked_read = False + + if te: + for enc in te: + if enc == "chunked": + self.chunked_read = True + else: + # Note that, even if we see "chunked", we must reject + # if there is an extension we don't recognize. + self.simple_response("501 Unimplemented") + self.close_connection = True + return False + + # From PEP 333: + # "Servers and gateways that implement HTTP 1.1 must provide + # transparent support for HTTP 1.1's "expect/continue" mechanism. + # This may be done in any of several ways: + # 1. Respond to requests containing an Expect: 100-continue request + # with an immediate "100 Continue" response, and proceed normally. + # 2. Proceed with the request normally, but provide the application + # with a wsgi.input stream that will send the "100 Continue" + # response if/when the application first attempts to read from + # the input stream. The read request must then remain blocked + # until the client responds. + # 3. Wait until the client decides that the server does not support + # expect/continue, and sends the request body on its own. + # (This is suboptimal, and is not recommended.) + # + # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, + # but it seems like it would be a big slowdown for such a rare case. + if self.inheaders.get("Expect", "") == "100-continue": + # Don't use simple_response here, because it emits headers + # we don't want. See http://www.cherrypy.org/ticket/951 + msg = self.server.protocol + " 100 Continue\r\n\r\n" + try: + self.conn.wfile.sendall(msg) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return True + + def parse_request_uri(self, uri): + """Parse a Request-URI into (scheme, authority, path). + + Note that Request-URI's must be one of:: + + Request-URI = "*" | absoluteURI | abs_path | authority + + Therefore, a Request-URI which starts with a double forward-slash + cannot be a "net_path":: + + net_path = "//" authority [ abs_path ] + + Instead, it must be interpreted as an "abs_path" with an empty first + path segment:: + + abs_path = "/" path_segments + path_segments = segment *( "/" segment ) + segment = *pchar *( ";" param ) + param = *pchar + """ + if uri == ASTERISK: + return None, None, uri + + i = uri.find('://') + if i > 0 and QUESTION_MARK not in uri[:i]: + # An absoluteURI. + # If there's a scheme (and it must be http or https), then: + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + scheme, remainder = uri[:i].lower(), uri[i + 3:] + authority, path = remainder.split(FORWARD_SLASH, 1) + path = FORWARD_SLASH + path + return scheme, authority, path + + if uri.startswith(FORWARD_SLASH): + # An abs_path. + return None, None, uri + else: + # An authority. + return None, uri, None + + def respond(self): + """Call the gateway and write its iterable output.""" + mrbs = self.server.max_request_body_size + if self.chunked_read: + self.rfile = ChunkedRFile(self.conn.rfile, mrbs) + else: + cl = int(self.inheaders.get("Content-Length", 0)) + if mrbs and mrbs < cl: + if not self.sent_headers: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return + self.rfile = KnownLengthRFile(self.conn.rfile, cl) + + self.server.gateway(self).respond() + + if (self.ready and not self.sent_headers): + self.sent_headers = True + self.send_headers() + if self.chunked_write: + self.conn.wfile.sendall("0\r\n\r\n") + + def simple_response(self, status, msg=""): + """Write a simple response back to the client.""" + status = str(status) + buf = [self.server.protocol + SPACE + + status + CRLF, + "Content-Length: %s\r\n" % len(msg), + "Content-Type: text/plain\r\n"] + + if status[:3] in ("413", "414"): + # Request Entity Too Large / Request-URI Too Long + self.close_connection = True + if self.response_protocol == 'HTTP/1.1': + # This will not be true for 414, since read_request_line + # usually raises 414 before reading the whole line, and we + # therefore cannot know the proper response_protocol. + buf.append("Connection: close\r\n") + else: + # HTTP/1.0 had no 413/414 status nor Connection header. + # Emit 400 instead and trust the message body is enough. + status = "400 Bad Request" + + buf.append(CRLF) + if msg: + if isinstance(msg, unicodestr): + msg = msg.encode("ISO-8859-1") + buf.append(msg) + + try: + self.conn.wfile.sendall("".join(buf)) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + + def write(self, chunk): + """Write unbuffered data to the client.""" + if self.chunked_write and chunk: + buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF] + self.conn.wfile.sendall(EMPTY.join(buf)) + else: + self.conn.wfile.sendall(chunk) + + def send_headers(self): + """Assert, process, and send the HTTP response message-headers. + + You must set self.status, and self.outheaders before calling this. + """ + hkeys = [key.lower() for key, value in self.outheaders] + status = int(self.status[:3]) + + if status == 413: + # Request Entity Too Large. Close conn to avoid garbage. + self.close_connection = True + elif "content-length" not in hkeys: + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." So no point chunking. + if status < 200 or status in (204, 205, 304): + pass + else: + if (self.response_protocol == 'HTTP/1.1' + and self.method != 'HEAD'): + # Use the chunked transfer-coding + self.chunked_write = True + self.outheaders.append(("Transfer-Encoding", "chunked")) + else: + # Closing the conn is the only way to determine len. + self.close_connection = True + + if "connection" not in hkeys: + if self.response_protocol == 'HTTP/1.1': + # Both server and client are HTTP/1.1 or better + if self.close_connection: + self.outheaders.append(("Connection", "close")) + else: + # Server and/or client are HTTP/1.0 + if not self.close_connection: + self.outheaders.append(("Connection", "Keep-Alive")) + + if (not self.close_connection) and (not self.chunked_read): + # Read any remaining request body data on the socket. + # "If an origin server receives a request that does not include an + # Expect request-header field with the "100-continue" expectation, + # the request includes a request body, and the server responds + # with a final status code before reading the entire request body + # from the transport connection, then the server SHOULD NOT close + # the transport connection until it has read the entire request, + # or until the client closes the connection. Otherwise, the client + # might not reliably receive the response message. However, this + # requirement is not be construed as preventing a server from + # defending itself against denial-of-service attacks, or from + # badly broken client implementations." + remaining = getattr(self.rfile, 'remaining', 0) + if remaining > 0: + self.rfile.read(remaining) + + if "date" not in hkeys: + self.outheaders.append(("Date", rfc822.formatdate())) + + if "server" not in hkeys: + self.outheaders.append(("Server", self.server.server_name)) + + buf = [self.server.protocol + SPACE + self.status + CRLF] + for k, v in self.outheaders: + buf.append(k + COLON + SPACE + v + CRLF) + buf.append(CRLF) + self.conn.wfile.sendall(EMPTY.join(buf)) + + +class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" + pass + + +class FatalSSLAlert(Exception): + """Exception raised when the SSL implementation signals a fatal alert.""" + pass + + +class CP_fileobject(socket._fileobject): + """Faux file object attached to a socket object.""" + + def __init__(self, *args, **kwargs): + self.bytes_read = 0 + self.bytes_written = 0 + socket._fileobject.__init__(self, *args, **kwargs) + + def sendall(self, data): + """Sendall for non-blocking sockets.""" + while data: + try: + bytes_sent = self.send(data) + data = data[bytes_sent:] + except socket.error, e: + if e.args[0] not in socket_errors_nonblocking: + raise + + def send(self, data): + bytes_sent = self._sock.send(data) + self.bytes_written += bytes_sent + return bytes_sent + + def flush(self): + if self._wbuf: + buffer = "".join(self._wbuf) + self._wbuf = [] + self.sendall(buffer) + + def recv(self, size): + while True: + try: + data = self._sock.recv(size) + self.bytes_read += len(data) + return data + except socket.error, e: + if (e.args[0] not in socket_errors_nonblocking + and e.args[0] not in socket_error_eintr): + raise + + if not _fileobject_uses_str_type: + def read(self, size=-1): + # Use max, disallow tiny reads in a loop as they are very inefficient. + # We never leave read() with any leftover data from a new recv() call + # in our internal buffer. + rbufsize = max(self._rbufsize, self.default_bufsize) + # Our use of StringIO rather than lists of string objects returned by + # recv() minimizes memory usage and fragmentation that occurs when + # rbufsize is large compared to the typical return value of recv(). + buf = self._rbuf + buf.seek(0, 2) # seek end + if size < 0: + # Read until EOF + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(rbufsize) + if not data: + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or EOF seen, whichever comes first + buf_len = buf.tell() + if buf_len >= size: + # Already have size bytes in our buffer? Extract and return. + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return rv + + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + left = size - buf_len + # recv() will malloc the amount of memory given as its + # parameter even though it often returns much less data + # than that. The returned data string is short lived + # as we copy it into a StringIO and free it. This avoids + # fragmentation issues on many platforms. + data = self.recv(left) + if not data: + break + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid buffer data copies when: + # - We have no data in our buffer. + # AND + # - Our call to recv returned exactly the + # number of bytes we were asked to read. + return data + if n == left: + buf.write(data) + del data # explicit free + break + assert n <= left, "recv(%d) returned %d bytes" % (left, n) + buf.write(data) + buf_len += n + del data # explicit free + #assert buf_len == buf.tell() + return buf.getvalue() + + def readline(self, size=-1): + buf = self._rbuf + buf.seek(0, 2) # seek end + if buf.tell() > 0: + # check if we already have it in our buffer + buf.seek(0) + bline = buf.readline(size) + if bline.endswith('\n') or len(bline) == size: + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return bline + del bline + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + buf.seek(0) + buffers = [buf.read()] + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + data = None + recv = self.recv + while data != "\n": + data = recv(1) + if not data: + break + buffers.append(data) + return "".join(buffers) + + buf.seek(0, 2) # seek end + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(self._rbufsize) + if not data: + break + nl = data.find('\n') + if nl >= 0: + nl += 1 + buf.write(data[:nl]) + self._rbuf.write(data[nl:]) + del data + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or \n or EOF seen, whichever comes first + buf.seek(0, 2) # seek end + buf_len = buf.tell() + if buf_len >= size: + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO.StringIO() + self._rbuf.write(buf.read()) + return rv + self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + while True: + data = self.recv(self._rbufsize) + if not data: + break + left = size - buf_len + # did we just receive a newline? + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + # save the excess data to _rbuf + self._rbuf.write(data[nl:]) + if buf_len: + buf.write(data[:nl]) + break + else: + # Shortcut. Avoid data copy through buf when returning + # a substring of our first recv(). + return data[:nl] + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid data copy through buf when + # returning exactly all of our first recv(). + return data + if n >= left: + buf.write(data[:left]) + self._rbuf.write(data[left:]) + break + buf.write(data) + buf_len += n + #assert buf_len == buf.tell() + return buf.getvalue() + else: + def read(self, size=-1): + if size < 0: + # Read until EOF + buffers = [self._rbuf] + self._rbuf = "" + if self._rbufsize <= 1: + recv_size = self.default_bufsize + else: + recv_size = self._rbufsize + + while True: + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + return "".join(buffers) + else: + # Read until size bytes or EOF seen, whichever comes first + data = self._rbuf + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + left = size - buf_len + recv_size = max(self._rbufsize, left) + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return "".join(buffers) + + def readline(self, size=-1): + data = self._rbuf + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + assert data == "" + buffers = [] + while data != "\n": + data = self.recv(1) + if not data: + break + buffers.append(data) + return "".join(buffers) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + return "".join(buffers) + else: + # Read until size bytes or \n or EOF seen, whichever comes first + nl = data.find('\n', 0, size) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = "" + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + left = size - buf_len + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return "".join(buffers) + + +class HTTPConnection(object): + """An HTTP connection (active socket). + + server: the Server object which received this connection. + socket: the raw socket object (usually TCP) for this connection. + makefile: a fileobject class for reading from the socket. + """ + + remote_addr = None + remote_port = None + ssl_env = None + rbufsize = DEFAULT_BUFFER_SIZE + wbufsize = DEFAULT_BUFFER_SIZE + RequestHandlerClass = HTTPRequest + + def __init__(self, server, sock, makefile=CP_fileobject): + self.server = server + self.socket = sock + self.rfile = makefile(sock, "rb", self.rbufsize) + self.wfile = makefile(sock, "wb", self.wbufsize) + self.requests_seen = 0 + + def communicate(self): + """Read each request and respond appropriately.""" + request_seen = False + try: + while True: + # (re)set req to None so that if something goes wrong in + # the RequestHandlerClass constructor, the error doesn't + # get written to the previous request. + req = None + req = self.RequestHandlerClass(self.server, self) + + # This order of operations should guarantee correct pipelining. + req.parse_request() + if self.server.stats['Enabled']: + self.requests_seen += 1 + if not req.ready: + # Something went wrong in the parsing (and the server has + # probably already made a simple_response). Return and + # let the conn close. + return + + request_seen = True + req.respond() + if req.close_connection: + return + except socket.error: + e = sys.exc_info()[1] + errnum = e.args[0] + # sadly SSL sockets return a different (longer) time out string + if errnum == 'timed out' or errnum == 'The read operation timed out': + # Don't error if we're between requests; only error + # if 1) no request has been started at all, or 2) we're + # in the middle of a request. + # See http://www.cherrypy.org/ticket/853 + if (not request_seen) or (req and req.started_request): + # Don't bother writing the 408 if the response + # has already started being written. + if req and not req.sent_headers: + try: + req.simple_response("408 Request Timeout") + except FatalSSLAlert: + # Close the connection. + return + elif errnum not in socket_errors_to_ignore: + self.server.error_log("socket.error %s" % repr(errnum), + level=logging.WARNING, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error") + except FatalSSLAlert: + # Close the connection. + return + return + except (KeyboardInterrupt, SystemExit): + raise + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + if req and not req.sent_headers: + # Unwrap our wfile + self.wfile = CP_fileobject(self.socket._sock, "wb", self.wbufsize) + req.simple_response("400 Bad Request", + "The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + self.linger = True + except Exception: + e = sys.exc_info()[1] + self.server.error_log(repr(e), level=logging.ERROR, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error") + except FatalSSLAlert: + # Close the connection. + return + + linger = False + + def close(self): + """Close the socket underlying this connection.""" + self.rfile.close() + + if not self.linger: + # Python's socket module does NOT call close on the kernel socket + # when you call socket.close(). We do so manually here because we + # want this server to send a FIN TCP segment immediately. Note this + # must be called *before* calling socket.close(), because the latter + # drops its reference to the kernel socket. + if hasattr(self.socket, '_sock'): + self.socket._sock.close() + self.socket.close() + else: + # On the other hand, sometimes we want to hang around for a bit + # to make sure the client has a chance to read our entire + # response. Skipping the close() calls here delays the FIN + # packet until the socket object is garbage-collected later. + # Someday, perhaps, we'll do the full lingering_close that + # Apache does, but not today. + pass + + +class TrueyZero(object): + """An object which equals and does math like the integer '0' but evals True.""" + def __add__(self, other): + return other + def __radd__(self, other): + return other +trueyzero = TrueyZero() + + +_SHUTDOWNREQUEST = None + +class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. + + Due to the timing issues of polling a Queue, a WorkerThread does not + check its own 'ready' flag after it has started. To stop the thread, + it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue + (one for each running WorkerThread). + """ + + conn = None + """The current connection pulled off the Queue, or None.""" + + server = None + """The HTTP Server which spawned this thread, and which owns the + Queue and is placing active connections into it.""" + + ready = False + """A simple flag for the calling server to know when this thread + has begun polling the Queue.""" + + + def __init__(self, server): + self.ready = False + self.server = server + + self.requests_seen = 0 + self.bytes_read = 0 + self.bytes_written = 0 + self.start_time = None + self.work_time = 0 + self.stats = { + 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and trueyzero or self.conn.requests_seen), + 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and trueyzero or self.conn.rfile.bytes_read), + 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and trueyzero or self.conn.wfile.bytes_written), + 'Work Time': lambda s: self.work_time + ((self.start_time is None) and trueyzero or time.time() - self.start_time), + 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), + } + threading.Thread.__init__(self) + + def run(self): + self.server.stats['Worker Threads'][self.getName()] = self.stats + try: + self.ready = True + while True: + conn = self.server.requests.get() + if conn is _SHUTDOWNREQUEST: + return + + self.conn = conn + if self.server.stats['Enabled']: + self.start_time = time.time() + try: + conn.communicate() + finally: + conn.close() + if self.server.stats['Enabled']: + self.requests_seen += self.conn.requests_seen + self.bytes_read += self.conn.rfile.bytes_read + self.bytes_written += self.conn.wfile.bytes_written + self.work_time += time.time() - self.start_time + self.start_time = None + self.conn = None + except (KeyboardInterrupt, SystemExit): + exc = sys.exc_info()[1] + self.server.interrupt = exc + + +class ThreadPool(object): + """A Request Queue for an HTTPServer which pools threads. + + ThreadPool objects must provide min, get(), put(obj), start() + and stop(timeout) attributes. + """ + + def __init__(self, server, min=10, max=-1): + self.server = server + self.min = min + self.max = max + self._threads = [] + self._queue = queue.Queue() + self.get = self._queue.get + + def start(self): + """Start the pool of threads.""" + for i in range(self.min): + self._threads.append(WorkerThread(self.server)) + for worker in self._threads: + worker.setName("CP Server " + worker.getName()) + worker.start() + for worker in self._threads: + while not worker.ready: + time.sleep(.1) + + def _get_idle(self): + """Number of worker threads which are idle. Read-only.""" + return len([t for t in self._threads if t.conn is None]) + idle = property(_get_idle, doc=_get_idle.__doc__) + + def put(self, obj): + self._queue.put(obj) + if obj is _SHUTDOWNREQUEST: + return + + def grow(self, amount): + """Spawn new worker threads (not above self.max).""" + for i in range(amount): + if self.max > 0 and len(self._threads) >= self.max: + break + worker = WorkerThread(self.server) + worker.setName("CP Server " + worker.getName()) + self._threads.append(worker) + worker.start() + + def shrink(self, amount): + """Kill off worker threads (not below self.min).""" + # Grow/shrink the pool if necessary. + # Remove any dead threads from our list + for t in self._threads: + if not t.isAlive(): + self._threads.remove(t) + amount -= 1 + + if amount > 0: + for i in range(min(amount, len(self._threads) - self.min)): + # Put a number of shutdown requests on the queue equal + # to 'amount'. Once each of those is processed by a worker, + # that worker will terminate and be culled from our list + # in self.put. + self._queue.put(_SHUTDOWNREQUEST) + + def stop(self, timeout=5): + # Must shut down threads here so the code that calls + # this method can know when all threads are stopped. + for worker in self._threads: + self._queue.put(_SHUTDOWNREQUEST) + + # Don't join currentThread (when stop is called inside a request). + current = threading.currentThread() + if timeout and timeout >= 0: + endtime = time.time() + timeout + while self._threads: + worker = self._threads.pop() + if worker is not current and worker.isAlive(): + try: + if timeout is None or timeout < 0: + worker.join() + else: + remaining_time = endtime - time.time() + if remaining_time > 0: + worker.join(remaining_time) + if worker.isAlive(): + # We exhausted the timeout. + # Forcibly shut down the socket. + c = worker.conn + if c and not c.rfile.closed: + try: + c.socket.shutdown(socket.SHUT_RD) + except TypeError: + # pyOpenSSL sockets don't take an arg + c.socket.shutdown() + worker.join() + except (AssertionError, + # Ignore repeated Ctrl-C. + # See http://www.cherrypy.org/ticket/691. + KeyboardInterrupt): + pass + + def _get_qsize(self): + return self._queue.qsize() + qsize = property(_get_qsize) + + + +try: + import fcntl +except ImportError: + try: + from ctypes import windll, WinError + except ImportError: + def prevent_socket_inheritance(sock): + """Dummy function, since neither fcntl nor ctypes are available.""" + pass + else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (Windows).""" + if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): + raise WinError() +else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (POSIX).""" + fd = sock.fileno() + old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) + + +class SSLAdapter(object): + """Base class for SSL driver library adapters. + + Required methods: + + * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` + """ + + def __init__(self, certificate, private_key, certificate_chain=None): + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + + def wrap(self, sock): + raise NotImplemented + + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + raise NotImplemented + + +class HTTPServer(object): + """An HTTP server.""" + + _bind_addr = "127.0.0.1" + _interrupt = None + + gateway = None + """A Gateway instance.""" + + minthreads = None + """The minimum number of worker threads to create (default 10).""" + + maxthreads = None + """The maximum number of worker threads to create (default -1 = no limit).""" + + server_name = None + """The name of the server; defaults to socket.gethostname().""" + + protocol = "HTTP/1.1" + """The version string to write in the Status-Line of all HTTP responses. + + For example, "HTTP/1.1" is the default. This also limits the supported + features used in the response.""" + + request_queue_size = 5 + """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + + shutdown_timeout = 5 + """The total time, in seconds, to wait for worker threads to cleanly exit.""" + + timeout = 10 + """The timeout in seconds for accepted connections (default 10).""" + + version = "CherryPy/3.2.2" + """A version string for the HTTPServer.""" + + software = None + """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. + + If None, this defaults to ``'%s Server' % self.version``.""" + + ready = False + """An internal flag which marks whether the socket is accepting connections.""" + + max_request_header_size = 0 + """The maximum size, in bytes, for request headers, or 0 for no limit.""" + + max_request_body_size = 0 + """The maximum size, in bytes, for request bodies, or 0 for no limit.""" + + nodelay = True + """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" + + ConnectionClass = HTTPConnection + """The class to use for handling HTTP connections.""" + + ssl_adapter = None + """An instance of SSLAdapter (or a subclass). + + You must have the corresponding SSL driver library installed.""" + + def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, + server_name=None): + self.bind_addr = bind_addr + self.gateway = gateway + + self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) + + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.clear_stats() + + def clear_stats(self): + self._start_time = None + self._run_time = 0 + self.stats = { + 'Enabled': False, + 'Bind Address': lambda s: repr(self.bind_addr), + 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), + 'Accepts': 0, + 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), + 'Queue': lambda s: getattr(self.requests, "qsize", None), + 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), + 'Threads Idle': lambda s: getattr(self.requests, "idle", None), + 'Socket Errors': 0, + 'Requests': lambda s: (not s['Enabled']) and -1 or sum([w['Requests'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Read'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Written'](w) for w + in s['Worker Threads'].values()], 0), + 'Work Time': lambda s: (not s['Enabled']) and -1 or sum([w['Work Time'](w) for w + in s['Worker Threads'].values()], 0), + 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Worker Threads': {}, + } + logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats + + def runtime(self): + if self._start_time is None: + return self._run_time + else: + return self._run_time + (time.time() - self._start_time) + + def __str__(self): + return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, + self.bind_addr) + + def _get_bind_addr(self): + return self._bind_addr + def _set_bind_addr(self, value): + if isinstance(value, tuple) and value[0] in ('', None): + # Despite the socket module docs, using '' does not + # allow AI_PASSIVE to work. Passing None instead + # returns '0.0.0.0' like we want. In other words: + # host AI_PASSIVE result + # '' Y 192.168.x.y + # '' N 192.168.x.y + # None Y 0.0.0.0 + # None N 127.0.0.1 + # But since you can get the same effect with an explicit + # '0.0.0.0', we deny both the empty string and None as values. + raise ValueError("Host values of '' or None are not allowed. " + "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " + "to listen on all active interfaces.") + self._bind_addr = value + bind_addr = property(_get_bind_addr, _set_bind_addr, + doc="""The interface on which to listen for connections. + + For TCP sockets, a (host, port) tuple. Host values may be any IPv4 + or IPv6 address, or any valid hostname. The string 'localhost' is a + synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). + The string '0.0.0.0' is a special IPv4 entry meaning "any active + interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for + IPv6. The empty string or None are not allowed. + + For UNIX sockets, supply the filename as a string.""") + + def start(self): + """Run the server forever.""" + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrpy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self._interrupt = None + + if self.software is None: + self.software = "%s Server" % self.version + + # SSL backward compatibility + if (self.ssl_adapter is None and + getattr(self, 'ssl_certificate', None) and + getattr(self, 'ssl_private_key', None)): + warnings.warn( + "SSL attributes are deprecated in CherryPy 3.2, and will " + "be removed in CherryPy 3.3. Use an ssl_adapter attribute " + "instead.", + DeprecationWarning + ) + try: + from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter + except ImportError: + pass + else: + self.ssl_adapter = pyOpenSSLAdapter( + self.ssl_certificate, self.ssl_private_key, + getattr(self, 'ssl_certificate_chain', None)) + + # Select the appropriate socket + if isinstance(self.bind_addr, basestring): + # AF_UNIX socket + + # So we can reuse the socket... + try: os.unlink(self.bind_addr) + except: pass + + # So everyone can access the socket... + try: os.chmod(self.bind_addr, 511) # 0777 + except: pass + + info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + else: + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 addresses) + host, port = self.bind_addr + try: + info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + except socket.gaierror: + if ':' in self.bind_addr[0]: + info = [(socket.AF_INET6, socket.SOCK_STREAM, + 0, "", self.bind_addr + (0, 0))] + else: + info = [(socket.AF_INET, socket.SOCK_STREAM, + 0, "", self.bind_addr)] + + self.socket = None + msg = "No socket could be created" + for res in info: + af, socktype, proto, canonname, sa = res + try: + self.bind(af, socktype, proto) + except socket.error: + if self.socket: + self.socket.close() + self.socket = None + continue + break + if not self.socket: + raise socket.error(msg) + + # Timeout so KeyboardInterrupt can be caught on Win32 + self.socket.settimeout(1) + self.socket.listen(self.request_queue_size) + + # Create worker threads + self.requests.start() + + self.ready = True + self._start_time = time.time() + while self.ready: + try: + self.tick() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.error_log("Error in HTTPServer.tick", level=logging.ERROR, + traceback=True) + + if self.interrupt: + while self.interrupt is True: + # Wait for self.stop() to complete. See _set_interrupt. + time.sleep(0.1) + if self.interrupt: + raise self.interrupt + + def error_log(self, msg="", level=20, traceback=False): + # Override this in subclasses as desired + sys.stderr.write(msg + '\n') + sys.stderr.flush() + if traceback: + tblines = format_exc() + sys.stderr.write(tblines) + sys.stderr.flush() + + def bind(self, family, type, proto=0): + """Create (or recreate) the actual socket object.""" + self.socket = socket.socket(family, type, proto) + prevent_socket_inheritance(self.socket) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.nodelay and not isinstance(self.bind_addr, str): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if self.ssl_adapter is not None: + self.socket = self.ssl_adapter.bind(self.socket) + + # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), + # activate dual-stack. See http://www.cherrypy.org/ticket/871. + if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + try: + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + except (AttributeError, socket.error): + # Apparently, the socket option is not available in + # this machine's TCP stack + pass + + self.socket.bind(self.bind_addr) + + def tick(self): + """Accept a new connection and put it on the Queue.""" + try: + s, addr = self.socket.accept() + if self.stats['Enabled']: + self.stats['Accepts'] += 1 + if not self.ready: + return + + prevent_socket_inheritance(s) + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + makefile = CP_fileobject + ssl_env = {} + # if ssl cert and key are set, we try to be a secure HTTP server + if self.ssl_adapter is not None: + try: + s, ssl_env = self.ssl_adapter.wrap(s) + except NoSSLError: + msg = ("The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + buf = ["%s 400 Bad Request\r\n" % self.protocol, + "Content-Length: %s\r\n" % len(msg), + "Content-Type: text/plain\r\n\r\n", + msg] + + wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) + try: + wfile.sendall("".join(buf)) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return + if not s: + return + makefile = self.ssl_adapter.makefile + # Re-apply our timeout since we may have a new socket object + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + conn = self.ConnectionClass(self, s, makefile) + + if not isinstance(self.bind_addr, basestring): + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + if addr is None: # sometimes this can happen + # figure out if AF_INET or AF_INET6. + if len(s.getsockname()) == 2: + # AF_INET + addr = ('0.0.0.0', 0) + else: + # AF_INET6 + addr = ('::', 0) + conn.remote_addr = addr[0] + conn.remote_port = addr[1] + + conn.ssl_env = ssl_env + + self.requests.put(conn) + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error: + x = sys.exc_info()[1] + if self.stats['Enabled']: + self.stats['Socket Errors'] += 1 + if x.args[0] in socket_error_eintr: + # I *think* this is right. EINTR should occur when a signal + # is received during the accept() call; all docs say retry + # the call, and I *think* I'm reading it right that Python + # will then go ahead and poll for and handle the signal + # elsewhere. See http://www.cherrypy.org/ticket/707. + return + if x.args[0] in socket_errors_nonblocking: + # Just try again. See http://www.cherrypy.org/ticket/479. + return + if x.args[0] in socket_errors_to_ignore: + # Our socket was closed. + # See http://www.cherrypy.org/ticket/686. + return + raise + + def _get_interrupt(self): + return self._interrupt + def _set_interrupt(self, interrupt): + self._interrupt = True + self.stop() + self._interrupt = interrupt + interrupt = property(_get_interrupt, _set_interrupt, + doc="Set this to an Exception instance to " + "interrupt the server.") + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + self.ready = False + if self._start_time is not None: + self._run_time += (time.time() - self._start_time) + self._start_time = None + + sock = getattr(self, "socket", None) + if sock: + if not isinstance(self.bind_addr, basestring): + # Touch our own socket to make accept() return immediately. + try: + host, port = sock.getsockname()[:2] + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + # Changed to use error code and not message + # See http://www.cherrypy.org/ticket/860. + raise + else: + # Note that we're explicitly NOT using AI_PASSIVE, + # here, because we want an actual IP to touch. + # localhost won't work if we've bound to a public IP, + # but it will if we bound to '0.0.0.0' (INADDR_ANY). + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + if hasattr(sock, "close"): + sock.close() + self.socket = None + + self.requests.stop(self.shutdown_timeout) + + +class Gateway(object): + """A base class to interface HTTPServer with other systems, such as WSGI.""" + + def __init__(self, req): + self.req = req + + def respond(self): + """Process the current request. Must be overridden in a subclass.""" + raise NotImplemented + + +# These may either be wsgiserver.SSLAdapter subclasses or the string names +# of such classes (in which case they will be lazily loaded). +ssl_adapters = { + 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', + 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', + } + +def get_ssl_adapter_class(name='pyopenssl'): + """Return an SSL adapter class for the given name.""" + adapter = ssl_adapters[name.lower()] + if isinstance(adapter, basestring): + last_dot = adapter.rfind(".") + attr_name = adapter[last_dot + 1:] + mod_path = adapter[:last_dot] + + try: + mod = sys.modules[mod_path] + if mod is None: + raise KeyError() + except KeyError: + # The last [''] is important. + mod = __import__(mod_path, globals(), locals(), ['']) + + # Let an AttributeError propagate outward. + try: + adapter = getattr(mod, attr_name) + except AttributeError: + raise AttributeError("'%s' object has no attribute '%s'" + % (mod_path, attr_name)) + + return adapter + +# -------------------------------- WSGI Stuff -------------------------------- # + + +class CherryPyWSGIServer(HTTPServer): + """A subclass of HTTPServer which calls a WSGI application.""" + + wsgi_version = (1, 0) + """The version of WSGI to produce.""" + + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): + self.requests = ThreadPool(self, min=numthreads or 1, max=max) + self.wsgi_app = wsgi_app + self.gateway = wsgi_gateways[self.wsgi_version] + + self.bind_addr = bind_addr + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.request_queue_size = request_queue_size + + self.timeout = timeout + self.shutdown_timeout = shutdown_timeout + self.clear_stats() + + def _get_numthreads(self): + return self.requests.min + def _set_numthreads(self, value): + self.requests.min = value + numthreads = property(_get_numthreads, _set_numthreads) + + +class WSGIGateway(Gateway): + """A base class to interface HTTPServer with WSGI.""" + + def __init__(self, req): + self.req = req + self.started_response = False + self.env = self.get_environ() + self.remaining_bytes_out = None + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + raise NotImplemented + + def respond(self): + """Process the current request.""" + response = self.req.server.wsgi_app(self.env, self.start_response) + try: + for chunk in response: + # "The start_response callable must not actually transmit + # the response headers. Instead, it must store them for the + # server or gateway to transmit only after the first + # iteration of the application return value that yields + # a NON-EMPTY string, or upon the application's first + # invocation of the write() callable." (PEP 333) + if chunk: + if isinstance(chunk, unicodestr): + chunk = chunk.encode('ISO-8859-1') + self.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + + def start_response(self, status, headers, exc_info = None): + """WSGI callable to begin the HTTP response.""" + # "The application may call start_response more than once, + # if and only if the exc_info argument is provided." + if self.started_response and not exc_info: + raise AssertionError("WSGI start_response called a second " + "time with no exc_info.") + self.started_response = True + + # "if exc_info is provided, and the HTTP headers have already been + # sent, start_response must raise an error, and should raise the + # exc_info tuple." + if self.req.sent_headers: + try: + raise exc_info[0], exc_info[1], exc_info[2] + finally: + exc_info = None + + self.req.status = status + for k, v in headers: + if not isinstance(k, str): + raise TypeError("WSGI response header key %r is not of type str." % k) + if not isinstance(v, str): + raise TypeError("WSGI response header value %r is not of type str." % v) + if k.lower() == 'content-length': + self.remaining_bytes_out = int(v) + self.req.outheaders.extend(headers) + + return self.write + + def write(self, chunk): + """WSGI callable to write unbuffered data to the client. + + This method is also used internally by start_response (to write + data from the iterable returned by the WSGI application). + """ + if not self.started_response: + raise AssertionError("WSGI write called before start_response.") + + chunklen = len(chunk) + rbo = self.remaining_bytes_out + if rbo is not None and chunklen > rbo: + if not self.req.sent_headers: + # Whew. We can send a 500 to the client. + self.req.simple_response("500 Internal Server Error", + "The requested resource returned more bytes than the " + "declared Content-Length.") + else: + # Dang. We have probably already sent data. Truncate the chunk + # to fit (so the client doesn't hang) and raise an error later. + chunk = chunk[:rbo] + + if not self.req.sent_headers: + self.req.sent_headers = True + self.req.send_headers() + + self.req.write(chunk) + + if rbo is not None: + rbo -= chunklen + if rbo < 0: + raise ValueError( + "Response body exceeds the declared Content-Length.") + + +class WSGIGateway_10(WSGIGateway): + """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env = { + # set a non-standard environ entry so the WSGI app can know what + # the *real* server protocol is (and what features to support). + # See http://www.faqs.org/rfcs/rfc2145.html. + 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, + 'PATH_INFO': req.path, + 'QUERY_STRING': req.qs, + 'REMOTE_ADDR': req.conn.remote_addr or '', + 'REMOTE_PORT': str(req.conn.remote_port or ''), + 'REQUEST_METHOD': req.method, + 'REQUEST_URI': req.uri, + 'SCRIPT_NAME': '', + 'SERVER_NAME': req.server.server_name, + # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. + 'SERVER_PROTOCOL': req.request_protocol, + 'SERVER_SOFTWARE': req.server.software, + 'wsgi.errors': sys.stderr, + 'wsgi.input': req.rfile, + 'wsgi.multiprocess': False, + 'wsgi.multithread': True, + 'wsgi.run_once': False, + 'wsgi.url_scheme': req.scheme, + 'wsgi.version': (1, 0), + } + + if isinstance(req.server.bind_addr, basestring): + # AF_UNIX. This isn't really allowed by WSGI, which doesn't + # address unix domain sockets. But it's better than nothing. + env["SERVER_PORT"] = "" + else: + env["SERVER_PORT"] = str(req.server.bind_addr[1]) + + # Request headers + for k, v in req.inheaders.iteritems(): + env["HTTP_" + k.upper().replace("-", "_")] = v + + # CONTENT_TYPE/CONTENT_LENGTH + ct = env.pop("HTTP_CONTENT_TYPE", None) + if ct is not None: + env["CONTENT_TYPE"] = ct + cl = env.pop("HTTP_CONTENT_LENGTH", None) + if cl is not None: + env["CONTENT_LENGTH"] = cl + + if req.conn.ssl_env: + env.update(req.conn.ssl_env) + + return env + + +class WSGIGateway_u0(WSGIGateway_10): + """A Gateway class to interface HTTPServer with WSGI u.0. + + WSGI u.0 is an experimental protocol, which uses unicode for keys and values + in both Python 2 and Python 3. + """ + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env_10 = WSGIGateway_10.get_environ(self) + env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) + env[u'wsgi.version'] = ('u', 0) + + # Request-URI + env.setdefault(u'wsgi.url_encoding', u'utf-8') + try: + for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: + env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) + except UnicodeDecodeError: + # Fall back to latin 1 so apps can transcode if needed. + env[u'wsgi.url_encoding'] = u'ISO-8859-1' + for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: + env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) + + for k, v in sorted(env.items()): + if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'): + env[k] = v.decode('ISO-8859-1') + + return env + +wsgi_gateways = { + (1, 0): WSGIGateway_10, + ('u', 0): WSGIGateway_u0, +} + +class WSGIPathInfoDispatcher(object): + """A WSGI dispatcher for dispatch based on the PATH_INFO. + + apps: a dict or list of (path_prefix, app) pairs. + """ + + def __init__(self, apps): + try: + apps = list(apps.items()) + except AttributeError: + pass + + # Sort the apps by len(path), descending + apps.sort(cmp=lambda x,y: cmp(len(x[0]), len(y[0]))) + apps.reverse() + + # The path_prefix strings must start, but not end, with a slash. + # Use "" instead of "/". + self.apps = [(p.rstrip("/"), a) for p, a in apps] + + def __call__(self, environ, start_response): + path = environ["PATH_INFO"] or "/" + for p, app in self.apps: + # The apps list should be sorted by length, descending. + if path.startswith(p + "/") or path == p: + environ = environ.copy() + environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p + environ["PATH_INFO"] = path[len(p):] + return app(environ, start_response) + + start_response('404 Not Found', [('Content-Type', 'text/plain'), + ('Content-Length', '0')]) + return [''] + diff --git a/cherrypy/wsgiserver/wsgiserver3.py b/cherrypy/wsgiserver/wsgiserver3.py new file mode 100644 index 00000000..62db5ffd --- /dev/null +++ b/cherrypy/wsgiserver/wsgiserver3.py @@ -0,0 +1,2040 @@ +"""A high-speed, production ready, thread pooled, generic HTTP server. + +Simplest example on how to use this module directly +(without using CherryPy's application machinery):: + + from cherrypy import wsgiserver + + def my_crazy_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type','text/plain')] + start_response(status, response_headers) + return ['Hello world!'] + + server = wsgiserver.CherryPyWSGIServer( + ('0.0.0.0', 8070), my_crazy_app, + server_name='www.cherrypy.example') + server.start() + +The CherryPy WSGI server can serve as many WSGI applications +as you want in one instance by using a WSGIPathInfoDispatcher:: + + d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) + server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) + +Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. + +This won't call the CherryPy engine (application side) at all, only the +HTTP server, which is independent from the rest of CherryPy. Don't +let the name "CherryPyWSGIServer" throw you; the name merely reflects +its origin, not its coupling. + +For those of you wanting to understand internals of this module, here's the +basic call flow. The server's listening thread runs a very tight loop, +sticking incoming connections onto a Queue:: + + server = CherryPyWSGIServer(...) + server.start() + while True: + tick() + # This blocks until a request comes in: + child = socket.accept() + conn = HTTPConnection(child, ...) + server.requests.put(conn) + +Worker threads are kept in a pool and poll the Queue, popping off and then +handling each connection in turn. Each connection can consist of an arbitrary +number of requests and their responses, so we run a nested loop:: + + while True: + conn = server.requests.get() + conn.communicate() + -> while True: + req = HTTPRequest(...) + req.parse_request() + -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" + req.rfile.readline() + read_headers(req.rfile, req.inheaders) + req.respond() + -> response = app(...) + try: + for chunk in response: + if chunk: + req.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + if req.close_connection: + return +""" + +__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', + 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', + 'CP_makefile', + 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', + 'WorkerThread', 'ThreadPool', 'SSLAdapter', + 'CherryPyWSGIServer', + 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', + 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class'] + +import os +try: + import queue +except: + import Queue as queue +import re +import email.utils +import socket +import sys +if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 +if sys.version_info < (3,1): + import io +else: + import _pyio as io +DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE + +import threading +import time +from traceback import format_exc +from urllib.parse import unquote +from urllib.parse import urlparse +from urllib.parse import scheme_chars +import warnings + +if sys.version_info >= (3, 0): + bytestr = bytes + unicodestr = str + basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 3, the native string type is unicode + return n.encode(encoding) +else: + bytestr = str + unicodestr = unicode + basestring = basestring + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given encoding.""" + # In Python 2, the native string type is bytes. Assume it's already + # in the given encoding, which for ISO-8859-1 is almost always what + # was intended. + return n + +LF = ntob('\n') +CRLF = ntob('\r\n') +TAB = ntob('\t') +SPACE = ntob(' ') +COLON = ntob(':') +SEMICOLON = ntob(';') +EMPTY = ntob('') +NUMBER_SIGN = ntob('#') +QUESTION_MARK = ntob('?') +ASTERISK = ntob('*') +FORWARD_SLASH = ntob('/') +quoted_slash = re.compile(ntob("(?i)%2F")) + +import errno + +def plat_specific_errors(*errnames): + """Return error numbers for all errors in errnames on this platform. + + The 'errno' module contains different global constants depending on + the specific platform (OS). This function will return the list of + numeric values for a given list of potential names. + """ + errno_names = dir(errno) + nums = [getattr(errno, k) for k in errnames if k in errno_names] + # de-dupe the list + return list(dict.fromkeys(nums).keys()) + +socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") + +socket_errors_to_ignore = plat_specific_errors( + "EPIPE", + "EBADF", "WSAEBADF", + "ENOTSOCK", "WSAENOTSOCK", + "ETIMEDOUT", "WSAETIMEDOUT", + "ECONNREFUSED", "WSAECONNREFUSED", + "ECONNRESET", "WSAECONNRESET", + "ECONNABORTED", "WSAECONNABORTED", + "ENETRESET", "WSAENETRESET", + "EHOSTDOWN", "EHOSTUNREACH", + ) +socket_errors_to_ignore.append("timed out") +socket_errors_to_ignore.append("The read operation timed out") + +socket_errors_nonblocking = plat_specific_errors( + 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') + +comma_separated_headers = [ntob(h) for h in + ['Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', + 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', + 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', + 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', + 'WWW-Authenticate']] + + +import logging +if not hasattr(logging, 'statistics'): logging.statistics = {} + + +def read_headers(rfile, hdict=None): + """Read headers from the given stream into the given header dict. + + If hdict is None, a new header dict is created. Returns the populated + header dict. + + Headers which are repeated are folded together using a comma if their + specification so dictates. + + This function raises ValueError when the read bytes violate the HTTP spec. + You should probably return "400 Bad Request" if this happens. + """ + if hdict is None: + hdict = {} + + while True: + line = rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + if line[0] in (SPACE, TAB): + # It's a continuation line. + v = line.strip() + else: + try: + k, v = line.split(COLON, 1) + except ValueError: + raise ValueError("Illegal header line.") + # TODO: what about TE and WWW-Authenticate? + k = k.strip().title() + v = v.strip() + hname = k + + if k in comma_separated_headers: + existing = hdict.get(hname) + if existing: + v = b", ".join((existing, v)) + hdict[hname] = v + + return hdict + + +class MaxSizeExceeded(Exception): + pass + +class SizeCheckWrapper(object): + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" + + def __init__(self, rfile, maxlen): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + + def _check_length(self): + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded() + + def read(self, size=None): + data = self.rfile.read(size) + self.bytes_read += len(data) + self._check_length() + return data + + def readline(self, size=None): + if size is not None: + data = self.rfile.readline(size) + self.bytes_read += len(data) + self._check_length() + return data + + # User didn't specify a size ... + # We read the line in chunks to make sure it's not a 100MB line ! + res = [] + while True: + data = self.rfile.readline(256) + self.bytes_read += len(data) + self._check_length() + res.append(data) + # See http://www.cherrypy.org/ticket/421 + if len(data) < 256 or data[-1:] == "\n": + return EMPTY.join(res) + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline() + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline() + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.bytes_read += len(data) + self._check_length() + return data + + def next(self): + data = self.rfile.next() + self.bytes_read += len(data) + self._check_length() + return data + + +class KnownLengthRFile(object): + """Wraps a file-like object, returning an empty string when exhausted.""" + + def __init__(self, rfile, content_length): + self.rfile = rfile + self.remaining = content_length + + def read(self, size=None): + if self.remaining == 0: + return b'' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.read(size) + self.remaining -= len(data) + return data + + def readline(self, size=None): + if self.remaining == 0: + return b'' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.readline(size) + self.remaining -= len(data) + return data + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.remaining -= len(data) + return data + + +class ChunkedRFile(object): + """Wraps a file-like object, returning an empty string when exhausted. + + This class is intended to provide a conforming wsgi.input value for + request entities that have been encoded with the 'chunked' transfer + encoding. + """ + + def __init__(self, rfile, maxlen, bufsize=8192): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + self.buffer = EMPTY + self.bufsize = bufsize + self.closed = False + + def _fetch(self): + if self.closed: + return + + line = self.rfile.readline() + self.bytes_read += len(line) + + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) + + line = line.strip().split(SEMICOLON, 1) + + try: + chunk_size = line.pop(0) + chunk_size = int(chunk_size, 16) + except ValueError: + raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) + + if chunk_size <= 0: + self.closed = True + return + +## if line: chunk_extension = line[0] + + if self.maxlen and self.bytes_read + chunk_size > self.maxlen: + raise IOError("Request Entity Too Large") + + chunk = self.rfile.read(chunk_size) + self.bytes_read += len(chunk) + self.buffer += chunk + + crlf = self.rfile.read(2) + if crlf != CRLF: + raise ValueError( + "Bad chunked transfer coding (expected '\\r\\n', " + "got " + repr(crlf) + ")") + + def read(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + if size: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + data += self.buffer + + def readline(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + newline_pos = self.buffer.find(LF) + if size: + if newline_pos == -1: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + remaining = min(size - len(data), newline_pos) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + if newline_pos == -1: + data += self.buffer + else: + data += self.buffer[:newline_pos] + self.buffer = self.buffer[newline_pos:] + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def read_trailer_lines(self): + if not self.closed: + raise ValueError( + "Cannot read trailers until the request body has been read.") + + while True: + line = self.rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError("Illegal end of headers.") + + self.bytes_read += len(line) + if self.maxlen and self.bytes_read > self.maxlen: + raise IOError("Request Entity Too Large") + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError("HTTP requires CRLF terminators") + + yield line + + def close(self): + self.rfile.close() + + def __iter__(self): + # Shamelessly stolen from StringIO + total = 0 + line = self.readline(sizehint) + while line: + yield line + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + + +class HTTPRequest(object): + """An HTTP Request (and response). + + A single HTTP connection may consist of multiple request/response pairs. + """ + + server = None + """The HTTPServer object which is receiving this request.""" + + conn = None + """The HTTPConnection object on which this request connected.""" + + inheaders = {} + """A dict of request headers.""" + + outheaders = [] + """A list of header tuples to write in the response.""" + + ready = False + """When True, the request has been parsed and is ready to begin generating + the response. When False, signals the calling Connection that the response + should not be generated and the connection should close.""" + + close_connection = False + """Signals the calling Connection that the request should close. This does + not imply an error! The client and/or server may each request that the + connection be closed.""" + + chunked_write = False + """If True, output will be encoded with the "chunked" transfer-coding. + + This value is set automatically inside send_headers.""" + + def __init__(self, server, conn): + self.server= server + self.conn = conn + + self.ready = False + self.started_request = False + self.scheme = ntob("http") + if self.server.ssl_adapter is not None: + self.scheme = ntob("https") + # Use the lowest-common protocol in case read_request_line errors. + self.response_protocol = 'HTTP/1.0' + self.inheaders = {} + + self.status = "" + self.outheaders = [] + self.sent_headers = False + self.close_connection = self.__class__.close_connection + self.chunked_read = False + self.chunked_write = self.__class__.chunked_write + + def parse_request(self): + """Parse the next HTTP request start-line and message-headers.""" + self.rfile = SizeCheckWrapper(self.conn.rfile, + self.server.max_request_header_size) + try: + success = self.read_request_line() + except MaxSizeExceeded: + self.simple_response("414 Request-URI Too Long", + "The Request-URI sent with the request exceeds the maximum " + "allowed bytes.") + return + else: + if not success: + return + + try: + success = self.read_request_headers() + except MaxSizeExceeded: + self.simple_response("413 Request Entity Too Large", + "The headers sent with the request exceed the maximum " + "allowed bytes.") + return + else: + if not success: + return + + self.ready = True + + def read_request_line(self): + # HTTP/1.1 connections are persistent by default. If a client + # requests a page, then idles (leaves the connection open), + # then rfile.readline() will raise socket.error("timed out"). + # Note that it does this based on the value given to settimeout(), + # and doesn't need the client to request or acknowledge the close + # (although your TCP stack might suffer for it: cf Apache's history + # with FIN_WAIT_2). + request_line = self.rfile.readline() + + # Set started_request to True so communicate() knows to send 408 + # from here on out. + self.started_request = True + if not request_line: + return False + + if request_line == CRLF: + # RFC 2616 sec 4.1: "...if the server is reading the protocol + # stream at the beginning of a message and receives a CRLF + # first, it should ignore the CRLF." + # But only ignore one leading line! else we enable a DoS. + request_line = self.rfile.readline() + if not request_line: + return False + + if not request_line.endswith(CRLF): + self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") + return False + + try: + method, uri, req_protocol = request_line.strip().split(SPACE, 2) + # The [x:y] slicing is necessary for byte strings to avoid getting ord's + rp = int(req_protocol[5:6]), int(req_protocol[7:8]) + except ValueError: + self.simple_response("400 Bad Request", "Malformed Request-Line") + return False + + self.uri = uri + self.method = method + + # uri may be an abs_path (including "http://host.domain.tld"); + scheme, authority, path = self.parse_request_uri(uri) + if NUMBER_SIGN in path: + self.simple_response("400 Bad Request", + "Illegal #fragment in Request-URI.") + return False + + if scheme: + self.scheme = scheme + + qs = EMPTY + if QUESTION_MARK in path: + path, qs = path.split(QUESTION_MARK, 1) + + # Unquote the path+params (e.g. "/this%20path" -> "/this path"). + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". + try: + atoms = [self.unquote_bytes(x) for x in quoted_slash.split(path)] + except ValueError: + ex = sys.exc_info()[1] + self.simple_response("400 Bad Request", ex.args[0]) + return False + path = b"%2F".join(atoms) + self.path = path + + # Note that, like wsgiref and most other HTTP servers, + # we "% HEX HEX"-unquote the path but not the query string. + self.qs = qs + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + # The [x:y] slicing is necessary for byte strings to avoid getting ord's + sp = int(self.server.protocol[5:6]), int(self.server.protocol[7:8]) + + if sp[0] != rp[0]: + self.simple_response("505 HTTP Version Not Supported") + return False + + self.request_protocol = req_protocol + self.response_protocol = "HTTP/%s.%s" % min(rp, sp) + return True + + def read_request_headers(self): + """Read self.rfile into self.inheaders. Return success.""" + + # then all the http headers + try: + read_headers(self.rfile, self.inheaders) + except ValueError: + ex = sys.exc_info()[1] + self.simple_response("400 Bad Request", ex.args[0]) + return False + + mrbs = self.server.max_request_body_size + if mrbs and int(self.inheaders.get(b"Content-Length", 0)) > mrbs: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return False + + # Persistent connection support + if self.response_protocol == "HTTP/1.1": + # Both server and client are HTTP/1.1 + if self.inheaders.get(b"Connection", b"") == b"close": + self.close_connection = True + else: + # Either the server or client (or both) are HTTP/1.0 + if self.inheaders.get(b"Connection", b"") != b"Keep-Alive": + self.close_connection = True + + # Transfer-Encoding support + te = None + if self.response_protocol == "HTTP/1.1": + te = self.inheaders.get(b"Transfer-Encoding") + if te: + te = [x.strip().lower() for x in te.split(b",") if x.strip()] + + self.chunked_read = False + + if te: + for enc in te: + if enc == b"chunked": + self.chunked_read = True + else: + # Note that, even if we see "chunked", we must reject + # if there is an extension we don't recognize. + self.simple_response("501 Unimplemented") + self.close_connection = True + return False + + # From PEP 333: + # "Servers and gateways that implement HTTP 1.1 must provide + # transparent support for HTTP 1.1's "expect/continue" mechanism. + # This may be done in any of several ways: + # 1. Respond to requests containing an Expect: 100-continue request + # with an immediate "100 Continue" response, and proceed normally. + # 2. Proceed with the request normally, but provide the application + # with a wsgi.input stream that will send the "100 Continue" + # response if/when the application first attempts to read from + # the input stream. The read request must then remain blocked + # until the client responds. + # 3. Wait until the client decides that the server does not support + # expect/continue, and sends the request body on its own. + # (This is suboptimal, and is not recommended.) + # + # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, + # but it seems like it would be a big slowdown for such a rare case. + if self.inheaders.get(b"Expect", b"") == b"100-continue": + # Don't use simple_response here, because it emits headers + # we don't want. See http://www.cherrypy.org/ticket/951 + msg = self.server.protocol.encode('ascii') + b" 100 Continue\r\n\r\n" + try: + self.conn.wfile.write(msg) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return True + + def parse_request_uri(self, uri): + """Parse a Request-URI into (scheme, authority, path). + + Note that Request-URI's must be one of:: + + Request-URI = "*" | absoluteURI | abs_path | authority + + Therefore, a Request-URI which starts with a double forward-slash + cannot be a "net_path":: + + net_path = "//" authority [ abs_path ] + + Instead, it must be interpreted as an "abs_path" with an empty first + path segment:: + + abs_path = "/" path_segments + path_segments = segment *( "/" segment ) + segment = *pchar *( ";" param ) + param = *pchar + """ + if uri == ASTERISK: + return None, None, uri + + scheme, sep, remainder = uri.partition(b'://') + if sep and QUESTION_MARK not in scheme: + # An absoluteURI. + # If there's a scheme (and it must be http or https), then: + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + authority, path_a, path_b = remainder.partition(FORWARD_SLASH) + return scheme.lower(), authority, path_a+path_b + + if uri.startswith(FORWARD_SLASH): + # An abs_path. + return None, None, uri + else: + # An authority. + return None, uri, None + + def unquote_bytes(self, path): + """takes quoted string and unquotes % encoded values""" + res = path.split(b'%') + + for i in range(1, len(res)): + item = res[i] + try: + res[i] = bytes([int(item[:2], 16)]) + item[2:] + except ValueError: + raise + return b''.join(res) + + def respond(self): + """Call the gateway and write its iterable output.""" + mrbs = self.server.max_request_body_size + if self.chunked_read: + self.rfile = ChunkedRFile(self.conn.rfile, mrbs) + else: + cl = int(self.inheaders.get(b"Content-Length", 0)) + if mrbs and mrbs < cl: + if not self.sent_headers: + self.simple_response("413 Request Entity Too Large", + "The entity sent with the request exceeds the maximum " + "allowed bytes.") + return + self.rfile = KnownLengthRFile(self.conn.rfile, cl) + + self.server.gateway(self).respond() + + if (self.ready and not self.sent_headers): + self.sent_headers = True + self.send_headers() + if self.chunked_write: + self.conn.wfile.write(b"0\r\n\r\n") + + def simple_response(self, status, msg=""): + """Write a simple response back to the client.""" + status = str(status) + buf = [bytes(self.server.protocol, "ascii") + SPACE + + bytes(status, "ISO-8859-1") + CRLF, + bytes("Content-Length: %s\r\n" % len(msg), "ISO-8859-1"), + b"Content-Type: text/plain\r\n"] + + if status[:3] in ("413", "414"): + # Request Entity Too Large / Request-URI Too Long + self.close_connection = True + if self.response_protocol == 'HTTP/1.1': + # This will not be true for 414, since read_request_line + # usually raises 414 before reading the whole line, and we + # therefore cannot know the proper response_protocol. + buf.append(b"Connection: close\r\n") + else: + # HTTP/1.0 had no 413/414 status nor Connection header. + # Emit 400 instead and trust the message body is enough. + status = "400 Bad Request" + + buf.append(CRLF) + if msg: + if isinstance(msg, unicodestr): + msg = msg.encode("ISO-8859-1") + buf.append(msg) + + try: + self.conn.wfile.write(b"".join(buf)) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + + def write(self, chunk): + """Write unbuffered data to the client.""" + if self.chunked_write and chunk: + buf = [bytes(hex(len(chunk)), 'ASCII')[2:], CRLF, chunk, CRLF] + self.conn.wfile.write(EMPTY.join(buf)) + else: + self.conn.wfile.write(chunk) + + def send_headers(self): + """Assert, process, and send the HTTP response message-headers. + + You must set self.status, and self.outheaders before calling this. + """ + hkeys = [key.lower() for key, value in self.outheaders] + status = int(self.status[:3]) + + if status == 413: + # Request Entity Too Large. Close conn to avoid garbage. + self.close_connection = True + elif b"content-length" not in hkeys: + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." So no point chunking. + if status < 200 or status in (204, 205, 304): + pass + else: + if (self.response_protocol == 'HTTP/1.1' + and self.method != b'HEAD'): + # Use the chunked transfer-coding + self.chunked_write = True + self.outheaders.append((b"Transfer-Encoding", b"chunked")) + else: + # Closing the conn is the only way to determine len. + self.close_connection = True + + if b"connection" not in hkeys: + if self.response_protocol == 'HTTP/1.1': + # Both server and client are HTTP/1.1 or better + if self.close_connection: + self.outheaders.append((b"Connection", b"close")) + else: + # Server and/or client are HTTP/1.0 + if not self.close_connection: + self.outheaders.append((b"Connection", b"Keep-Alive")) + + if (not self.close_connection) and (not self.chunked_read): + # Read any remaining request body data on the socket. + # "If an origin server receives a request that does not include an + # Expect request-header field with the "100-continue" expectation, + # the request includes a request body, and the server responds + # with a final status code before reading the entire request body + # from the transport connection, then the server SHOULD NOT close + # the transport connection until it has read the entire request, + # or until the client closes the connection. Otherwise, the client + # might not reliably receive the response message. However, this + # requirement is not be construed as preventing a server from + # defending itself against denial-of-service attacks, or from + # badly broken client implementations." + remaining = getattr(self.rfile, 'remaining', 0) + if remaining > 0: + self.rfile.read(remaining) + + if b"date" not in hkeys: + self.outheaders.append( + (b"Date", email.utils.formatdate(usegmt=True).encode('ISO-8859-1'))) + + if b"server" not in hkeys: + self.outheaders.append( + (b"Server", self.server.server_name.encode('ISO-8859-1'))) + + buf = [self.server.protocol.encode('ascii') + SPACE + self.status + CRLF] + for k, v in self.outheaders: + buf.append(k + COLON + SPACE + v + CRLF) + buf.append(CRLF) + self.conn.wfile.write(EMPTY.join(buf)) + + +class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" + pass + + +class FatalSSLAlert(Exception): + """Exception raised when the SSL implementation signals a fatal alert.""" + pass + + +class CP_BufferedWriter(io.BufferedWriter): + """Faux file object attached to a socket object.""" + + def write(self, b): + self._checkClosed() + if isinstance(b, str): + raise TypeError("can't write str to binary stream") + + with self._write_lock: + self._write_buf.extend(b) + self._flush_unlocked() + return len(b) + + def _flush_unlocked(self): + self._checkClosed("flush of closed file") + while self._write_buf: + try: + # ssl sockets only except 'bytes', not bytearrays + # so perhaps we should conditionally wrap this for perf? + n = self.raw.write(bytes(self._write_buf)) + except io.BlockingIOError as e: + n = e.characters_written + del self._write_buf[:n] + + +def CP_makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + if 'r' in mode: + return io.BufferedReader(socket.SocketIO(sock, mode), bufsize) + else: + return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize) + +class HTTPConnection(object): + """An HTTP connection (active socket). + + server: the Server object which received this connection. + socket: the raw socket object (usually TCP) for this connection. + makefile: a fileobject class for reading from the socket. + """ + + remote_addr = None + remote_port = None + ssl_env = None + rbufsize = DEFAULT_BUFFER_SIZE + wbufsize = DEFAULT_BUFFER_SIZE + RequestHandlerClass = HTTPRequest + + def __init__(self, server, sock, makefile=CP_makefile): + self.server = server + self.socket = sock + self.rfile = makefile(sock, "rb", self.rbufsize) + self.wfile = makefile(sock, "wb", self.wbufsize) + self.requests_seen = 0 + + def communicate(self): + """Read each request and respond appropriately.""" + request_seen = False + try: + while True: + # (re)set req to None so that if something goes wrong in + # the RequestHandlerClass constructor, the error doesn't + # get written to the previous request. + req = None + req = self.RequestHandlerClass(self.server, self) + + # This order of operations should guarantee correct pipelining. + req.parse_request() + if self.server.stats['Enabled']: + self.requests_seen += 1 + if not req.ready: + # Something went wrong in the parsing (and the server has + # probably already made a simple_response). Return and + # let the conn close. + return + + request_seen = True + req.respond() + if req.close_connection: + return + except socket.error: + e = sys.exc_info()[1] + errnum = e.args[0] + # sadly SSL sockets return a different (longer) time out string + if errnum == 'timed out' or errnum == 'The read operation timed out': + # Don't error if we're between requests; only error + # if 1) no request has been started at all, or 2) we're + # in the middle of a request. + # See http://www.cherrypy.org/ticket/853 + if (not request_seen) or (req and req.started_request): + # Don't bother writing the 408 if the response + # has already started being written. + if req and not req.sent_headers: + try: + req.simple_response("408 Request Timeout") + except FatalSSLAlert: + # Close the connection. + return + elif errnum not in socket_errors_to_ignore: + self.server.error_log("socket.error %s" % repr(errnum), + level=logging.WARNING, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error") + except FatalSSLAlert: + # Close the connection. + return + return + except (KeyboardInterrupt, SystemExit): + raise + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + if req and not req.sent_headers: + # Unwrap our wfile + self.wfile = CP_makefile(self.socket._sock, "wb", self.wbufsize) + req.simple_response("400 Bad Request", + "The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + self.linger = True + except Exception: + e = sys.exc_info()[1] + self.server.error_log(repr(e), level=logging.ERROR, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response("500 Internal Server Error") + except FatalSSLAlert: + # Close the connection. + return + + linger = False + + def close(self): + """Close the socket underlying this connection.""" + self.rfile.close() + + if not self.linger: + # Python's socket module does NOT call close on the kernel socket + # when you call socket.close(). We do so manually here because we + # want this server to send a FIN TCP segment immediately. Note this + # must be called *before* calling socket.close(), because the latter + # drops its reference to the kernel socket. + # Python 3 *probably* fixed this with socket._real_close; hard to tell. +## self.socket._sock.close() + self.socket.close() + else: + # On the other hand, sometimes we want to hang around for a bit + # to make sure the client has a chance to read our entire + # response. Skipping the close() calls here delays the FIN + # packet until the socket object is garbage-collected later. + # Someday, perhaps, we'll do the full lingering_close that + # Apache does, but not today. + pass + + +class TrueyZero(object): + """An object which equals and does math like the integer '0' but evals True.""" + def __add__(self, other): + return other + def __radd__(self, other): + return other +trueyzero = TrueyZero() + + +_SHUTDOWNREQUEST = None + +class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. + + Due to the timing issues of polling a Queue, a WorkerThread does not + check its own 'ready' flag after it has started. To stop the thread, + it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue + (one for each running WorkerThread). + """ + + conn = None + """The current connection pulled off the Queue, or None.""" + + server = None + """The HTTP Server which spawned this thread, and which owns the + Queue and is placing active connections into it.""" + + ready = False + """A simple flag for the calling server to know when this thread + has begun polling the Queue.""" + + + def __init__(self, server): + self.ready = False + self.server = server + + self.requests_seen = 0 + self.bytes_read = 0 + self.bytes_written = 0 + self.start_time = None + self.work_time = 0 + self.stats = { + 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and trueyzero or self.conn.requests_seen), + 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and trueyzero or self.conn.rfile.bytes_read), + 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and trueyzero or self.conn.wfile.bytes_written), + 'Work Time': lambda s: self.work_time + ((self.start_time is None) and trueyzero or time.time() - self.start_time), + 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), + } + threading.Thread.__init__(self) + + def run(self): + self.server.stats['Worker Threads'][self.getName()] = self.stats + try: + self.ready = True + while True: + conn = self.server.requests.get() + if conn is _SHUTDOWNREQUEST: + return + + self.conn = conn + if self.server.stats['Enabled']: + self.start_time = time.time() + try: + conn.communicate() + finally: + conn.close() + if self.server.stats['Enabled']: + self.requests_seen += self.conn.requests_seen + self.bytes_read += self.conn.rfile.bytes_read + self.bytes_written += self.conn.wfile.bytes_written + self.work_time += time.time() - self.start_time + self.start_time = None + self.conn = None + except (KeyboardInterrupt, SystemExit): + exc = sys.exc_info()[1] + self.server.interrupt = exc + + +class ThreadPool(object): + """A Request Queue for an HTTPServer which pools threads. + + ThreadPool objects must provide min, get(), put(obj), start() + and stop(timeout) attributes. + """ + + def __init__(self, server, min=10, max=-1): + self.server = server + self.min = min + self.max = max + self._threads = [] + self._queue = queue.Queue() + self.get = self._queue.get + + def start(self): + """Start the pool of threads.""" + for i in range(self.min): + self._threads.append(WorkerThread(self.server)) + for worker in self._threads: + worker.setName("CP Server " + worker.getName()) + worker.start() + for worker in self._threads: + while not worker.ready: + time.sleep(.1) + + def _get_idle(self): + """Number of worker threads which are idle. Read-only.""" + return len([t for t in self._threads if t.conn is None]) + idle = property(_get_idle, doc=_get_idle.__doc__) + + def put(self, obj): + self._queue.put(obj) + if obj is _SHUTDOWNREQUEST: + return + + def grow(self, amount): + """Spawn new worker threads (not above self.max).""" + for i in range(amount): + if self.max > 0 and len(self._threads) >= self.max: + break + worker = WorkerThread(self.server) + worker.setName("CP Server " + worker.getName()) + self._threads.append(worker) + worker.start() + + def shrink(self, amount): + """Kill off worker threads (not below self.min).""" + # Grow/shrink the pool if necessary. + # Remove any dead threads from our list + for t in self._threads: + if not t.isAlive(): + self._threads.remove(t) + amount -= 1 + + if amount > 0: + for i in range(min(amount, len(self._threads) - self.min)): + # Put a number of shutdown requests on the queue equal + # to 'amount'. Once each of those is processed by a worker, + # that worker will terminate and be culled from our list + # in self.put. + self._queue.put(_SHUTDOWNREQUEST) + + def stop(self, timeout=5): + # Must shut down threads here so the code that calls + # this method can know when all threads are stopped. + for worker in self._threads: + self._queue.put(_SHUTDOWNREQUEST) + + # Don't join currentThread (when stop is called inside a request). + current = threading.currentThread() + if timeout and timeout >= 0: + endtime = time.time() + timeout + while self._threads: + worker = self._threads.pop() + if worker is not current and worker.isAlive(): + try: + if timeout is None or timeout < 0: + worker.join() + else: + remaining_time = endtime - time.time() + if remaining_time > 0: + worker.join(remaining_time) + if worker.isAlive(): + # We exhausted the timeout. + # Forcibly shut down the socket. + c = worker.conn + if c and not c.rfile.closed: + try: + c.socket.shutdown(socket.SHUT_RD) + except TypeError: + # pyOpenSSL sockets don't take an arg + c.socket.shutdown() + worker.join() + except (AssertionError, + # Ignore repeated Ctrl-C. + # See http://www.cherrypy.org/ticket/691. + KeyboardInterrupt): + pass + + def _get_qsize(self): + return self._queue.qsize() + qsize = property(_get_qsize) + + + +try: + import fcntl +except ImportError: + try: + from ctypes import windll, WinError + except ImportError: + def prevent_socket_inheritance(sock): + """Dummy function, since neither fcntl nor ctypes are available.""" + pass + else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (Windows).""" + if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): + raise WinError() +else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (POSIX).""" + fd = sock.fileno() + old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) + + +class SSLAdapter(object): + """Base class for SSL driver library adapters. + + Required methods: + + * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` + """ + + def __init__(self, certificate, private_key, certificate_chain=None): + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + + def wrap(self, sock): + raise NotImplemented + + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + raise NotImplemented + + +class HTTPServer(object): + """An HTTP server.""" + + _bind_addr = "127.0.0.1" + _interrupt = None + + gateway = None + """A Gateway instance.""" + + minthreads = None + """The minimum number of worker threads to create (default 10).""" + + maxthreads = None + """The maximum number of worker threads to create (default -1 = no limit).""" + + server_name = None + """The name of the server; defaults to socket.gethostname().""" + + protocol = "HTTP/1.1" + """The version string to write in the Status-Line of all HTTP responses. + + For example, "HTTP/1.1" is the default. This also limits the supported + features used in the response.""" + + request_queue_size = 5 + """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + + shutdown_timeout = 5 + """The total time, in seconds, to wait for worker threads to cleanly exit.""" + + timeout = 10 + """The timeout in seconds for accepted connections (default 10).""" + + version = "CherryPy/3.2.2" + """A version string for the HTTPServer.""" + + software = None + """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. + + If None, this defaults to ``'%s Server' % self.version``.""" + + ready = False + """An internal flag which marks whether the socket is accepting connections.""" + + max_request_header_size = 0 + """The maximum size, in bytes, for request headers, or 0 for no limit.""" + + max_request_body_size = 0 + """The maximum size, in bytes, for request bodies, or 0 for no limit.""" + + nodelay = True + """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" + + ConnectionClass = HTTPConnection + """The class to use for handling HTTP connections.""" + + ssl_adapter = None + """An instance of SSLAdapter (or a subclass). + + You must have the corresponding SSL driver library installed.""" + + def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, + server_name=None): + self.bind_addr = bind_addr + self.gateway = gateway + + self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) + + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.clear_stats() + + def clear_stats(self): + self._start_time = None + self._run_time = 0 + self.stats = { + 'Enabled': False, + 'Bind Address': lambda s: repr(self.bind_addr), + 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), + 'Accepts': 0, + 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), + 'Queue': lambda s: getattr(self.requests, "qsize", None), + 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), + 'Threads Idle': lambda s: getattr(self.requests, "idle", None), + 'Socket Errors': 0, + 'Requests': lambda s: (not s['Enabled']) and -1 or sum([w['Requests'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Read'](w) for w + in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Written'](w) for w + in s['Worker Threads'].values()], 0), + 'Work Time': lambda s: (not s['Enabled']) and -1 or sum([w['Work Time'](w) for w + in s['Worker Threads'].values()], 0), + 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Worker Threads': {}, + } + logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats + + def runtime(self): + if self._start_time is None: + return self._run_time + else: + return self._run_time + (time.time() - self._start_time) + + def __str__(self): + return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, + self.bind_addr) + + def _get_bind_addr(self): + return self._bind_addr + def _set_bind_addr(self, value): + if isinstance(value, tuple) and value[0] in ('', None): + # Despite the socket module docs, using '' does not + # allow AI_PASSIVE to work. Passing None instead + # returns '0.0.0.0' like we want. In other words: + # host AI_PASSIVE result + # '' Y 192.168.x.y + # '' N 192.168.x.y + # None Y 0.0.0.0 + # None N 127.0.0.1 + # But since you can get the same effect with an explicit + # '0.0.0.0', we deny both the empty string and None as values. + raise ValueError("Host values of '' or None are not allowed. " + "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " + "to listen on all active interfaces.") + self._bind_addr = value + bind_addr = property(_get_bind_addr, _set_bind_addr, + doc="""The interface on which to listen for connections. + + For TCP sockets, a (host, port) tuple. Host values may be any IPv4 + or IPv6 address, or any valid hostname. The string 'localhost' is a + synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). + The string '0.0.0.0' is a special IPv4 entry meaning "any active + interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for + IPv6. The empty string or None are not allowed. + + For UNIX sockets, supply the filename as a string.""") + + def start(self): + """Run the server forever.""" + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrpy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self._interrupt = None + + if self.software is None: + self.software = "%s Server" % self.version + + # Select the appropriate socket + if isinstance(self.bind_addr, basestring): + # AF_UNIX socket + + # So we can reuse the socket... + try: os.unlink(self.bind_addr) + except: pass + + # So everyone can access the socket... + try: os.chmod(self.bind_addr, 511) # 0777 + except: pass + + info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + else: + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 addresses) + host, port = self.bind_addr + try: + info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + except socket.gaierror: + if ':' in self.bind_addr[0]: + info = [(socket.AF_INET6, socket.SOCK_STREAM, + 0, "", self.bind_addr + (0, 0))] + else: + info = [(socket.AF_INET, socket.SOCK_STREAM, + 0, "", self.bind_addr)] + + self.socket = None + msg = "No socket could be created" + for res in info: + af, socktype, proto, canonname, sa = res + try: + self.bind(af, socktype, proto) + except socket.error: + if self.socket: + self.socket.close() + self.socket = None + continue + break + if not self.socket: + raise socket.error(msg) + + # Timeout so KeyboardInterrupt can be caught on Win32 + self.socket.settimeout(1) + self.socket.listen(self.request_queue_size) + + # Create worker threads + self.requests.start() + + self.ready = True + self._start_time = time.time() + while self.ready: + try: + self.tick() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.error_log("Error in HTTPServer.tick", level=logging.ERROR, + traceback=True) + if self.interrupt: + while self.interrupt is True: + # Wait for self.stop() to complete. See _set_interrupt. + time.sleep(0.1) + if self.interrupt: + raise self.interrupt + + def error_log(self, msg="", level=20, traceback=False): + # Override this in subclasses as desired + sys.stderr.write(msg + '\n') + sys.stderr.flush() + if traceback: + tblines = format_exc() + sys.stderr.write(tblines) + sys.stderr.flush() + + def bind(self, family, type, proto=0): + """Create (or recreate) the actual socket object.""" + self.socket = socket.socket(family, type, proto) + prevent_socket_inheritance(self.socket) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.nodelay and not isinstance(self.bind_addr, str): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if self.ssl_adapter is not None: + self.socket = self.ssl_adapter.bind(self.socket) + + # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), + # activate dual-stack. See http://www.cherrypy.org/ticket/871. + if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + try: + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + except (AttributeError, socket.error): + # Apparently, the socket option is not available in + # this machine's TCP stack + pass + + self.socket.bind(self.bind_addr) + + def tick(self): + """Accept a new connection and put it on the Queue.""" + try: + s, addr = self.socket.accept() + if self.stats['Enabled']: + self.stats['Accepts'] += 1 + if not self.ready: + return + + prevent_socket_inheritance(s) + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + makefile = CP_makefile + ssl_env = {} + # if ssl cert and key are set, we try to be a secure HTTP server + if self.ssl_adapter is not None: + try: + s, ssl_env = self.ssl_adapter.wrap(s) + except NoSSLError: + msg = ("The client sent a plain HTTP request, but " + "this server only speaks HTTPS on this port.") + buf = ["%s 400 Bad Request\r\n" % self.protocol, + "Content-Length: %s\r\n" % len(msg), + "Content-Type: text/plain\r\n\r\n", + msg] + + wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) + try: + wfile.write("".join(buf).encode('ISO-8859-1')) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return + if not s: + return + makefile = self.ssl_adapter.makefile + # Re-apply our timeout since we may have a new socket object + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + conn = self.ConnectionClass(self, s, makefile) + + if not isinstance(self.bind_addr, basestring): + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + if addr is None: # sometimes this can happen + # figure out if AF_INET or AF_INET6. + if len(s.getsockname()) == 2: + # AF_INET + addr = ('0.0.0.0', 0) + else: + # AF_INET6 + addr = ('::', 0) + conn.remote_addr = addr[0] + conn.remote_port = addr[1] + + conn.ssl_env = ssl_env + + self.requests.put(conn) + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error: + x = sys.exc_info()[1] + if self.stats['Enabled']: + self.stats['Socket Errors'] += 1 + if x.args[0] in socket_error_eintr: + # I *think* this is right. EINTR should occur when a signal + # is received during the accept() call; all docs say retry + # the call, and I *think* I'm reading it right that Python + # will then go ahead and poll for and handle the signal + # elsewhere. See http://www.cherrypy.org/ticket/707. + return + if x.args[0] in socket_errors_nonblocking: + # Just try again. See http://www.cherrypy.org/ticket/479. + return + if x.args[0] in socket_errors_to_ignore: + # Our socket was closed. + # See http://www.cherrypy.org/ticket/686. + return + raise + + def _get_interrupt(self): + return self._interrupt + def _set_interrupt(self, interrupt): + self._interrupt = True + self.stop() + self._interrupt = interrupt + interrupt = property(_get_interrupt, _set_interrupt, + doc="Set this to an Exception instance to " + "interrupt the server.") + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + self.ready = False + if self._start_time is not None: + self._run_time += (time.time() - self._start_time) + self._start_time = None + + sock = getattr(self, "socket", None) + if sock: + if not isinstance(self.bind_addr, basestring): + # Touch our own socket to make accept() return immediately. + try: + host, port = sock.getsockname()[:2] + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + # Changed to use error code and not message + # See http://www.cherrypy.org/ticket/860. + raise + else: + # Note that we're explicitly NOT using AI_PASSIVE, + # here, because we want an actual IP to touch. + # localhost won't work if we've bound to a public IP, + # but it will if we bound to '0.0.0.0' (INADDR_ANY). + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + if hasattr(sock, "close"): + sock.close() + self.socket = None + + self.requests.stop(self.shutdown_timeout) + + +class Gateway(object): + """A base class to interface HTTPServer with other systems, such as WSGI.""" + + def __init__(self, req): + self.req = req + + def respond(self): + """Process the current request. Must be overridden in a subclass.""" + raise NotImplemented + + +# These may either be wsgiserver.SSLAdapter subclasses or the string names +# of such classes (in which case they will be lazily loaded). +ssl_adapters = { + 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', + } + +def get_ssl_adapter_class(name='builtin'): + """Return an SSL adapter class for the given name.""" + adapter = ssl_adapters[name.lower()] + if isinstance(adapter, basestring): + last_dot = adapter.rfind(".") + attr_name = adapter[last_dot + 1:] + mod_path = adapter[:last_dot] + + try: + mod = sys.modules[mod_path] + if mod is None: + raise KeyError() + except KeyError: + # The last [''] is important. + mod = __import__(mod_path, globals(), locals(), ['']) + + # Let an AttributeError propagate outward. + try: + adapter = getattr(mod, attr_name) + except AttributeError: + raise AttributeError("'%s' object has no attribute '%s'" + % (mod_path, attr_name)) + + return adapter + +# -------------------------------- WSGI Stuff -------------------------------- # + + +class CherryPyWSGIServer(HTTPServer): + """A subclass of HTTPServer which calls a WSGI application.""" + + wsgi_version = (1, 0) + """The version of WSGI to produce.""" + + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): + self.requests = ThreadPool(self, min=numthreads or 1, max=max) + self.wsgi_app = wsgi_app + self.gateway = wsgi_gateways[self.wsgi_version] + + self.bind_addr = bind_addr + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.request_queue_size = request_queue_size + + self.timeout = timeout + self.shutdown_timeout = shutdown_timeout + self.clear_stats() + + def _get_numthreads(self): + return self.requests.min + def _set_numthreads(self, value): + self.requests.min = value + numthreads = property(_get_numthreads, _set_numthreads) + + +class WSGIGateway(Gateway): + """A base class to interface HTTPServer with WSGI.""" + + def __init__(self, req): + self.req = req + self.started_response = False + self.env = self.get_environ() + self.remaining_bytes_out = None + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + raise NotImplemented + + def respond(self): + """Process the current request.""" + response = self.req.server.wsgi_app(self.env, self.start_response) + try: + for chunk in response: + # "The start_response callable must not actually transmit + # the response headers. Instead, it must store them for the + # server or gateway to transmit only after the first + # iteration of the application return value that yields + # a NON-EMPTY string, or upon the application's first + # invocation of the write() callable." (PEP 333) + if chunk: + if isinstance(chunk, unicodestr): + chunk = chunk.encode('ISO-8859-1') + self.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + + def start_response(self, status, headers, exc_info = None): + """WSGI callable to begin the HTTP response.""" + # "The application may call start_response more than once, + # if and only if the exc_info argument is provided." + if self.started_response and not exc_info: + raise AssertionError("WSGI start_response called a second " + "time with no exc_info.") + self.started_response = True + + # "if exc_info is provided, and the HTTP headers have already been + # sent, start_response must raise an error, and should raise the + # exc_info tuple." + if self.req.sent_headers: + try: + raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) + finally: + exc_info = None + + # According to PEP 3333, when using Python 3, the response status + # and headers must be bytes masquerading as unicode; that is, they + # must be of type "str" but are restricted to code points in the + # "latin-1" set. + if not isinstance(status, str): + raise TypeError("WSGI response status is not of type str.") + self.req.status = status.encode('ISO-8859-1') + + for k, v in headers: + if not isinstance(k, str): + raise TypeError("WSGI response header key %r is not of type str." % k) + if not isinstance(v, str): + raise TypeError("WSGI response header value %r is not of type str." % v) + if k.lower() == 'content-length': + self.remaining_bytes_out = int(v) + self.req.outheaders.append((k.encode('ISO-8859-1'), v.encode('ISO-8859-1'))) + + return self.write + + def write(self, chunk): + """WSGI callable to write unbuffered data to the client. + + This method is also used internally by start_response (to write + data from the iterable returned by the WSGI application). + """ + if not self.started_response: + raise AssertionError("WSGI write called before start_response.") + + chunklen = len(chunk) + rbo = self.remaining_bytes_out + if rbo is not None and chunklen > rbo: + if not self.req.sent_headers: + # Whew. We can send a 500 to the client. + self.req.simple_response("500 Internal Server Error", + "The requested resource returned more bytes than the " + "declared Content-Length.") + else: + # Dang. We have probably already sent data. Truncate the chunk + # to fit (so the client doesn't hang) and raise an error later. + chunk = chunk[:rbo] + + if not self.req.sent_headers: + self.req.sent_headers = True + self.req.send_headers() + + self.req.write(chunk) + + if rbo is not None: + rbo -= chunklen + if rbo < 0: + raise ValueError( + "Response body exceeds the declared Content-Length.") + + +class WSGIGateway_10(WSGIGateway): + """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env = { + # set a non-standard environ entry so the WSGI app can know what + # the *real* server protocol is (and what features to support). + # See http://www.faqs.org/rfcs/rfc2145.html. + 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, + 'PATH_INFO': req.path.decode('ISO-8859-1'), + 'QUERY_STRING': req.qs.decode('ISO-8859-1'), + 'REMOTE_ADDR': req.conn.remote_addr or '', + 'REMOTE_PORT': str(req.conn.remote_port or ''), + 'REQUEST_METHOD': req.method.decode('ISO-8859-1'), + 'REQUEST_URI': req.uri, + 'SCRIPT_NAME': '', + 'SERVER_NAME': req.server.server_name, + # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. + 'SERVER_PROTOCOL': req.request_protocol.decode('ISO-8859-1'), + 'SERVER_SOFTWARE': req.server.software, + 'wsgi.errors': sys.stderr, + 'wsgi.input': req.rfile, + 'wsgi.multiprocess': False, + 'wsgi.multithread': True, + 'wsgi.run_once': False, + 'wsgi.url_scheme': req.scheme.decode('ISO-8859-1'), + 'wsgi.version': (1, 0), + } + + if isinstance(req.server.bind_addr, basestring): + # AF_UNIX. This isn't really allowed by WSGI, which doesn't + # address unix domain sockets. But it's better than nothing. + env["SERVER_PORT"] = "" + else: + env["SERVER_PORT"] = str(req.server.bind_addr[1]) + + # Request headers + for k, v in req.inheaders.items(): + k = k.decode('ISO-8859-1').upper().replace("-", "_") + env["HTTP_" + k] = v.decode('ISO-8859-1') + + # CONTENT_TYPE/CONTENT_LENGTH + ct = env.pop("HTTP_CONTENT_TYPE", None) + if ct is not None: + env["CONTENT_TYPE"] = ct + cl = env.pop("HTTP_CONTENT_LENGTH", None) + if cl is not None: + env["CONTENT_LENGTH"] = cl + + if req.conn.ssl_env: + env.update(req.conn.ssl_env) + + return env + + +class WSGIGateway_u0(WSGIGateway_10): + """A Gateway class to interface HTTPServer with WSGI u.0. + + WSGI u.0 is an experimental protocol, which uses unicode for keys and values + in both Python 2 and Python 3. + """ + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env_10 = WSGIGateway_10.get_environ(self) + env = env_10.copy() + env['wsgi.version'] = ('u', 0) + + # Request-URI + env.setdefault('wsgi.url_encoding', 'utf-8') + try: + # SCRIPT_NAME is the empty string, who cares what encoding it is? + env["PATH_INFO"] = req.path.decode(env['wsgi.url_encoding']) + env["QUERY_STRING"] = req.qs.decode(env['wsgi.url_encoding']) + except UnicodeDecodeError: + # Fall back to latin 1 so apps can transcode if needed. + env['wsgi.url_encoding'] = 'ISO-8859-1' + env["PATH_INFO"] = env_10["PATH_INFO"] + env["QUERY_STRING"] = env_10["QUERY_STRING"] + + return env + +wsgi_gateways = { + (1, 0): WSGIGateway_10, + ('u', 0): WSGIGateway_u0, +} + +class WSGIPathInfoDispatcher(object): + """A WSGI dispatcher for dispatch based on the PATH_INFO. + + apps: a dict or list of (path_prefix, app) pairs. + """ + + def __init__(self, apps): + try: + apps = list(apps.items()) + except AttributeError: + pass + + # Sort the apps by len(path), descending + apps.sort() + apps.reverse() + + # The path_prefix strings must start, but not end, with a slash. + # Use "" instead of "/". + self.apps = [(p.rstrip("/"), a) for p, a in apps] + + def __call__(self, environ, start_response): + path = environ["PATH_INFO"] or "/" + for p, app in self.apps: + # The apps list should be sorted by length, descending. + if path.startswith(p + "/") or path == p: + environ = environ.copy() + environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p + environ["PATH_INFO"] = path[len(p):] + return app(environ, start_response) + + start_response('404 Not Found', [('Content-Type', 'text/plain'), + ('Content-Length', '0')]) + return [''] +