diff --git a/mako/__init__.py b/mako/__init__.py index c0f78adc..a16564be 100644 --- a/mako/__init__.py +++ b/mako/__init__.py @@ -1,9 +1,9 @@ # mako/__init__.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -__version__ = '0.4.1' +__version__ = '0.7.2' diff --git a/mako/_ast_util.py b/mako/_ast_util.py index 9521ccbb..a1bd54c4 100644 --- a/mako/_ast_util.py +++ b/mako/_ast_util.py @@ -1,5 +1,5 @@ # mako/_ast_util.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/mako/ast.py b/mako/ast.py index 4365b0b1..76311e9d 100644 --- a/mako/ast.py +++ b/mako/ast.py @@ -1,10 +1,10 @@ # mako/ast.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""utilities for analyzing expressions and blocks of Python +"""utilities for analyzing expressions and blocks of Python code, as well as generating Python from AST nodes""" from mako import exceptions, pyparser, util @@ -14,20 +14,23 @@ class PythonCode(object): """represents information about a string containing Python code""" def __init__(self, code, **exception_kwargs): self.code = code - - # represents all identifiers which are assigned to at some point in the code - self.declared_identifiers = set() - - # represents all identifiers which are referenced before their assignment, if any - self.undeclared_identifiers = set() - - # note that an identifier can be in both the undeclared and declared lists. - # using AST to parse instead of using code.co_varnames, + # represents all identifiers which are assigned to at some point in + # the code + self.declared_identifiers = set() + + # represents all identifiers which are referenced before their + # assignment, if any + self.undeclared_identifiers = set() + + # note that an identifier can be in both the undeclared and declared + # lists. + + # using AST to parse instead of using code.co_varnames, # code.co_names has several advantages: - # - we can locate an identifier as "undeclared" even if + # - we can locate an identifier as "undeclared" even if # its declared later in the same block of code - # - AST is less likely to break with version changes + # - AST is less likely to break with version changes # (for example, the behavior of co_names changed a little bit # in python version 2.5) if isinstance(code, basestring): @@ -56,11 +59,12 @@ class ArgumentList(object): f = pyparser.FindTuple(self, PythonCode, **exception_kwargs) f.visit(expr) - + class PythonFragment(PythonCode): - """extends PythonCode to provide identifier lookups in partial control statements - - e.g. + """extends PythonCode to provide identifier lookups in partial control + statements + + e.g. for x in 5: elif y==9: except (MyException, e): @@ -70,8 +74,8 @@ class PythonFragment(PythonCode): m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S) if not m: raise exceptions.CompileException( - "Fragment '%s' is not a partial control statement" % - code, **exception_kwargs) + "Fragment '%s' is not a partial control statement" % + code, **exception_kwargs) if m.group(3): code = code[:m.start(3)] (keyword, expr) = m.group(1,2) @@ -83,33 +87,36 @@ class PythonFragment(PythonCode): code = "if False:pass\n" + code + "pass" elif keyword == 'except': code = "try:pass\n" + code + "pass" + elif keyword == 'with': + code = code + "pass" else: raise exceptions.CompileException( - "Unsupported control keyword: '%s'" % + "Unsupported control keyword: '%s'" % keyword, **exception_kwargs) super(PythonFragment, self).__init__(code, **exception_kwargs) - - + + class FunctionDecl(object): """function declaration""" def __init__(self, code, allow_kwargs=True, **exception_kwargs): self.code = code expr = pyparser.parse(code, "exec", **exception_kwargs) - + f = pyparser.ParseFunc(self, **exception_kwargs) f.visit(expr) if not hasattr(self, 'funcname'): raise exceptions.CompileException( - "Code '%s' is not a function declaration" % code, - **exception_kwargs) + "Code '%s' is not a function declaration" % code, + **exception_kwargs) if not allow_kwargs and self.kwargs: raise exceptions.CompileException( - "'**%s' keyword argument not allowed here" % + "'**%s' keyword argument not allowed here" % self.argnames[-1], **exception_kwargs) - + def get_argument_expressions(self, include_defaults=True): - """return the argument declarations of this FunctionDecl as a printable list.""" - + """return the argument declarations of this FunctionDecl as a printable + list.""" + namedecls = [] defaults = [d for d in self.defaults] kwargs = self.kwargs @@ -127,8 +134,8 @@ class FunctionDecl(object): else: default = len(defaults) and defaults.pop() or None if include_defaults and default: - namedecls.insert(0, "%s=%s" % - (arg, + namedecls.insert(0, "%s=%s" % + (arg, pyparser.ExpressionGenerator(default).value() ) ) @@ -138,6 +145,7 @@ class FunctionDecl(object): class FunctionArgs(FunctionDecl): """the argument portion of a function declaration""" - + def __init__(self, code, **kwargs): - super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, **kwargs) + super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, + **kwargs) diff --git a/mako/cache.py b/mako/cache.py index ce73ae5c..f50ce58a 100644 --- a/mako/cache.py +++ b/mako/cache.py @@ -1,124 +1,236 @@ # mako/cache.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from mako import exceptions +from mako import exceptions, util -cache = None +_cache_plugins = util.PluginLoader("mako.cache") + +register_plugin = _cache_plugins.register +register_plugin("beaker", "mako.ext.beaker_cache", "BeakerCacheImpl") -class BeakerMissing(object): - def get_cache(self, name, **kwargs): - raise exceptions.RuntimeException("the Beaker package is required to use cache functionality.") class Cache(object): """Represents a data content cache made available to the module - space of a :class:`.Template` object. - - :class:`.Cache` is a wrapper on top of a Beaker CacheManager object. - This object in turn references any number of "containers", each of - which defines its own backend (i.e. file, memory, memcached, etc.) - independently of the rest. - - """ - - def __init__(self, id, starttime): - self.id = id - self.starttime = starttime - self.def_regions = {} - - def put(self, key, value, **kwargs): - """Place a value in the cache. - - :param key: the value's key. - :param value: the value - :param \**kwargs: cache configuration arguments. The - backend is configured using these arguments upon first request. - Subsequent requests that use the same series of configuration - values will use that same backend. - - """ - - defname = kwargs.pop('defname', None) - expiretime = kwargs.pop('expiretime', None) - createfunc = kwargs.pop('createfunc', None) - - self._get_cache(defname, **kwargs).put_value(key, starttime=self.starttime, expiretime=expiretime) - - def get(self, key, **kwargs): - """Retrieve a value from the cache. - - :param key: the value's key. - :param \**kwargs: cache configuration arguments. The - backend is configured using these arguments upon first request. - Subsequent requests that use the same series of configuration - values will use that same backend. - - """ - - defname = kwargs.pop('defname', None) - expiretime = kwargs.pop('expiretime', None) - createfunc = kwargs.pop('createfunc', None) - - return self._get_cache(defname, **kwargs).get_value(key, starttime=self.starttime, expiretime=expiretime, createfunc=createfunc) - - def invalidate(self, key, **kwargs): - """Invalidate a value in the cache. - - :param key: the value's key. - :param \**kwargs: cache configuration arguments. The - backend is configured using these arguments upon first request. - Subsequent requests that use the same series of configuration - values will use that same backend. - - """ - defname = kwargs.pop('defname', None) - expiretime = kwargs.pop('expiretime', None) - createfunc = kwargs.pop('createfunc', None) - - self._get_cache(defname, **kwargs).remove_value(key, starttime=self.starttime, expiretime=expiretime) - - def invalidate_body(self): - """Invalidate the cached content of the "body" method for this template. - - """ - self.invalidate('render_body', defname='render_body') - - def invalidate_def(self, name): - """Invalidate the cached content of a particular <%def> within this template.""" - - self.invalidate('render_%s' % name, defname='render_%s' % name) - - def invalidate_closure(self, name): - """Invalidate a nested <%def> within this template. - - Caching of nested defs is a blunt tool as there is no - management of scope - nested defs that use cache tags - need to have names unique of all other nested defs in the - template, else their content will be overwritten by - each other. - - """ - - self.invalidate(name, defname=name) - - def _get_cache(self, defname, type=None, **kw): - global cache - if not cache: - try: - from beaker import cache as beaker_cache - cache = beaker_cache.CacheManager() - except ImportError: - # keep a fake cache around so subsequent - # calls don't attempt to re-import - cache = BeakerMissing() + space of a specific :class:`.Template` object. - if type == 'memcached': - type = 'ext:memcached' - if not type: - (type, kw) = self.def_regions.get(defname, ('memory', {})) + .. versionadded:: 0.6 + :class:`.Cache` by itself is mostly a + container for a :class:`.CacheImpl` object, which implements + a fixed API to provide caching services; specific subclasses exist to + implement different + caching strategies. Mako includes a backend that works with + the Beaker caching system. Beaker itself then supports + a number of backends (i.e. file, memory, memcached, etc.) + + The construction of a :class:`.Cache` is part of the mechanics + of a :class:`.Template`, and programmatic access to this + cache is typically via the :attr:`.Template.cache` attribute. + + """ + + impl = None + """Provide the :class:`.CacheImpl` in use by this :class:`.Cache`. + + This accessor allows a :class:`.CacheImpl` with additional + methods beyond that of :class:`.Cache` to be used programmatically. + + """ + + id = None + """Return the 'id' that identifies this cache. + + This is a value that should be globally unique to the + :class:`.Template` associated with this cache, and can + be used by a caching system to name a local container + for data specific to this template. + + """ + + starttime = None + """Epochal time value for when the owning :class:`.Template` was + first compiled. + + A cache implementation may wish to invalidate data earlier than + this timestamp; this has the effect of the cache for a specific + :class:`.Template` starting clean any time the :class:`.Template` + is recompiled, such as when the original template file changed on + the filesystem. + + """ + + def __init__(self, template, *args): + # check for a stale template calling the + # constructor + if isinstance(template, basestring) and args: + return + self.template = template + self.id = template.module.__name__ + self.starttime = template.module._modified_time + self._def_regions = {} + self.impl = self._load_impl(self.template.cache_impl) + + def _load_impl(self, name): + return _cache_plugins.load(name)(self) + + def get_or_create(self, key, creation_function, **kw): + """Retrieve a value from the cache, using the given creation function + to generate a new value.""" + + return self._ctx_get_or_create(key, creation_function, None, **kw) + + def _ctx_get_or_create(self, key, creation_function, context, **kw): + """Retrieve a value from the cache, using the given creation function + to generate a new value.""" + + if not self.template.cache_enabled: + return creation_function() + + return self.impl.get_or_create(key, + creation_function, + **self._get_cache_kw(kw, context)) + + def set(self, key, value, **kw): + """Place a value in the cache. + + :param key: the value's key. + :param value: the value. + :param \**kw: cache configuration arguments. + + """ + + self.impl.set(key, value, **self._get_cache_kw(kw, None)) + + put = set + """A synonym for :meth:`.Cache.set`. + + This is here for backwards compatibility. + + """ + + def get(self, key, **kw): + """Retrieve a value from the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. The + backend is configured using these arguments upon first request. + Subsequent requests that use the same series of configuration + values will use that same backend. + + """ + return self.impl.get(key, **self._get_cache_kw(kw, None)) + + def invalidate(self, key, **kw): + """Invalidate a value in the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. The + backend is configured using these arguments upon first request. + Subsequent requests that use the same series of configuration + values will use that same backend. + + """ + self.impl.invalidate(key, **self._get_cache_kw(kw, None)) + + def invalidate_body(self): + """Invalidate the cached content of the "body" method for this + template. + + """ + self.invalidate('render_body', __M_defname='render_body') + + def invalidate_def(self, name): + """Invalidate the cached content of a particular ``<%def>`` within this + template. + + """ + + self.invalidate('render_%s' % name, __M_defname='render_%s' % name) + + def invalidate_closure(self, name): + """Invalidate a nested ``<%def>`` within this template. + + Caching of nested defs is a blunt tool as there is no + management of scope -- nested defs that use cache tags + need to have names unique of all other nested defs in the + template, else their content will be overwritten by + each other. + + """ + + self.invalidate(name, __M_defname=name) + + def _get_cache_kw(self, kw, context): + defname = kw.pop('__M_defname', None) + if not defname: + tmpl_kw = self.template.cache_args.copy() + tmpl_kw.update(kw) + elif defname in self._def_regions: + tmpl_kw = self._def_regions[defname] else: - self.def_regions[defname] = (type, kw) - return cache.get_cache(self.id, type=type, **kw) - \ No newline at end of file + tmpl_kw = self.template.cache_args.copy() + tmpl_kw.update(kw) + self._def_regions[defname] = tmpl_kw + if context and self.impl.pass_context: + tmpl_kw = tmpl_kw.copy() + tmpl_kw.setdefault('context', context) + return tmpl_kw + +class CacheImpl(object): + """Provide a cache implementation for use by :class:`.Cache`.""" + + def __init__(self, cache): + self.cache = cache + + pass_context = False + """If ``True``, the :class:`.Context` will be passed to + :meth:`get_or_create <.CacheImpl.get_or_create>` as the name ``'context'``. + """ + + def get_or_create(self, key, creation_function, **kw): + """Retrieve a value from the cache, using the given creation function + to generate a new value. + + This function *must* return a value, either from + the cache, or via the given creation function. + If the creation function is called, the newly + created value should be populated into the cache + under the given key before being returned. + + :param key: the value's key. + :param creation_function: function that when called generates + a new value. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def set(self, key, value, **kw): + """Place a value in the cache. + + :param key: the value's key. + :param value: the value. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def get(self, key, **kw): + """Retrieve a value from the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def invalidate(self, key, **kw): + """Invalidate a value in the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() diff --git a/mako/codegen.py b/mako/codegen.py index 53691807..3cec0eec 100644 --- a/mako/codegen.py +++ b/mako/codegen.py @@ -1,67 +1,79 @@ # mako/codegen.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""provides functionality for rendering a parsetree constructing into module source code.""" +"""provides functionality for rendering a parsetree constructing into module +source code.""" import time import re from mako.pygen import PythonPrinter from mako import util, ast, parsetree, filters, exceptions -MAGIC_NUMBER = 6 +MAGIC_NUMBER = 8 -def compile(node, - uri, - filename=None, - default_filters=None, - buffer_filters=None, - imports=None, - source_encoding=None, +# names which are hardwired into the +# template and are not accessed via the +# context itself +RESERVED_NAMES = set(['context', 'loop', 'UNDEFINED']) + +def compile(node, + uri, + filename=None, + default_filters=None, + buffer_filters=None, + imports=None, + source_encoding=None, generate_magic_comment=True, disable_unicode=False, - strict_undefined=False): - - """Generate module source code given a parsetree node, + strict_undefined=False, + enable_loop=True, + reserved_names=()): + + """Generate module source code given a parsetree node, uri, and optional source filename""" # if on Py2K, push the "source_encoding" string to be - # a bytestring itself, as we will be embedding it into - # the generated source and we don't want to coerce the + # a bytestring itself, as we will be embedding it into + # the generated source and we don't want to coerce the # result into a unicode object, in "disable_unicode" mode if not util.py3k and isinstance(source_encoding, unicode): source_encoding = source_encoding.encode(source_encoding) - - + + buf = util.FastEncodingBuffer() printer = PythonPrinter(buf) - _GenerateRenderMethod(printer, - _CompileContext(uri, - filename, - default_filters, + _GenerateRenderMethod(printer, + _CompileContext(uri, + filename, + default_filters, buffer_filters, - imports, + imports, source_encoding, generate_magic_comment, disable_unicode, - strict_undefined), + strict_undefined, + enable_loop, + reserved_names), node) return buf.getvalue() class _CompileContext(object): - def __init__(self, - uri, - filename, - default_filters, - buffer_filters, - imports, - source_encoding, + def __init__(self, + uri, + filename, + default_filters, + buffer_filters, + imports, + source_encoding, generate_magic_comment, disable_unicode, - strict_undefined): + strict_undefined, + enable_loop, + reserved_names): self.uri = uri self.filename = filename self.default_filters = default_filters @@ -71,11 +83,13 @@ class _CompileContext(object): self.generate_magic_comment = generate_magic_comment self.disable_unicode = disable_unicode self.strict_undefined = strict_undefined - + self.enable_loop = enable_loop + self.reserved_names = reserved_names + class _GenerateRenderMethod(object): - """A template visitor object which generates the + """A template visitor object which generates the full module source for a template. - + """ def __init__(self, printer, compiler, node): self.printer = printer @@ -83,13 +97,13 @@ class _GenerateRenderMethod(object): self.compiler = compiler self.node = node self.identifier_stack = [None] - + self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag)) if self.in_def: name = "render_%s" % node.funcname args = node.get_argument_expressions() - filtered = len(node.filter_args.args) > 0 + filtered = len(node.filter_args.args) > 0 buffered = eval(node.attributes.get('buffered', 'False')) cached = eval(node.attributes.get('cached', 'False')) defs = None @@ -105,6 +119,10 @@ class _GenerateRenderMethod(object): if not pagetag.body_decl.kwargs: args += ['**pageargs'] cached = eval(pagetag.attributes.get('cached', 'False')) + self.compiler.enable_loop = self.compiler.enable_loop or eval( + pagetag.attributes.get( + 'enable_loop', 'False') + ) else: args = ['**pageargs'] cached = False @@ -113,24 +131,24 @@ class _GenerateRenderMethod(object): args = ['context'] else: args = [a for a in ['context'] + args] - + self.write_render_callable( - pagetag or node, - name, args, + pagetag or node, + name, args, buffered, filtered, cached) - + if defs is not None: for node in defs: _GenerateRenderMethod(printer, compiler, node) - + @property def identifiers(self): return self.identifier_stack[-1] - + def write_toplevel(self): """Traverse a template structure for module-level directives and generate the start of module-level code. - + """ inherit = [] namespaces = {} @@ -138,7 +156,7 @@ class _GenerateRenderMethod(object): encoding =[None] self.compiler.pagetag = None - + class FindTopLevel(object): def visitInheritTag(s, node): inherit.append(node) @@ -149,7 +167,7 @@ class _GenerateRenderMethod(object): def visitCode(s, node): if node.ismodule: module_code.append(node) - + f = FindTopLevel() for n in self.node.nodes: n.accept_visitor(f) @@ -160,41 +178,40 @@ class _GenerateRenderMethod(object): for n in module_code: module_ident = module_ident.union(n.declared_identifiers()) - module_identifiers = _Identifiers() + module_identifiers = _Identifiers(self.compiler) module_identifiers.declared = module_ident - + # module-level names, python code if self.compiler.generate_magic_comment and \ self.compiler.source_encoding: self.printer.writeline("# -*- encoding:%s -*-" % self.compiler.source_encoding) - + self.printer.writeline("from mako import runtime, filters, cache") self.printer.writeline("UNDEFINED = runtime.UNDEFINED") self.printer.writeline("__M_dict_builtin = dict") self.printer.writeline("__M_locals_builtin = locals") self.printer.writeline("_magic_number = %r" % MAGIC_NUMBER) self.printer.writeline("_modified_time = %r" % time.time()) + self.printer.writeline("_enable_loop = %r" % self.compiler.enable_loop) self.printer.writeline( - "_template_filename=%r" % self.compiler.filename) - self.printer.writeline("_template_uri=%r" % self.compiler.uri) + "_template_filename = %r" % self.compiler.filename) + self.printer.writeline("_template_uri = %r" % self.compiler.uri) self.printer.writeline( - "_template_cache=cache.Cache(__name__, _modified_time)") - self.printer.writeline( - "_source_encoding=%r" % self.compiler.source_encoding) + "_source_encoding = %r" % self.compiler.source_encoding) if self.compiler.imports: buf = '' for imp in self.compiler.imports: buf += imp + "\n" self.printer.writeline(imp) impcode = ast.PythonCode( - buf, - source='', lineno=0, - pos=0, + buf, + source='', lineno=0, + pos=0, filename='template defined imports') else: impcode = None - + main_identifiers = module_identifiers.branch(self.node) module_identifiers.topleveldefs = \ module_identifiers.topleveldefs.\ @@ -202,9 +219,9 @@ class _GenerateRenderMethod(object): module_identifiers.declared.add("UNDEFINED") if impcode: module_identifiers.declared.update(impcode.declared_identifiers) - + self.compiler.identifiers = module_identifiers - self.printer.writeline("_exports = %r" % + self.printer.writeline("_exports = %r" % [n.name for n in main_identifiers.topleveldefs.values()] ) @@ -221,25 +238,29 @@ class _GenerateRenderMethod(object): return main_identifiers.topleveldefs.values() - def write_render_callable(self, node, name, args, buffered, filtered, cached): + def write_render_callable(self, node, name, args, buffered, filtered, + cached): """write a top-level render callable. - + this could be the main render() method or that of a top-level def.""" - + if self.in_def: decorator = node.decorator if decorator: - self.printer.writeline("@runtime._decorate_toplevel(%s)" % decorator) - + self.printer.writeline( + "@runtime._decorate_toplevel(%s)" % decorator) + self.printer.writelines( "def %s(%s):" % (name, ','.join(args)), - "context.caller_stack._push_frame()", + # push new frame, assign current frame to __M_caller + "__M_caller = context.caller_stack._push_frame()", "try:" ) if buffered or filtered or cached: self.printer.writeline("context._push_buffer()") - - self.identifier_stack.append(self.compiler.identifiers.branch(self.node)) + + self.identifier_stack.append( + self.compiler.identifiers.branch(self.node)) if (not self.in_def or self.node.is_block) and '**pageargs' in args: self.identifier_stack[-1].argument_declared.add('pageargs') @@ -247,7 +268,7 @@ class _GenerateRenderMethod(object): len(self.identifiers.locally_assigned) > 0 or len(self.identifiers.argument_declared) > 0 ): - self.printer.writeline("__M_locals = __M_dict_builtin(%s)" % + self.printer.writeline("__M_locals = __M_dict_builtin(%s)" % ','.join([ "%s=%s" % (x, x) for x in self.identifiers.argument_declared @@ -263,12 +284,12 @@ class _GenerateRenderMethod(object): self.printer.write("\n\n") if cached: self.write_cache_decorator( - node, name, - args, buffered, + node, name, + args, buffered, self.identifiers, toplevel=True) - + def write_module_code(self, module_code): - """write module-level template code, i.e. that which + """write module-level template code, i.e. that which is enclosed in <%! %> tags in the template.""" for n in module_code: self.write_source_comment(n) @@ -276,7 +297,7 @@ class _GenerateRenderMethod(object): def write_inherit(self, node): """write the module-level inheritance-determination callable.""" - + self.printer.writelines( "def _mako_inherit(template, context):", "_mako_generate_namespaces(context)", @@ -298,7 +319,7 @@ class _GenerateRenderMethod(object): ) self.printer.writeline("def _mako_generate_namespaces(context):") - + for node in namespaces.values(): if node.attributes.has_key('import'): self.compiler.has_ns_imports = True @@ -318,7 +339,8 @@ class _GenerateRenderMethod(object): def visitDefOrBase(s, node): if node.is_anonymous: raise exceptions.CompileException( - "Can't put anonymous blocks inside <%namespace>", + "Can't put anonymous blocks inside " + "<%namespace>", **node.exception_kwargs ) self.write_inline_def(node, identifiers, nested=False) @@ -335,45 +357,51 @@ class _GenerateRenderMethod(object): if 'file' in node.parsed_attributes: self.printer.writeline( - "ns = runtime.TemplateNamespace(%r, context._clean_inheritance_tokens()," - " templateuri=%s, callables=%s, calling_uri=_template_uri)" % + "ns = runtime.TemplateNamespace(%r," + " context._clean_inheritance_tokens()," + " templateuri=%s, callables=%s, " + " calling_uri=_template_uri)" % ( - node.name, - node.parsed_attributes.get('file', 'None'), - callable_name, + node.name, + node.parsed_attributes.get('file', 'None'), + callable_name, ) ) elif 'module' in node.parsed_attributes: self.printer.writeline( - "ns = runtime.ModuleNamespace(%r, context._clean_inheritance_tokens()," - " callables=%s, calling_uri=_template_uri, module=%s)" % + "ns = runtime.ModuleNamespace(%r," + " context._clean_inheritance_tokens()," + " callables=%s, calling_uri=_template_uri," + " module=%s)" % ( - node.name, - callable_name, - node.parsed_attributes.get('module', 'None') + node.name, + callable_name, + node.parsed_attributes.get('module', 'None') ) ) else: self.printer.writeline( - "ns = runtime.Namespace(%r, context._clean_inheritance_tokens()," + "ns = runtime.Namespace(%r," + " context._clean_inheritance_tokens()," " callables=%s, calling_uri=_template_uri)" % ( node.name, - callable_name, + callable_name, ) ) if eval(node.attributes.get('inheritable', "False")): self.printer.writeline("context['self'].%s = ns" % (node.name)) - - self.printer.writeline("context.namespaces[(__name__, %s)] = ns" % repr(node.name)) + + self.printer.writeline( + "context.namespaces[(__name__, %s)] = ns" % repr(node.name)) self.printer.write("\n") if not len(namespaces): self.printer.writeline("pass") self.printer.writeline(None) - + def write_variable_declares(self, identifiers, toplevel=False, limit=None): """write variable declarations at the top of a function. - + the variable declarations are in the form of callable definitions for defs and/or name lookup within the function's context argument. the names declared are based @@ -382,53 +410,67 @@ class _GenerateRenderMethod(object): operation. names that are assigned within the body are assumed to be locally-scoped variables and are not separately declared. - + for def callable definitions, if the def is a top-level callable then a 'stub' callable is generated which wraps the current Context into a closure. if the def is not top-level, it is fully rendered as a local closure. - + """ + # collection of all defs available to us in this scope comp_idents = dict([(c.funcname, c) for c in identifiers.defs]) to_write = set() - - # write "context.get()" for all variables we are going to + + # write "context.get()" for all variables we are going to # need that arent in the namespace yet to_write = to_write.union(identifiers.undeclared) - - # write closure functions for closures that we define - # right here - to_write = to_write.union([c.funcname for c in identifiers.closuredefs.values()]) - # remove identifiers that are declared in the argument + # write closure functions for closures that we define + # right here + to_write = to_write.union( + [c.funcname for c in identifiers.closuredefs.values()]) + + # remove identifiers that are declared in the argument # signature of the callable to_write = to_write.difference(identifiers.argument_declared) - # remove identifiers that we are going to assign to. + # remove identifiers that we are going to assign to. # in this way we mimic Python's behavior, - # i.e. assignment to a variable within a block + # i.e. assignment to a variable within a block # means that variable is now a "locally declared" var, - # which cannot be referenced beforehand. + # which cannot be referenced beforehand. to_write = to_write.difference(identifiers.locally_declared) - + + if self.compiler.enable_loop: + has_loop = "loop" in to_write + to_write.discard("loop") + else: + has_loop = False + # if a limiting set was sent, constraint to those items in that list # (this is used for the caching decorator) if limit is not None: to_write = to_write.intersection(limit) - + if toplevel and getattr(self.compiler, 'has_ns_imports', False): self.printer.writeline("_import_ns = {}") self.compiler.has_imports = True for ident, ns in self.compiler.namespaces.iteritems(): if ns.attributes.has_key('import'): self.printer.writeline( - "_mako_get_namespace(context, %r)._populate(_import_ns, %r)" % + "_mako_get_namespace(context, %r)."\ + "_populate(_import_ns, %r)" % ( ident, re.split(r'\s*,\s*', ns.attributes['import']) )) - + + if has_loop: + self.printer.writeline( + 'loop = __M_loop = runtime.LoopStack()' + ) + for ident in to_write: if ident in comp_idents: comp = comp_idents[ident] @@ -445,26 +487,26 @@ class _GenerateRenderMethod(object): elif ident in self.compiler.namespaces: self.printer.writeline( - "%s = _mako_get_namespace(context, %r)" % + "%s = _mako_get_namespace(context, %r)" % (ident, ident) ) else: if getattr(self.compiler, 'has_ns_imports', False): if self.compiler.strict_undefined: self.printer.writelines( - "%s = _import_ns.get(%r, UNDEFINED)" % + "%s = _import_ns.get(%r, UNDEFINED)" % (ident, ident), "if %s is UNDEFINED:" % ident, "try:", "%s = context[%r]" % (ident, ident), "except KeyError:", - "raise NameError(\"'%s' is not defined\")" % + "raise NameError(\"'%s' is not defined\")" % ident, None, None ) else: self.printer.writeline( - "%s = _import_ns.get(%r, context.get(%r, UNDEFINED))" % + "%s = _import_ns.get(%r, context.get(%r, UNDEFINED))" % (ident, ident, ident)) else: if self.compiler.strict_undefined: @@ -472,7 +514,7 @@ class _GenerateRenderMethod(object): "try:", "%s = context[%r]" % (ident, ident), "except KeyError:", - "raise NameError(\"'%s' is not defined\")" % + "raise NameError(\"'%s' is not defined\")" % ident, None ) @@ -480,11 +522,12 @@ class _GenerateRenderMethod(object): self.printer.writeline( "%s = context.get(%r, UNDEFINED)" % (ident, ident) ) - + self.printer.writeline("__M_writer = context.writer()") - + def write_source_comment(self, node): - """write a source comment containing the line number of the corresponding template line.""" + """write a source comment containing the line number of the + corresponding template line.""" if self.last_source_line != node.lineno: self.printer.writeline("# SOURCE LINE %d" % node.lineno) self.last_source_line = node.lineno @@ -494,7 +537,7 @@ class _GenerateRenderMethod(object): funcname = node.funcname namedecls = node.get_argument_expressions() nameargs = node.get_argument_expressions(include_defaults=False) - + if not self.in_def and ( len(self.identifiers.locally_assigned) > 0 or len(self.identifiers.argument_declared) > 0): @@ -502,23 +545,27 @@ class _GenerateRenderMethod(object): else: nameargs.insert(0, 'context') self.printer.writeline("def %s(%s):" % (funcname, ",".join(namedecls))) - self.printer.writeline("return render_%s(%s)" % (funcname, ",".join(nameargs))) + self.printer.writeline( + "return render_%s(%s)" % (funcname, ",".join(nameargs))) self.printer.writeline(None) - + def write_inline_def(self, node, identifiers, nested): """write a locally-available def callable inside an enclosing def.""" namedecls = node.get_argument_expressions() - + decorator = node.decorator if decorator: - self.printer.writeline("@runtime._decorate_inline(context, %s)" % decorator) - self.printer.writeline("def %s(%s):" % (node.funcname, ",".join(namedecls))) - filtered = len(node.filter_args.args) > 0 + self.printer.writeline( + "@runtime._decorate_inline(context, %s)" % decorator) + self.printer.writeline( + "def %s(%s):" % (node.funcname, ",".join(namedecls))) + filtered = len(node.filter_args.args) > 0 buffered = eval(node.attributes.get('buffered', 'False')) cached = eval(node.attributes.get('cached', 'False')) self.printer.writelines( - "context.caller_stack._push_frame()", + # push new frame, assign current frame to __M_caller + "__M_caller = context.caller_stack._push_frame()", "try:" ) if buffered or filtered or cached: @@ -529,26 +576,29 @@ class _GenerateRenderMethod(object): identifiers = identifiers.branch(node, nested=nested) self.write_variable_declares(identifiers) - + self.identifier_stack.append(identifiers) for n in node.nodes: n.accept_visitor(self) self.identifier_stack.pop() - + self.write_def_finish(node, buffered, filtered, cached) self.printer.writeline(None) if cached: - self.write_cache_decorator(node, node.funcname, - namedecls, False, identifiers, + self.write_cache_decorator(node, node.funcname, + namedecls, False, identifiers, inline=True, toplevel=False) - - def write_def_finish(self, node, buffered, filtered, cached, callstack=True): - """write the end section of a rendering function, either outermost or inline. - - this takes into account if the rendering function was filtered, buffered, etc. - and closes the corresponding try: block if any, and writes code to retrieve - captured content, apply filters, send proper return value.""" - + + def write_def_finish(self, node, buffered, filtered, cached, + callstack=True): + """write the end section of a rendering function, either outermost or + inline. + + this takes into account if the rendering function was filtered, + buffered, etc. and closes the corresponding try: block if any, and + writes code to retrieve captured content, apply filters, send proper + return value.""" + if not buffered and not cached and not filtered: self.printer.writeline("return ''") if callstack: @@ -557,7 +607,7 @@ class _GenerateRenderMethod(object): "context.caller_stack._pop_frame()", None ) - + if buffered or filtered or cached: if buffered or cached: # in a caching scenario, don't try to get a writer @@ -570,19 +620,21 @@ class _GenerateRenderMethod(object): ) else: self.printer.writelines( - "finally:", - "__M_buf, __M_writer = context._pop_buffer_and_writer()" + "finally:", + "__M_buf, __M_writer = context._pop_buffer_and_writer()" ) - + if callstack: self.printer.writeline("context.caller_stack._pop_frame()") - + s = "__M_buf.getvalue()" if filtered: - s = self.create_filter_callable(node.filter_args.args, s, False) + s = self.create_filter_callable(node.filter_args.args, s, + False) self.printer.writeline(None) if buffered and not cached: - s = self.create_filter_callable(self.compiler.buffer_filters, s, False) + s = self.create_filter_callable(self.compiler.buffer_filters, + s, False) if buffered or cached: self.printer.writeline("return %s" % s) else: @@ -591,71 +643,81 @@ class _GenerateRenderMethod(object): "return ''" ) - def write_cache_decorator(self, node_or_pagetag, name, - args, buffered, identifiers, + def write_cache_decorator(self, node_or_pagetag, name, + args, buffered, identifiers, inline=False, toplevel=False): - """write a post-function decorator to replace a rendering + """write a post-function decorator to replace a rendering callable with a cached version of itself.""" - + self.printer.writeline("__M_%s = %s" % (name, name)) - cachekey = node_or_pagetag.parsed_attributes.get('cache_key', repr(name)) - cacheargs = {} - for arg in ( - ('cache_type', 'type'), ('cache_dir', 'data_dir'), - ('cache_timeout', 'expiretime'), ('cache_url', 'url')): - val = node_or_pagetag.parsed_attributes.get(arg[0], None) - if val is not None: - if arg[1] == 'expiretime': - cacheargs[arg[1]] = int(eval(val)) - else: - cacheargs[arg[1]] = val - else: - if self.compiler.pagetag is not None: - val = self.compiler.pagetag.parsed_attributes.get(arg[0], None) - if val is not None: - if arg[1] == 'expiretime': - cacheargs[arg[1]] == int(eval(val)) - else: - cacheargs[arg[1]] = val - + cachekey = node_or_pagetag.parsed_attributes.get('cache_key', + repr(name)) + + cache_args = {} + if self.compiler.pagetag is not None: + cache_args.update( + ( + pa[6:], + self.compiler.pagetag.parsed_attributes[pa] + ) + for pa in self.compiler.pagetag.parsed_attributes + if pa.startswith('cache_') and pa != 'cache_key' + ) + cache_args.update( + ( + pa[6:], + node_or_pagetag.parsed_attributes[pa] + ) for pa in node_or_pagetag.parsed_attributes + if pa.startswith('cache_') and pa != 'cache_key' + ) + if 'timeout' in cache_args: + cache_args['timeout'] = int(eval(cache_args['timeout'])) + self.printer.writeline("def %s(%s):" % (name, ','.join(args))) - + # form "arg1, arg2, arg3=arg3, arg4=arg4", etc. pass_args = [ - '=' in a and "%s=%s" % ((a.split('=')[0],)*2) or a + '=' in a and "%s=%s" % ((a.split('=')[0],)*2) or a for a in args ] self.write_variable_declares( - identifiers, - toplevel=toplevel, + identifiers, + toplevel=toplevel, limit=node_or_pagetag.undeclared_identifiers() ) if buffered: s = "context.get('local')."\ - "get_cached(%s, defname=%r, %screatefunc=lambda:__M_%s(%s))" % \ - (cachekey, name, - ''.join(["%s=%s, " % (k,v) for k, v in cacheargs.iteritems()]), - name, ','.join(pass_args)) + "cache._ctx_get_or_create("\ + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r)" % \ + (cachekey, name, ','.join(pass_args), + ''.join(["%s=%s, " % (k,v) + for k, v in cache_args.items()]), + name + ) # apply buffer_filters - s = self.create_filter_callable(self.compiler.buffer_filters, s, False) + s = self.create_filter_callable(self.compiler.buffer_filters, s, + False) self.printer.writelines("return " + s,None) else: self.printer.writelines( "__M_writer(context.get('local')." - "get_cached(%s, defname=%r, %screatefunc=lambda:__M_%s(%s)))" % - (cachekey, name, - ''.join(["%s=%s, " % (k,v) for k, v in cacheargs.iteritems()]), - name, ','.join(pass_args)), + "cache._ctx_get_or_create("\ + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r))" % + (cachekey, name, ','.join(pass_args), + ''.join(["%s=%s, " % (k,v) + for k, v in cache_args.items()]), + name, + ), "return ''", None ) def create_filter_callable(self, args, target, is_expression): - """write a filter-applying expression based on the filters - present in the given filter names, adjusting for the global + """write a filter-applying expression based on the filters + present in the given filter names, adjusting for the global 'default' filter aliases as needed.""" - + def locate_encode(name): if re.match(r'decode\..+', name): return "filters." + name @@ -663,7 +725,7 @@ class _GenerateRenderMethod(object): return filters.NON_UNICODE_ESCAPES.get(name, name) else: return filters.DEFAULT_ESCAPES.get(name, name) - + if 'n' not in args: if is_expression: if self.compiler.pagetag: @@ -685,7 +747,7 @@ class _GenerateRenderMethod(object): assert e is not None target = "%s(%s)" % (e, target) return target - + def visitExpression(self, node): self.write_source_comment(node) if len(node.escapes) or \ @@ -694,25 +756,46 @@ class _GenerateRenderMethod(object): len(self.compiler.pagetag.filter_args.args) ) or \ len(self.compiler.default_filters): - - s = self.create_filter_callable(node.escapes_code.args, "%s" % node.text, True) + + s = self.create_filter_callable(node.escapes_code.args, + "%s" % node.text, True) self.printer.writeline("__M_writer(%s)" % s) else: self.printer.writeline("__M_writer(%s)" % node.text) - + def visitControlLine(self, node): if node.isend: - if not node.get_children(): - self.printer.writeline("pass") self.printer.writeline(None) + if node.has_loop_context: + self.printer.writeline('finally:') + self.printer.writeline("loop = __M_loop._exit()") + self.printer.writeline(None) else: self.write_source_comment(node) - self.printer.writeline(node.text) - + if self.compiler.enable_loop and node.keyword == 'for': + text = mangle_mako_loop(node, self.printer) + else: + text = node.text + self.printer.writeline(text) + children = node.get_children() + # this covers the three situations where we want to insert a pass: + # 1) a ternary control line with no children, + # 2) a primary control line with nothing but its own ternary + # and end control lines, and + # 3) any control line with no content other than comments + if not children or ( + util.all(isinstance(c, (parsetree.Comment, + parsetree.ControlLine)) + for c in children) and + util.all((node.is_ternary(c.keyword) or c.isend) + for c in children + if isinstance(c, parsetree.ControlLine))): + self.printer.writeline("pass") + def visitText(self, node): self.write_source_comment(node) self.printer.writeline("__M_writer(%s)" % repr(node.content)) - + def visitTextTag(self, node): filtered = len(node.filter_args.args) > 0 if filtered: @@ -726,46 +809,47 @@ class _GenerateRenderMethod(object): self.printer.writelines( "finally:", "__M_buf, __M_writer = context._pop_buffer_and_writer()", - "__M_writer(%s)" % + "__M_writer(%s)" % self.create_filter_callable( - node.filter_args.args, - "__M_buf.getvalue()", + node.filter_args.args, + "__M_buf.getvalue()", False), None ) - + def visitCode(self, node): if not node.ismodule: self.write_source_comment(node) self.printer.write_indented_block(node.text) if not self.in_def and len(self.identifiers.locally_assigned) > 0: - # if we are the "template" def, fudge locally + # if we are the "template" def, fudge locally # declared/modified variables into the "__M_locals" dictionary, - # which is used for def calls within the same template, + # which is used for def calls within the same template, # to simulate "enclosing scope" - self.printer.writeline('__M_locals_builtin_stored = __M_locals_builtin()') self.printer.writeline( - '__M_locals.update(__M_dict_builtin([(__M_key,' - ' __M_locals_builtin_stored[__M_key]) for ' - '__M_key in [%s] if __M_key in __M_locals_builtin_stored]))' % - ','.join([repr(x) for x in node.declared_identifiers()])) + '__M_locals_builtin_stored = __M_locals_builtin()') + self.printer.writeline( + '__M_locals.update(__M_dict_builtin([(__M_key,' + ' __M_locals_builtin_stored[__M_key]) for __M_key in' + ' [%s] if __M_key in __M_locals_builtin_stored]))' % + ','.join([repr(x) for x in node.declared_identifiers()])) def visitIncludeTag(self, node): self.write_source_comment(node) args = node.attributes.get('args') if args: self.printer.writeline( - "runtime._include_file(context, %s, _template_uri, %s)" % - (node.parsed_attributes['file'], args)) + "runtime._include_file(context, %s, _template_uri, %s)" % + (node.parsed_attributes['file'], args)) else: self.printer.writeline( "runtime._include_file(context, %s, _template_uri)" % (node.parsed_attributes['file'])) - + def visitNamespaceTag(self, node): pass - + def visitDefTag(self, node): pass @@ -776,9 +860,10 @@ class _GenerateRenderMethod(object): nameargs = node.get_argument_expressions(include_defaults=False) nameargs += ['**pageargs'] self.printer.writeline("if 'parent' not in context._data or " - "not hasattr(context._data['parent'], '%s'):" - % node.funcname) - self.printer.writeline("context['self'].%s(%s)" % (node.funcname, ",".join(nameargs))) + "not hasattr(context._data['parent'], '%s'):" + % node.funcname) + self.printer.writeline( + "context['self'].%s(%s)" % (node.funcname, ",".join(nameargs))) self.printer.writeline("\n") def visitCallNamespaceTag(self, node): @@ -786,18 +871,18 @@ class _GenerateRenderMethod(object): # as ensure the given namespace will be imported, # pre-import the namespace, etc. self.visitCallTag(node) - + def visitCallTag(self, node): self.printer.writeline("def ccall(caller):") export = ['body'] callable_identifiers = self.identifiers.branch(node, nested=True) body_identifiers = callable_identifiers.branch(node, nested=False) - # we want the 'caller' passed to ccall to be used - # for the body() function, but for other non-body() - # <%def>s within <%call> we want the current caller + # we want the 'caller' passed to ccall to be used + # for the body() function, but for other non-body() + # <%def>s within <%call> we want the current caller # off the call stack (if any) body_identifiers.add_declared('caller') - + self.identifier_stack.append(body_identifiers) class DefVisitor(object): def visitDefTag(s, node): @@ -810,8 +895,8 @@ class _GenerateRenderMethod(object): self.write_inline_def(node, callable_identifiers, nested=False) if not node.is_anonymous: export.append(node.funcname) - # remove defs that are within the <%call> from the "closuredefs" defined - # in the body, so they dont render twice + # remove defs that are within the <%call> from the + # "closuredefs" defined in the body, so they dont render twice if node.funcname in body_identifiers.closuredefs: del body_identifiers.closuredefs[node.funcname] @@ -819,11 +904,11 @@ class _GenerateRenderMethod(object): for n in node.nodes: n.accept_visitor(vis) self.identifier_stack.pop() - - bodyargs = node.body_decl.get_argument_expressions() + + bodyargs = node.body_decl.get_argument_expressions() self.printer.writeline("def body(%s):" % ','.join(bodyargs)) - - # TODO: figure out best way to specify + + # TODO: figure out best way to specify # buffering/nonbuffering (at call time would be better) buffered = False if buffered: @@ -833,11 +918,11 @@ class _GenerateRenderMethod(object): ) self.write_variable_declares(body_identifiers) self.identifier_stack.append(body_identifiers) - + for n in node.nodes: n.accept_visitor(self) self.identifier_stack.pop() - + self.write_def_finish(node, buffered, False, False, callstack=False) self.printer.writelines( None, @@ -846,15 +931,15 @@ class _GenerateRenderMethod(object): ) self.printer.writelines( - # get local reference to current caller, if any - "caller = context.caller_stack._get_caller()", # push on caller for nested call "context.caller_stack.nextcaller = " - "runtime.Namespace('caller', context, callables=ccall(caller))", + "runtime.Namespace('caller', context, " + "callables=ccall(__M_caller))", "try:") self.write_source_comment(node) self.printer.writelines( - "__M_writer(%s)" % self.create_filter_callable([], node.expression, True), + "__M_writer(%s)" % self.create_filter_callable( + [], node.expression, True), "finally:", "context.caller_stack.nextcaller = None", None @@ -862,9 +947,8 @@ class _GenerateRenderMethod(object): class _Identifiers(object): """tracks the status of identifier names as template code is rendered.""" - - def __init__(self, node=None, parent=None, nested=False): - + + def __init__(self, compiler, node=None, parent=None, nested=False): if parent is not None: # if we are the branch created in write_namespaces(), # we don't share any context from the main body(). @@ -872,65 +956,76 @@ class _Identifiers(object): self.declared = set() self.topleveldefs = util.SetLikeDict() else: - # things that have already been declared + # things that have already been declared # in an enclosing namespace (i.e. names we can just use) self.declared = set(parent.declared).\ - union([c.name for c in parent.closuredefs.values()]).\ - union(parent.locally_declared).\ - union(parent.argument_declared) - - # if these identifiers correspond to a "nested" - # scope, it means whatever the parent identifiers - # had as undeclared will have been declared by that parent, + union([c.name for c in parent.closuredefs.values()]).\ + union(parent.locally_declared).\ + union(parent.argument_declared) + + # if these identifiers correspond to a "nested" + # scope, it means whatever the parent identifiers + # had as undeclared will have been declared by that parent, # and therefore we have them in our scope. if nested: self.declared = self.declared.union(parent.undeclared) - + # top level defs that are available self.topleveldefs = util.SetLikeDict(**parent.topleveldefs) else: self.declared = set() self.topleveldefs = util.SetLikeDict() - - # things within this level that are referenced before they + + self.compiler = compiler + + # things within this level that are referenced before they # are declared (e.g. assigned to) self.undeclared = set() - - # things that are declared locally. some of these things - # could be in the "undeclared" list as well if they are + + # things that are declared locally. some of these things + # could be in the "undeclared" list as well if they are # referenced before declared self.locally_declared = set() - - # assignments made in explicit python blocks. - # these will be propagated to + + # assignments made in explicit python blocks. + # these will be propagated to # the context of local def calls. self.locally_assigned = set() - - # things that are declared in the argument + + # things that are declared in the argument # signature of the def callable self.argument_declared = set() - + # closure defs that are defined in this level self.closuredefs = util.SetLikeDict() - + self.node = node - + if node is not None: node.accept_visitor(self) - + + illegal_names = self.compiler.reserved_names.intersection( + self.locally_declared) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words declared in template: %s" % + ", ".join(illegal_names)) + + def branch(self, node, **kwargs): - """create a new Identifiers for a new Node, with + """create a new Identifiers for a new Node, with this Identifiers as the parent.""" - - return _Identifiers(node, self, **kwargs) - + + return _Identifiers(self.compiler, node, self, **kwargs) + @property def defs(self): return set(self.topleveldefs.union(self.closuredefs).values()) - + def __repr__(self): return "Identifiers(declared=%r, locally_declared=%r, "\ - "undeclared=%r, topleveldefs=%r, closuredefs=%r, argumentdeclared=%r)" %\ + "undeclared=%r, topleveldefs=%r, closuredefs=%r, "\ + "argumentdeclared=%r)" %\ ( list(self.declared), list(self.locally_declared), @@ -938,36 +1033,38 @@ class _Identifiers(object): [c.name for c in self.topleveldefs.values()], [c.name for c in self.closuredefs.values()], self.argument_declared) - + def check_declared(self, node): - """update the state of this Identifiers with the undeclared + """update the state of this Identifiers with the undeclared and declared identifiers of the given node.""" - + for ident in node.undeclared_identifiers(): - if ident != 'context' and ident not in self.declared.union(self.locally_declared): + if ident != 'context' and\ + ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.locally_declared.add(ident) - + def add_declared(self, ident): self.declared.add(ident) if ident in self.undeclared: self.undeclared.remove(ident) - + def visitExpression(self, node): self.check_declared(node) - + def visitControlLine(self, node): self.check_declared(node) - + def visitCode(self, node): if not node.ismodule: self.check_declared(node) - self.locally_assigned = self.locally_assigned.union(node.declared_identifiers()) - + self.locally_assigned = self.locally_assigned.union( + node.declared_identifiers()) + def visitNamespaceTag(self, node): - # only traverse into the sub-elements of a - # <%namespace> tag if we are the branch created in + # only traverse into the sub-elements of a + # <%namespace> tag if we are the branch created in # write_namespaces() if self.node is node: for n in node.nodes: @@ -981,7 +1078,7 @@ class _Identifiers(object): (node.is_block or existing.is_block): raise exceptions.CompileException( "%%def or %%block named '%s' already " - "exists in this template." % + "exists in this template." % node.funcname, **node.exception_kwargs) def visitDefTag(self, node): @@ -991,13 +1088,15 @@ class _Identifiers(object): self._check_name_exists(self.closuredefs, node) for ident in node.undeclared_identifiers(): - if ident != 'context' and ident not in self.declared.union(self.locally_declared): + if ident != 'context' and\ + ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) - + # visit defs only one level deep if node is self.node: for ident in node.declared_identifiers(): self.argument_declared.add(ident) + for n in node.nodes: n.accept_visitor(self) @@ -1007,13 +1106,19 @@ class _Identifiers(object): if isinstance(self.node, parsetree.DefTag): raise exceptions.CompileException( - "Named block '%s' not allowed inside of def '%s'" + "Named block '%s' not allowed inside of def '%s'" % (node.name, self.node.name), **node.exception_kwargs) - elif isinstance(self.node, (parsetree.CallTag, parsetree.CallNamespaceTag)): + elif isinstance(self.node, + (parsetree.CallTag, parsetree.CallNamespaceTag)): raise exceptions.CompileException( - "Named block '%s' not allowed inside of <%%call> tag" + "Named block '%s' not allowed inside of <%%call> tag" % (node.name, ), **node.exception_kwargs) + for ident in node.undeclared_identifiers(): + if ident != 'context' and\ + ident not in self.declared.union(self.locally_declared): + self.undeclared.add(ident) + if not node.is_anonymous: self._check_name_exists(self.topleveldefs, node) self.undeclared.add(node.funcname) @@ -1026,19 +1131,20 @@ class _Identifiers(object): def visitIncludeTag(self, node): self.check_declared(node) - + def visitPageTag(self, node): for ident in node.declared_identifiers(): self.argument_declared.add(ident) self.check_declared(node) - + def visitCallNamespaceTag(self, node): self.visitCallTag(node) - + def visitCallTag(self, node): if node is self.node: for ident in node.undeclared_identifiers(): - if ident != 'context' and ident not in self.declared.union(self.locally_declared): + if ident != 'context' and\ + ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.argument_declared.add(ident) @@ -1046,6 +1152,58 @@ class _Identifiers(object): n.accept_visitor(self) else: for ident in node.undeclared_identifiers(): - if ident != 'context' and ident not in self.declared.union(self.locally_declared): + if ident != 'context' and\ + ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) - + + +_FOR_LOOP = re.compile( + r'^for\s+((?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*' + r'(?:\s*,\s*(?:[A-Za-z_][A-Za-z0-9_]*),??)*\s*(?:\)?))\s+in\s+(.*):' + ) + +def mangle_mako_loop(node, printer): + """converts a for loop into a context manager wrapped around a for loop + when access to the `loop` variable has been detected in the for loop body + """ + loop_variable = LoopVariable() + node.accept_visitor(loop_variable) + if loop_variable.detected: + node.nodes[-1].has_loop_context = True + match = _FOR_LOOP.match(node.text) + if match: + printer.writelines( + 'loop = __M_loop._enter(%s)' % match.group(2), + 'try:' + #'with __M_loop(%s) as loop:' % match.group(2) + ) + text = 'for %s in loop:' % match.group(1) + else: + raise SyntaxError("Couldn't apply loop context: %s" % node.text) + else: + text = node.text + return text + + +class LoopVariable(object): + """A node visitor which looks for the name 'loop' within undeclared + identifiers.""" + + def __init__(self): + self.detected = False + + def _loop_reference_detected(self, node): + if 'loop' in node.undeclared_identifiers(): + self.detected = True + else: + for n in node.get_children(): + n.accept_visitor(self) + + def visitControlLine(self, node): + self._loop_reference_detected(node) + + def visitCode(self, node): + self._loop_reference_detected(node) + + def visitExpression(self, node): + self._loop_reference_detected(node) diff --git a/mako/exceptions.py b/mako/exceptions.py index 491d2af5..b8d5ef33 100644 --- a/mako/exceptions.py +++ b/mako/exceptions.py @@ -1,5 +1,5 @@ # mako/exceptions.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -20,19 +20,21 @@ def _format_filepos(lineno, pos, filename): return " at line: %d char: %d" % (lineno, pos) else: return " in file '%s' at line: %d char: %d" % (filename, lineno, pos) - - + + class CompileException(MakoException): def __init__(self, message, source, lineno, pos, filename): - MakoException.__init__(self, message + _format_filepos(lineno, pos, filename)) + MakoException.__init__(self, + message + _format_filepos(lineno, pos, filename)) self.lineno =lineno self.pos = pos self.filename = filename self.source = source - + class SyntaxException(MakoException): def __init__(self, message, source, lineno, pos, filename): - MakoException.__init__(self, message + _format_filepos(lineno, pos, filename)) + MakoException.__init__(self, + message + _format_filepos(lineno, pos, filename)) self.lineno =lineno self.pos = pos self.filename = filename @@ -40,47 +42,50 @@ class SyntaxException(MakoException): class UnsupportedError(MakoException): """raised when a retired feature is used.""" - + +class NameConflictError(MakoException): + """raised when a reserved word is used inappropriately""" + class TemplateLookupException(MakoException): pass class TopLevelLookupException(TemplateLookupException): pass - + class RichTraceback(object): - """Pulls the current exception from the sys traceback and extracts + """Pull the current exception from the ``sys`` traceback and extracts Mako-specific template information. - + See the usage examples in :ref:`handling_exceptions`. - + """ def __init__(self, error=None, traceback=None): self.source, self.lineno = "", 0 if error is None or traceback is None: t, value, tback = sys.exc_info() - + if error is None: error = value or t - + if traceback is None: traceback = tback - + self.error = error self.records = self._init(traceback) - + if isinstance(self.error, (CompileException, SyntaxException)): import mako.template self.source = self.error.source self.lineno = self.error.lineno self._has_source = True - + self._init_message() - + @property def errorname(self): return util.exception_name(self.error) - + def _init_message(self): """Find a unicode representation of self.error""" try: @@ -101,25 +106,25 @@ class RichTraceback(object): yield (rec[4], rec[5], rec[2], rec[6]) else: yield tuple(rec[0:4]) - + @property def traceback(self): - """return a list of 4-tuple traceback records (i.e. normal python + """Return a list of 4-tuple traceback records (i.e. normal python format) with template-corresponding lines remapped to the originating template. - + """ return list(self._get_reformatted_records(self.records)) - + @property def reverse_records(self): return reversed(self.records) - + @property def reverse_traceback(self): - """return the same data as traceback, except in reverse order. + """Return the same data as traceback, except in reverse order. """ - + return list(self._get_reformatted_records(self.reverse_records)) def _init(self, trcback): @@ -156,7 +161,7 @@ class RichTraceback(object): line = line.decode(encoding) else: line = line.decode('ascii', 'replace') - new_trcback.append((filename, lineno, function, line, + new_trcback.append((filename, lineno, function, line, None, None, None, None)) continue @@ -177,8 +182,8 @@ class RichTraceback(object): template_line = template_lines[template_ln - 1] else: template_line = None - new_trcback.append((filename, lineno, function, - line, template_filename, template_ln, + new_trcback.append((filename, lineno, function, + line, template_filename, template_ln, template_line, template_source)) if not self.source: for l in range(len(new_trcback)-1, 0, -1): @@ -202,13 +207,13 @@ class RichTraceback(object): self.lineno = new_trcback[-1][1] return new_trcback - + def text_error_template(lookup=None): """Provides a template that renders a stack trace in a similar format to the Python interpreter, substituting source template filenames, line numbers and code for that of the originating source template, as applicable. - + """ import mako.template return mako.template.Template(r""" @@ -227,22 +232,33 @@ Traceback (most recent call last): ${tback.errorname}: ${tback.message} """) + +try: + from mako.ext.pygmentplugin import syntax_highlight,\ + pygments_html_formatter +except ImportError: + from mako.filters import html_escape + pygments_html_formatter = None + def syntax_highlight(filename='', language=None): + return html_escape + def html_error_template(): """Provides a template that renders a stack trace in an HTML format, providing an excerpt of code as well as substituting source template filenames, line numbers and code for that of the originating source template, as applicable. - The template's default encoding_errors value is 'htmlentityreplace'. the - template has two options. With the full option disabled, only a section of - an HTML document is returned. with the css option disabled, the default + The template's default ``encoding_errors`` value is ``'htmlentityreplace'``. The + template has two options. With the ``full`` option disabled, only a section of + an HTML document is returned. With the ``css`` option disabled, the default stylesheet won't be included. - + """ import mako.template return mako.template.Template(r""" <%! - from mako.exceptions import RichTraceback + from mako.exceptions import RichTraceback, syntax_highlight,\ + pygments_html_formatter %> <%page args="full=True, css=True, error=None, traceback=None"/> % if full: @@ -256,10 +272,29 @@ def html_error_template(): .stacktrace { margin:5px 5px 5px 5px; } .highlight { padding:0px 10px 0px 10px; background-color:#9F9FDF; } .nonhighlight { padding:0px; background-color:#DFDFDF; } - .sample { padding:10px; margin:10px 10px 10px 10px; font-family:monospace; } + .sample { padding:10px; margin:10px 10px 10px 10px; + font-family:monospace; } .sampleline { padding:0px 10px 0px 10px; } .sourceline { margin:5px 5px 10px 5px; font-family:monospace;} .location { font-size:80%; } + .highlight { white-space:pre; } + .sampleline { white-space:pre; } + + % if pygments_html_formatter: + ${pygments_html_formatter.get_style_defs()} + .linenos { min-width: 2.5em; text-align: right; } + pre { margin: 0; } + .syntax-highlighted { padding: 0 10px; } + .syntax-highlightedtable { border-spacing: 1px; } + .nonhighlight { border-top: 1px solid #DFDFDF; + border-bottom: 1px solid #DFDFDF; } + .stacktrace .nonhighlight { margin: 5px 15px 10px; } + .sourceline { margin: 0 0; font-family:monospace; } + .code { background-color: #F8F8F8; width: 100%; } + .error .code { background-color: #FFBDBD; } + .error .syntax-highlighted { background-color: #FFBDBD; } + % endif + % endif % if full: @@ -277,16 +312,29 @@ def html_error_template(): else: lines = None %> -

${tback.errorname}: ${tback.message}

+

${tback.errorname}: ${tback.message|h}

% if lines:
% for index in range(max(0, line-4),min(len(lines), line+5)): + <% + if pygments_html_formatter: + pygments_html_formatter.linenostart = index + 1 + %> % if index + 1 == line: -
${index + 1} ${lines[index] | h}
+ <% + if pygments_html_formatter: + old_cssclass = pygments_html_formatter.cssclass + pygments_html_formatter.cssclass = 'error ' + old_cssclass + %> + ${lines[index] | syntax_highlight(language='mako')} + <% + if pygments_html_formatter: + pygments_html_formatter.cssclass = old_cssclass + %> % else: -
${index + 1} ${lines[index] | h}
+ ${lines[index] | syntax_highlight(language='mako')} % endif % endfor
@@ -296,7 +344,13 @@ def html_error_template():
% for (filename, lineno, function, line) in tback.reverse_traceback:
${filename}, line ${lineno}:
-
${line | h}
+
+ <% + if pygments_html_formatter: + pygments_html_formatter.linenostart = lineno + %> +
${line | syntax_highlight(filename)}
+
% endfor
@@ -304,4 +358,5 @@ def html_error_template(): % endif -""", output_encoding=sys.getdefaultencoding(), encoding_errors='htmlentityreplace') +""", output_encoding=sys.getdefaultencoding(), + encoding_errors='htmlentityreplace') diff --git a/mako/ext/autohandler.py b/mako/ext/autohandler.py index 5d89ac59..93c60866 100644 --- a/mako/ext/autohandler.py +++ b/mako/ext/autohandler.py @@ -1,5 +1,5 @@ # ext/autohandler.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/mako/ext/babelplugin.py b/mako/ext/babelplugin.py index 6b7c1d35..65f7e02f 100644 --- a/mako/ext/babelplugin.py +++ b/mako/ext/babelplugin.py @@ -1,5 +1,5 @@ # ext/babelplugin.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -68,6 +68,9 @@ def extract_nodes(nodes, keywords, comment_tags, options): if isinstance(node, parsetree.DefTag): code = node.function_decl.code child_nodes = node.nodes + elif isinstance(node, parsetree.BlockTag): + code = node.body_decl.code + child_nodes = node.nodes elif isinstance(node, parsetree.CallTag): code = node.code.code child_nodes = node.nodes diff --git a/mako/ext/beaker_cache.py b/mako/ext/beaker_cache.py new file mode 100644 index 00000000..f0b50fac --- /dev/null +++ b/mako/ext/beaker_cache.py @@ -0,0 +1,70 @@ +"""Provide a :class:`.CacheImpl` for the Beaker caching system.""" + +from mako import exceptions + +from mako.cache import CacheImpl + +_beaker_cache = None +class BeakerCacheImpl(CacheImpl): + """A :class:`.CacheImpl` provided for the Beaker caching system. + + This plugin is used by default, based on the default + value of ``'beaker'`` for the ``cache_impl`` parameter of the + :class:`.Template` or :class:`.TemplateLookup` classes. + + """ + + def __init__(self, cache): + global _beaker_cache + if _beaker_cache is None: + try: + from beaker import cache as beaker_cache + except ImportError, e: + raise exceptions.RuntimeException( + "the Beaker package is required to use cache " + "functionality.") + + if 'manager' in cache.template.cache_args: + _beaker_cache = cache.template.cache_args['manager'] + else: + _beaker_cache = beaker_cache.CacheManager() + super(BeakerCacheImpl, self).__init__(cache) + + def _get_cache(self, **kw): + expiretime = kw.pop('timeout', None) + if 'dir' in kw: + kw['data_dir'] = kw.pop('dir') + elif self.cache.template.module_directory: + kw['data_dir'] = self.cache.template.module_directory + + if 'manager' in kw: + kw.pop('manager') + + if kw.get('type') == 'memcached': + kw['type'] = 'ext:memcached' + + if 'region' in kw: + region = kw.pop('region') + cache = _beaker_cache.get_cache_region(self.cache.id, region, **kw) + else: + cache = _beaker_cache.get_cache(self.cache.id, **kw) + cache_args = {'starttime':self.cache.starttime} + if expiretime: + cache_args['expiretime'] = expiretime + return cache, cache_args + + def get_or_create(self, key, creation_function, **kw): + cache, kw = self._get_cache(**kw) + return cache.get(key, createfunc=creation_function, **kw) + + def put(self, key, value, **kw): + cache, kw = self._get_cache(**kw) + cache.put(key, value, **kw) + + def get(self, key, **kw): + cache, kw = self._get_cache(**kw) + return cache.get(key, **kw) + + def invalidate(self, key, **kw): + cache, kw = self._get_cache(**kw) + cache.remove_value(key, **kw) diff --git a/mako/ext/preprocessors.py b/mako/ext/preprocessors.py index 2c0d9935..fcc55007 100644 --- a/mako/ext/preprocessors.py +++ b/mako/ext/preprocessors.py @@ -1,5 +1,5 @@ # ext/preprocessors.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/mako/ext/pygmentplugin.py b/mako/ext/pygmentplugin.py index 0ce57c47..773f47a7 100644 --- a/mako/ext/pygmentplugin.py +++ b/mako/ext/pygmentplugin.py @@ -1,23 +1,19 @@ # ext/pygmentplugin.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -import re -try: - set -except NameError: - from sets import Set as set - from pygments.lexers.web import \ HtmlLexer, XmlLexer, JavascriptLexer, CssLexer -from pygments.lexers.agile import PythonLexer -from pygments.lexer import Lexer, DelegatingLexer, RegexLexer, bygroups, \ - include, using, this -from pygments.token import Error, Punctuation, \ - Text, Comment, Operator, Keyword, Name, String, Number, Other, Literal -from pygments.util import html_doctype_matches, looks_like_xml +from pygments.lexers.agile import PythonLexer, Python3Lexer +from pygments.lexer import DelegatingLexer, RegexLexer, bygroups, \ + include, using +from pygments.token import \ + Text, Comment, Operator, Keyword, Name, String, Other +from pygments.formatters.html import HtmlFormatter +from pygments import highlight +from mako import util class MakoLexer(RegexLexer): name = 'Mako' @@ -30,13 +26,16 @@ class MakoLexer(RegexLexer): bygroups(Text, Comment.Preproc, Keyword, Other)), (r'(\s*)(\%(?!%))([^\n]*)(\n|\Z)', bygroups(Text, Comment.Preproc, using(PythonLexer), Other)), - (r'(\s*)(##[^\n]*)(\n|\Z)', + (r'(\s*)(##[^\n]*)(\n|\Z)', bygroups(Text, Comment.Preproc, Other)), - (r'''(?s)<%doc>.*?''', Comment.Preproc), - (r'(<%)([\w\.\:]+)', bygroups(Comment.Preproc, Name.Builtin), 'tag'), - (r'()', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), + (r'''(?s)<%doc>.*?''', Comment.Preproc), + (r'(<%)([\w\.\:]+)', + bygroups(Comment.Preproc, Name.Builtin), 'tag'), + (r'()', + bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), (r'<%(?=([\w\.\:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), + (r'(<%(?:!?))(.*?)(%>)(?s)', + bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'(\$\{)(.*?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'''(?sx) @@ -105,3 +104,19 @@ class MakoCssLexer(DelegatingLexer): def __init__(self, **options): super(MakoCssLexer, self).__init__(CssLexer, MakoLexer, **options) + + +pygments_html_formatter = HtmlFormatter(cssclass='syntax-highlighted', + linenos=True) +def syntax_highlight(filename='', language=None): + mako_lexer = MakoLexer() + if util.py3k: + python_lexer = Python3Lexer() + else: + python_lexer = PythonLexer() + if filename.startswith('memory:') or language == 'mako': + return lambda string: highlight(string, mako_lexer, + pygments_html_formatter) + return lambda string: highlight(string, python_lexer, + pygments_html_formatter) + diff --git a/mako/ext/turbogears.py b/mako/ext/turbogears.py index f7822eea..e453ada1 100644 --- a/mako/ext/turbogears.py +++ b/mako/ext/turbogears.py @@ -1,5 +1,5 @@ # ext/turbogears.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -39,7 +39,8 @@ class TGPlugin(object): return Template(template_string, **self.tmpl_options) # Translate TG dot notation to normal / template path if '/' not in templatename: - templatename = '/' + templatename.replace('.', '/') + '.' + self.extension + templatename = '/' + templatename.replace('.', '/') + '.' +\ + self.extension # Lookup template return self.lookup.get_template(templatename) diff --git a/mako/filters.py b/mako/filters.py index 30c792f2..37c8fe4c 100644 --- a/mako/filters.py +++ b/mako/filters.py @@ -1,5 +1,5 @@ # mako/filters.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -11,10 +11,10 @@ from mako import util xml_escapes = { '&' : '&', - '>' : '>', - '<' : '<', + '>' : '>', + '<' : '<', '"' : '"', # also " in html-only - "'" : ''' # also ' in html-only + "'" : ''' # also ' in html-only } # XXX: " is valid in HTML and XML @@ -31,7 +31,7 @@ try: except ImportError: html_escape = legacy_html_escape - + def xml_escape(string): return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string) @@ -61,14 +61,14 @@ class Decode(object): return unicode(x, encoding=key) return decode decode = Decode() - - + + _ASCII_re = re.compile(r'\A[\x00-\x7f]*\Z') def is_ascii_str(text): return isinstance(text, str) and _ASCII_re.match(text) -################################################################ +################################################################ class XMLEntityEscaper(object): def __init__(self, codepoint2name, name2codepoint): @@ -115,7 +115,7 @@ class XMLEntityEscaper(object): | ( (?!\d) [:\w] [-.:\w]+ ) ) ;''', re.X | re.UNICODE) - + def __unescape(self, m): dval, hval, name = m.groups() if dval: @@ -128,7 +128,7 @@ class XMLEntityEscaper(object): if codepoint < 128: return chr(codepoint) return unichr(codepoint) - + def unescape(self, text): """Unescape character references. @@ -165,7 +165,8 @@ def htmlentityreplace_errors(ex): codecs.register_error('htmlentityreplace', htmlentityreplace_errors) -# TODO: options to make this dynamic per-compilation will be added in a later release +# TODO: options to make this dynamic per-compilation will be added in a later +# release DEFAULT_ESCAPES = { 'x':'filters.xml_escape', 'h':'filters.html_escape', diff --git a/mako/lexer.py b/mako/lexer.py index cf06bb52..267c0d13 100644 --- a/mako/lexer.py +++ b/mako/lexer.py @@ -1,5 +1,5 @@ # mako/lexer.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -13,8 +13,8 @@ from mako.pygen import adjust_whitespace _regexp_cache = {} class Lexer(object): - def __init__(self, text, filename=None, - disable_unicode=False, + def __init__(self, text, filename=None, + disable_unicode=False, input_encoding=None, preprocessor=None): self.text = text self.filename = filename @@ -25,31 +25,32 @@ class Lexer(object): self.match_position = 0 self.tag = [] self.control_line = [] + self.ternary_stack = [] self.disable_unicode = disable_unicode self.encoding = input_encoding - + if util.py3k and disable_unicode: raise exceptions.UnsupportedError( "Mako for Python 3 does not " "support disabling Unicode") - + if preprocessor is None: self.preprocessor = [] elif not hasattr(preprocessor, '__iter__'): self.preprocessor = [preprocessor] else: self.preprocessor = preprocessor - + @property def exception_kwargs(self): - return {'source':self.text, - 'lineno':self.matched_lineno, - 'pos':self.matched_charpos, + return {'source':self.text, + 'lineno':self.matched_lineno, + 'pos':self.matched_charpos, 'filename':self.filename} - + def match(self, regexp, flags=None): """compile the given regexp, cache the reg, and call match_reg().""" - + try: reg = _regexp_cache[(regexp, flags)] except KeyError: @@ -58,14 +59,15 @@ class Lexer(object): else: reg = re.compile(regexp) _regexp_cache[(regexp, flags)] = reg - + return self.match_reg(reg) - + def match_reg(self, reg): - """match the given regular expression object to the current text position. - + """match the given regular expression object to the current text + position. + if a match occurs, update the current text and line position. - + """ mp = self.match_position @@ -84,39 +86,43 @@ class Lexer(object): cp -=1 self.matched_charpos = mp - cp self.lineno += len(lines) - #print "MATCHED:", match.group(0), "LINE START:", + #print "MATCHED:", match.group(0), "LINE START:", # self.matched_lineno, "LINE END:", self.lineno - #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], (match and "TRUE" or "FALSE") + #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], \ + # (match and "TRUE" or "FALSE") return match - + def parse_until_text(self, *text): startpos = self.match_position + text_re = r'|'.join(text) + brace_level = 0 while True: match = self.match(r'#.*\n') if match: continue - match = self.match(r'(\"\"\"|\'\'\'|\"|\')') + match = self.match(r'(\"\"\"|\'\'\'|\"|\')((? 0: + brace_level -= 1 + continue + return \ + self.text[startpos:\ + self.match_position-len(match.group(1))],\ + match.group(1) + match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) + if match: + brace_level += match.group(1).count('{') + brace_level -= match.group(1).count('}') + continue + raise exceptions.SyntaxException( + "Expected: %s" % + ','.join(text), + **self.exception_kwargs) + def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault('source', self.text) kwargs.setdefault('lineno', self.matched_lineno) @@ -127,6 +133,17 @@ class Lexer(object): self.tag[-1].nodes.append(node) else: self.template.nodes.append(node) + # build a set of child nodes for the control line + # (used for loop variable detection) + # also build a set of child nodes on ternary control lines + # (used for determining if a pass needs to be auto-inserted + if self.control_line: + control_frame = self.control_line[-1] + control_frame.nodes.append(node) + if not (isinstance(node, parsetree.ControlLine) and + control_frame.is_ternary(node.keyword)): + if self.ternary_stack and self.ternary_stack[-1]: + self.ternary_stack[-1][-1].nodes.append(node) if isinstance(node, parsetree.Tag): if len(self.tag): node.parent = self.tag[-1] @@ -134,14 +151,19 @@ class Lexer(object): elif isinstance(node, parsetree.ControlLine): if node.isend: self.control_line.pop() + self.ternary_stack.pop() elif node.is_primary: self.control_line.append(node) - elif len(self.control_line) and \ + self.ternary_stack.append([]) + elif self.control_line and \ + self.control_line[-1].is_ternary(node.keyword): + self.ternary_stack[-1].append(node) + elif self.control_line and \ not self.control_line[-1].is_ternary(node.keyword): raise exceptions.SyntaxException( - "Keyword '%s' not a legal ternary for keyword '%s'" % - (node.keyword, self.control_line[-1].keyword), - **self.exception_kwargs) + "Keyword '%s' not a legal ternary for keyword '%s'" % + (node.keyword, self.control_line[-1].keyword), + **self.exception_kwargs) _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n') @@ -163,8 +185,8 @@ class Lexer(object): if m is not None and m.group(1) != 'utf-8': raise exceptions.CompileException( "Found utf-8 BOM in file, with conflicting " - "magic encoding comment of '%s'" % m.group(1), - text.decode('utf-8', 'ignore'), + "magic encoding comment of '%s'" % m.group(1), + text.decode('utf-8', 'ignore'), 0, 0, filename) else: m = self._coding_re.match(text.decode('utf-8', 'ignore')) @@ -178,32 +200,32 @@ class Lexer(object): text = text.decode(parsed_encoding) except UnicodeDecodeError, e: raise exceptions.CompileException( - "Unicode decode operation of encoding '%s' failed" % - parsed_encoding, - text.decode('utf-8', 'ignore'), - 0, 0, filename) + "Unicode decode operation of encoding '%s' failed" % + parsed_encoding, + text.decode('utf-8', 'ignore'), + 0, 0, filename) return parsed_encoding, text def parse(self): - self.encoding, self.text = self.decode_raw_stream(self.text, - not self.disable_unicode, + self.encoding, self.text = self.decode_raw_stream(self.text, + not self.disable_unicode, self.encoding, self.filename,) for preproc in self.preprocessor: self.text = preproc(self.text) - - # push the match marker past the + + # push the match marker past the # encoding comment. self.match_reg(self._coding_re) - + self.textlength = len(self.text) - + while (True): - if self.match_position > self.textlength: + if self.match_position > self.textlength: break - + if self.match_end(): break if self.match_expression(): @@ -212,53 +234,56 @@ class Lexer(object): continue if self.match_comment(): continue - if self.match_tag_start(): + if self.match_tag_start(): continue if self.match_tag_end(): continue if self.match_python_block(): continue - if self.match_text(): + if self.match_text(): continue - - if self.match_position > self.textlength: + + if self.match_position > self.textlength: break raise exceptions.CompileException("assertion failed") - + if len(self.tag): - raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % - self.tag[-1].keyword, + raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % + self.tag[-1].keyword, **self.exception_kwargs) if len(self.control_line): - raise exceptions.SyntaxException("Unterminated control keyword: '%s'" % - self.control_line[-1].keyword, - self.text, - self.control_line[-1].lineno, - self.control_line[-1].pos, self.filename) + raise exceptions.SyntaxException( + "Unterminated control keyword: '%s'" % + self.control_line[-1].keyword, + self.text, + self.control_line[-1].lineno, + self.control_line[-1].pos, self.filename) return self.template def match_tag_start(self): match = self.match(r''' \<% # opening tag - + ([\w\.\:]+) # keyword - - ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = sign, string expression - + + ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = \ + # sign, string expression + \s* # more whitespace - + (/)?> # closing - - ''', - + + ''', + re.I | re.S | re.X) - + if match: - keyword, attr, isend = match.group(1), match.group(2), match.group(3) + keyword, attr, isend = match.groups() self.keyword = keyword attributes = {} if attr: - for att in re.findall(r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr): + for att in re.findall( + r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr): key, val1, val2 = att text = val1 or val2 text = text.replace('\r\n', '\n') @@ -271,33 +296,33 @@ class Lexer(object): match = self.match(r'(.*?)(?=\)', re.S) if not match: raise exceptions.SyntaxException( - "Unclosed tag: <%%%s>" % - self.tag[-1].keyword, + "Unclosed tag: <%%%s>" % + self.tag[-1].keyword, **self.exception_kwargs) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True - else: + else: return False - + def match_tag_end(self): match = self.match(r'\') if match: if not len(self.tag): raise exceptions.SyntaxException( - "Closing tag without opening tag: " % - match.group(1), - **self.exception_kwargs) + "Closing tag without opening tag: " % + match.group(1), + **self.exception_kwargs) elif self.tag[-1].keyword != match.group(1): raise exceptions.SyntaxException( - "Closing tag does not match tag: <%%%s>" % - (match.group(1), self.tag[-1].keyword), - **self.exception_kwargs) + "Closing tag does not match tag: <%%%s>" % + (match.group(1), self.tag[-1].keyword), + **self.exception_kwargs) self.tag.pop() return True else: return False - + def match_end(self): match = self.match(r'\Z', re.S) if match: @@ -308,13 +333,13 @@ class Lexer(object): return True else: return False - + def match_text(self): match = self.match(r""" (.*?) # anything, followed by: ( - (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based - # comment preceded by a + (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based + # comment preceded by a # consumed newline and whitespace | (?=\${) # an expression @@ -328,7 +353,7 @@ class Lexer(object): | \Z # end of string )""", re.X | re.S) - + if match: text = match.group(1) if text: @@ -336,23 +361,23 @@ class Lexer(object): return True else: return False - + def match_python_block(self): match = self.match(r"<%(!)?") if match: line, pos = self.matched_lineno, self.matched_charpos text, end = self.parse_until_text(r'%>') - # the trailing newline helps + # the trailing newline helps # compiler.parse() not complain about indentation - text = adjust_whitespace(text) + "\n" + text = adjust_whitespace(text) + "\n" self.append_node( - parsetree.Code, - text, + parsetree.Code, + text, match.group(1)=='!', lineno=line, pos=pos) return True else: return False - + def match_expression(self): match = self.match(r"\${") if match: @@ -364,15 +389,17 @@ class Lexer(object): escapes = "" text = text.replace('\r\n', '\n') self.append_node( - parsetree.Expression, - text, escapes.strip(), + parsetree.Expression, + text, escapes.strip(), lineno=line, pos=pos) return True else: return False def match_control_line(self): - match = self.match(r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)(?:\r?\n|\Z)", re.M) + match = self.match( + r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)" + r"(?:\r?\n|\Z)", re.M) if match: operator = match.group(1) text = match.group(2) @@ -380,22 +407,22 @@ class Lexer(object): m2 = re.match(r'(end)?(\w+)\s*(.*)', text) if not m2: raise exceptions.SyntaxException( - "Invalid control line: '%s'" % - text, + "Invalid control line: '%s'" % + text, **self.exception_kwargs) isend, keyword = m2.group(1, 2) isend = (isend is not None) - + if isend: if not len(self.control_line): raise exceptions.SyntaxException( - "No starting keyword '%s' for '%s'" % - (keyword, text), + "No starting keyword '%s' for '%s'" % + (keyword, text), **self.exception_kwargs) elif self.control_line[-1].keyword != keyword: raise exceptions.SyntaxException( - "Keyword '%s' doesn't match keyword '%s'" % - (text, self.control_line[-1].keyword), + "Keyword '%s' doesn't match keyword '%s'" % + (text, self.control_line[-1].keyword), **self.exception_kwargs) self.append_node(parsetree.ControlLine, keyword, isend, text) else: @@ -412,4 +439,4 @@ class Lexer(object): return True else: return False - + diff --git a/mako/lookup.py b/mako/lookup.py index b397d21f..4d86696a 100644 --- a/mako/lookup.py +++ b/mako/lookup.py @@ -1,5 +1,5 @@ # mako/lookup.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,30 +12,30 @@ try: import threading except: import dummy_threading as threading - + class TemplateCollection(object): - """Represent a collection of :class:`.Template` objects, - identifiable via uri. - + """Represent a collection of :class:`.Template` objects, + identifiable via URI. + A :class:`.TemplateCollection` is linked to the usage of - all template tags that address other templates, such + all template tags that address other templates, such as ``<%include>``, ``<%namespace>``, and ``<%inherit>``. The ``file`` attribute of each of those tags refers to a string URI that is passed to that :class:`.Template` object's :class:`.TemplateCollection` for resolution. - + :class:`.TemplateCollection` is an abstract class, with the usual default implementation being :class:`.TemplateLookup`. - + """ def has_template(self, uri): """Return ``True`` if this :class:`.TemplateLookup` is capable of returning a :class:`.Template` object for the - given URL. + given ``uri``. + + :param uri: String URI of the template to be resolved. - :param uri: String uri of the template to be resolved. - """ try: self.get_template(uri) @@ -44,124 +44,135 @@ class TemplateCollection(object): return False def get_template(self, uri, relativeto=None): - """Return a :class:`.Template` object corresponding to the given - URL. - + """Return a :class:`.Template` object corresponding to the given + ``uri``. + The default implementation raises :class:`.NotImplementedError`. Implementations should - raise :class:`.TemplateLookupException` if the given uri + raise :class:`.TemplateLookupException` if the given ``uri`` cannot be resolved. - - :param uri: String uri of the template to be resolved. - :param relativeto: if present, the given URI is assumed to - be relative to this uri. - + + :param uri: String URI of the template to be resolved. + :param relativeto: if present, the given ``uri`` is assumed to + be relative to this URI. + """ raise NotImplementedError() def filename_to_uri(self, uri, filename): - """Convert the given filename to a uri relative to - this TemplateCollection.""" - + """Convert the given ``filename`` to a URI relative to + this :class:`.TemplateCollection`.""" + return uri - + def adjust_uri(self, uri, filename): - """Adjust the given uri based on the calling filename. - + """Adjust the given ``uri`` based on the calling ``filename``. + When this method is called from the runtime, the - 'filename' parameter is taken directly to the 'filename' + ``filename`` parameter is taken directly to the ``filename`` attribute of the calling template. Therefore a custom - TemplateCollection subclass can place any string - identifier desired in the "filename" parameter of the - Template objects it constructs and have them come back + :class:`.TemplateCollection` subclass can place any string + identifier desired in the ``filename`` parameter of the + :class:`.Template` objects it constructs and have them come back here. - + """ return uri - + class TemplateLookup(TemplateCollection): """Represent a collection of templates that locates template source files from the local filesystem. - + The primary argument is the ``directories`` argument, the list of - directories to search:: - + directories to search: + + .. sourcecode:: python + lookup = TemplateLookup(["/path/to/templates"]) some_template = lookup.get_template("/index.html") - + The :class:`.TemplateLookup` can also be given :class:`.Template` objects - programatically using :meth:`.put_string` or :meth:`.put_template`:: - + programatically using :meth:`.put_string` or :meth:`.put_template`: + + .. sourcecode:: python + lookup = TemplateLookup() lookup.put_string("base.html", ''' ${self.next()} ''') lookup.put_string("hello.html", ''' <%include file='base.html'/> - + Hello, world ! ''') - - - :param directories: A list of directory names which will be + + + :param directories: A list of directory names which will be searched for a particular template URI. The URI is appended to each directory and the filesystem checked. - - :param collection_size: Approximate size of the collection used - to store templates. If left at its default of -1, the size + + :param collection_size: Approximate size of the collection used + to store templates. If left at its default of ``-1``, the size is unbounded, and a plain Python dictionary is used to relate URI strings to :class:`.Template` instances. Otherwise, a least-recently-used cache object is used which will maintain the size of the collection approximately to the number given. - - :param filesystem_checks: When at its default value of ``True``, - each call to :meth:`TemplateLookup.get_template()` will + + :param filesystem_checks: When at its default value of ``True``, + each call to :meth:`.TemplateLookup.get_template()` will compare the filesystem last modified time to the time in which an existing :class:`.Template` object was created. This allows the :class:`.TemplateLookup` to regenerate a new :class:`.Template` whenever the original source has been updated. Set this to ``False`` for a very minor performance increase. - - :param modulename_callable: A callable which, when present, + + :param modulename_callable: A callable which, when present, is passed the path of the source file as well as the requested URI, and then returns the full path of the generated Python module file. This is used to inject - alternate schemes for Pyhton module location. If left at + alternate schemes for Python module location. If left at its default of ``None``, the built in system of generation based on ``module_directory`` plus ``uri`` is used. - + All other keyword parameters available for :class:`.Template` are mirrored here. When new :class:`.Template` objects are created, the keywords established with this :class:`.TemplateLookup` are passed on to each new :class:`.Template`. - + """ - - def __init__(self, - directories=None, - module_directory=None, - filesystem_checks=True, - collection_size=-1, - format_exceptions=False, - error_handler=None, - disable_unicode=False, + + def __init__(self, + directories=None, + module_directory=None, + filesystem_checks=True, + collection_size=-1, + format_exceptions=False, + error_handler=None, + disable_unicode=False, bytestring_passthrough=False, - output_encoding=None, - encoding_errors='strict', - cache_type=None, - cache_dir=None, cache_url=None, - cache_enabled=True, - modulename_callable=None, - default_filters=None, - buffer_filters=(), + output_encoding=None, + encoding_errors='strict', + + cache_args=None, + cache_impl='beaker', + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + + modulename_callable=None, + module_writer=None, + default_filters=None, + buffer_filters=(), strict_undefined=False, - imports=None, - input_encoding=None, + imports=None, + enable_loop=True, + input_encoding=None, preprocessor=None): - + self.directories = [posixpath.normpath(d) for d in util.to_list(directories, ()) ] @@ -170,23 +181,34 @@ class TemplateLookup(TemplateCollection): self.filesystem_checks = filesystem_checks self.collection_size = collection_size + if cache_args is None: + cache_args = {} + # transfer deprecated cache_* args + if cache_dir: + cache_args.setdefault('dir', cache_dir) + if cache_url: + cache_args.setdefault('url', cache_url) + if cache_type: + cache_args.setdefault('type', cache_type) + self.template_args = { - 'format_exceptions':format_exceptions, - 'error_handler':error_handler, - 'disable_unicode':disable_unicode, + 'format_exceptions':format_exceptions, + 'error_handler':error_handler, + 'disable_unicode':disable_unicode, 'bytestring_passthrough':bytestring_passthrough, - 'output_encoding':output_encoding, - 'encoding_errors':encoding_errors, - 'input_encoding':input_encoding, - 'module_directory':module_directory, - 'cache_type':cache_type, - 'cache_dir':cache_dir or module_directory, - 'cache_url':cache_url, - 'cache_enabled':cache_enabled, - 'default_filters':default_filters, - 'buffer_filters':buffer_filters, + 'output_encoding':output_encoding, + 'cache_impl':cache_impl, + 'encoding_errors':encoding_errors, + 'input_encoding':input_encoding, + 'module_directory':module_directory, + 'module_writer':module_writer, + 'cache_args':cache_args, + 'cache_enabled':cache_enabled, + 'default_filters':default_filters, + 'buffer_filters':buffer_filters, 'strict_undefined':strict_undefined, - 'imports':imports, + 'imports':imports, + 'enable_loop':enable_loop, 'preprocessor':preprocessor} if collection_size == -1: @@ -196,15 +218,15 @@ class TemplateLookup(TemplateCollection): self._collection = util.LRUCache(collection_size) self._uri_cache = util.LRUCache(collection_size) self._mutex = threading.Lock() - + def get_template(self, uri): - """Return a :class:`.Template` object corresponding to the given - URL. - - Note the "relativeto" argument is not supported here at the moment. - + """Return a :class:`.Template` object corresponding to the given + ``uri``. + + .. note:: The ``relativeto`` argument is not supported here at the moment. + """ - + try: if self.filesystem_checks: return self._check(uri, self._collection[uri]) @@ -221,25 +243,26 @@ class TemplateLookup(TemplateCollection): "Cant locate template for uri %r" % uri) def adjust_uri(self, uri, relativeto): - """adjust the given uri based on the given relative uri.""" - + """Adjust the given ``uri`` based on the given relative URI.""" + key = (uri, relativeto) if key in self._uri_cache: return self._uri_cache[key] if uri[0] != '/': if relativeto is not None: - v = self._uri_cache[key] = posixpath.join(posixpath.dirname(relativeto), uri) + v = self._uri_cache[key] = posixpath.join( + posixpath.dirname(relativeto), uri) else: v = self._uri_cache[key] = '/' + uri else: v = self._uri_cache[key] = uri return v - - + + def filename_to_uri(self, filename): - """Convert the given filename to a uri relative to - this TemplateCollection.""" + """Convert the given ``filename`` to a URI relative to + this :class:`.TemplateCollection`.""" try: return self._uri_cache[filename] @@ -247,25 +270,25 @@ class TemplateLookup(TemplateCollection): value = self._relativeize(filename) self._uri_cache[filename] = value return value - + def _relativeize(self, filename): - """Return the portion of a filename that is 'relative' + """Return the portion of a filename that is 'relative' to the directories in this lookup. - + """ - + filename = posixpath.normpath(filename) for dir in self.directories: if filename[0:len(dir)] == dir: return filename[len(dir):] else: return None - + def _load(self, filename, uri): self._mutex.acquire() try: try: - # try returning from collection one + # try returning from collection one # more time in case concurrent thread already loaded return self._collection[uri] except KeyError: @@ -278,19 +301,19 @@ class TemplateLookup(TemplateCollection): self._collection[uri] = template = Template( uri=uri, filename=posixpath.normpath(filename), - lookup=self, + lookup=self, module_filename=module_filename, **self.template_args) return template except: - # if compilation fails etc, ensure + # if compilation fails etc, ensure # template is removed from collection, # re-raise self._collection.pop(uri, None) raise finally: self._mutex.release() - + def _check(self, uri, template): if template.filename is None: return template @@ -308,24 +331,24 @@ class TemplateLookup(TemplateCollection): raise exceptions.TemplateLookupException( "Cant locate template for uri %r" % uri) - + def put_string(self, uri, text): """Place a new :class:`.Template` object into this :class:`.TemplateLookup`, based on the given string of - text. - + ``text``. + """ self._collection[uri] = Template( - text, - lookup=self, - uri=uri, + text, + lookup=self, + uri=uri, **self.template_args) - + def put_template(self, uri, template): """Place a new :class:`.Template` object into this :class:`.TemplateLookup`, based on the given :class:`.Template` object. - + """ self._collection[uri] = template - + diff --git a/mako/parsetree.py b/mako/parsetree.py index 31b9b4f0..ecd82425 100644 --- a/mako/parsetree.py +++ b/mako/parsetree.py @@ -1,5 +1,5 @@ # mako/parsetree.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -11,117 +11,125 @@ import re class Node(object): """base class for a Node in the parse tree.""" + def __init__(self, source, lineno, pos, filename): self.source = source self.lineno = lineno self.pos = pos self.filename = filename - + @property def exception_kwargs(self): - return {'source':self.source, 'lineno':self.lineno, + return {'source':self.source, 'lineno':self.lineno, 'pos':self.pos, 'filename':self.filename} - + def get_children(self): return [] - + def accept_visitor(self, visitor): def traverse(node): for n in node.get_children(): n.accept_visitor(visitor) + method = getattr(visitor, "visit" + self.__class__.__name__, traverse) method(self) class TemplateNode(Node): """a 'container' node that stores the overall collection of nodes.""" - + def __init__(self, filename): super(TemplateNode, self).__init__('', 0, 0, filename) self.nodes = [] self.page_attributes = {} - + def get_children(self): return self.nodes - + def __repr__(self): return "TemplateNode(%s, %r)" % ( - util.sorted_dict_repr(self.page_attributes), + util.sorted_dict_repr(self.page_attributes), self.nodes) - + class ControlLine(Node): """defines a control line, a line-oriented python line or end tag. - + e.g.:: % if foo: (markup) % endif - + """ + has_loop_context = False + def __init__(self, keyword, isend, text, **kwargs): super(ControlLine, self).__init__(**kwargs) self.text = text self.keyword = keyword self.isend = isend - self.is_primary = keyword in ['for','if', 'while', 'try'] + self.is_primary = keyword in ['for', 'if', 'while', 'try', 'with'] + self.nodes = [] if self.isend: self._declared_identifiers = [] self._undeclared_identifiers = [] else: code = ast.PythonFragment(text, **self.exception_kwargs) - self._declared_identifiers = code.declared_identifiers + self._declared_identifiers = code.declared_identifiers self._undeclared_identifiers = code.undeclared_identifiers + def get_children(self): + return self.nodes + def declared_identifiers(self): return self._declared_identifiers def undeclared_identifiers(self): return self._undeclared_identifiers - + def is_ternary(self, keyword): """return true if the given keyword is a ternary keyword for this ControlLine""" - + return keyword in { 'if':set(['else', 'elif']), 'try':set(['except', 'finally']), 'for':set(['else']) }.get(self.keyword, []) - + def __repr__(self): return "ControlLine(%r, %r, %r, %r)" % ( - self.keyword, - self.text, - self.isend, + self.keyword, + self.text, + self.isend, (self.lineno, self.pos) ) class Text(Node): """defines plain text in the template.""" - + def __init__(self, content, **kwargs): super(Text, self).__init__(**kwargs) self.content = content - + def __repr__(self): return "Text(%r, %r)" % (self.content, (self.lineno, self.pos)) - + class Code(Node): """defines a Python code block, either inline or module level. - + e.g.:: inline: <% x = 12 %> - + module level: <%! import logger %> - + """ def __init__(self, text, ismodule, **kwargs): @@ -138,32 +146,32 @@ class Code(Node): def __repr__(self): return "Code(%r, %r, %r)" % ( - self.text, - self.ismodule, + self.text, + self.ismodule, (self.lineno, self.pos) ) - + class Comment(Node): """defines a comment line. - + # this is a comment - + """ - + def __init__(self, text, **kwargs): super(Comment, self).__init__(**kwargs) self.text = text def __repr__(self): return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos)) - + class Expression(Node): """defines an inline expression. - + ${x+y} - + """ - + def __init__(self, text, escapes, **kwargs): super(Expression, self).__init__(**kwargs) self.text = text @@ -184,74 +192,74 @@ class Expression(Node): def __repr__(self): return "Expression(%r, %r, %r)" % ( - self.text, - self.escapes_code.args, + self.text, + self.escapes_code.args, (self.lineno, self.pos) ) - + class _TagMeta(type): """metaclass to allow Tag to produce a subclass according to its keyword""" - + _classmap = {} - + def __init__(cls, clsname, bases, dict): if cls.__keyword__ is not None: cls._classmap[cls.__keyword__] = cls super(_TagMeta, cls).__init__(clsname, bases, dict) - + def __call__(cls, keyword, attributes, **kwargs): if ":" in keyword: ns, defname = keyword.split(':') - return type.__call__(CallNamespaceTag, ns, defname, + return type.__call__(CallNamespaceTag, ns, defname, attributes, **kwargs) try: cls = _TagMeta._classmap[keyword] except KeyError: raise exceptions.CompileException( - "No such tag: '%s'" % keyword, - source=kwargs['source'], - lineno=kwargs['lineno'], - pos=kwargs['pos'], + "No such tag: '%s'" % keyword, + source=kwargs['source'], + lineno=kwargs['lineno'], + pos=kwargs['pos'], filename=kwargs['filename'] ) return type.__call__(cls, keyword, attributes, **kwargs) - + class Tag(Node): """abstract base class for tags. - + <%sometag/> - + <%someothertag> stuff - + """ - + __metaclass__ = _TagMeta __keyword__ = None - - def __init__(self, keyword, attributes, expressions, + + def __init__(self, keyword, attributes, expressions, nonexpressions, required, **kwargs): """construct a new Tag instance. - + this constructor not called directly, and is only called by subclasses. - + :param keyword: the tag keyword - + :param attributes: raw dictionary of attribute key/value pairs - - :param expressions: a set of identifiers that are legal attributes, + + :param expressions: a set of identifiers that are legal attributes, which can also contain embedded expressions - - :param nonexpressions: a set of identifiers that are legal + + :param nonexpressions: a set of identifiers that are legal attributes, which cannot contain embedded expressions - + :param \**kwargs: other arguments passed to the Node superclass (lineno, pos) - + """ super(Tag, self).__init__(**kwargs) self.keyword = keyword @@ -260,18 +268,18 @@ class Tag(Node): missing = [r for r in required if r not in self.parsed_attributes] if len(missing): raise exceptions.CompileException( - "Missing attribute(s): %s" % - ",".join([repr(m) for m in missing]), + "Missing attribute(s): %s" % + ",".join([repr(m) for m in missing]), **self.exception_kwargs) self.parent = None self.nodes = [] - + def is_root(self): return self.parent is None - + def get_children(self): return self.nodes - + def _parse_attributes(self, expressions, nonexpressions): undeclared_identifiers = set() self.parsed_attributes = {} @@ -285,8 +293,8 @@ class Tag(Node): code = ast.PythonCode(m.group(1).rstrip(), **self.exception_kwargs) # we aren't discarding "declared_identifiers" here, - # which we do so that list comprehension-declared - # variables aren't counted. As yet can't find a + # which we do so that list comprehension-declared + # variables aren't counted. As yet can't find a # condition that requires it here. undeclared_identifiers = \ undeclared_identifiers.union( @@ -299,14 +307,14 @@ class Tag(Node): elif key in nonexpressions: if re.search(r'\${.+?}', self.attributes[key]): raise exceptions.CompileException( - "Attibute '%s' in tag '%s' does not allow embedded " - "expressions" % (key, self.keyword), - **self.exception_kwargs) + "Attibute '%s' in tag '%s' does not allow embedded " + "expressions" % (key, self.keyword), + **self.exception_kwargs) self.parsed_attributes[key] = repr(self.attributes[key]) else: raise exceptions.CompileException( "Invalid attribute for tag '%s': '%s'" % - (self.keyword, key), + (self.keyword, key), **self.exception_kwargs) self.expression_undeclared_identifiers = undeclared_identifiers @@ -317,21 +325,21 @@ class Tag(Node): return self.expression_undeclared_identifiers def __repr__(self): - return "%s(%r, %s, %r, %r)" % (self.__class__.__name__, - self.keyword, + return "%s(%r, %s, %r, %r)" % (self.__class__.__name__, + self.keyword, util.sorted_dict_repr(self.attributes), - (self.lineno, self.pos), + (self.lineno, self.pos), self.nodes ) - + class IncludeTag(Tag): __keyword__ = 'include' def __init__(self, keyword, attributes, **kwargs): super(IncludeTag, self).__init__( - keyword, - attributes, - ('file', 'import', 'args'), + keyword, + attributes, + ('file', 'import', 'args'), (), ('file',), **kwargs) self.page_args = ast.PythonCode( "__DUMMY(%s)" % attributes.get('args', ''), @@ -346,18 +354,18 @@ class IncludeTag(Tag): difference(self.page_args.declared_identifiers) return identifiers.union(super(IncludeTag, self). undeclared_identifiers()) - + class NamespaceTag(Tag): __keyword__ = 'namespace' def __init__(self, keyword, attributes, **kwargs): super(NamespaceTag, self).__init__( - keyword, attributes, - ('file',), + keyword, attributes, + ('file',), ('name','inheritable', - 'import','module'), + 'import','module'), (), **kwargs) - + self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self)))) if not 'name' in attributes and not 'import' in attributes: raise exceptions.CompileException( @@ -378,36 +386,39 @@ class TextTag(Tag): def __init__(self, keyword, attributes, **kwargs): super(TextTag, self).__init__( - keyword, - attributes, (), + keyword, + attributes, (), ('filter'), (), **kwargs) self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), + attributes.get('filter', ''), **self.exception_kwargs) - + class DefTag(Tag): __keyword__ = 'def' def __init__(self, keyword, attributes, **kwargs): + expressions = ['buffered', 'cached'] + [ + c for c in attributes if c.startswith('cache_')] + + super(DefTag, self).__init__( - keyword, - attributes, - ('buffered', 'cached', 'cache_key', 'cache_timeout', - 'cache_type', 'cache_dir', 'cache_url'), - ('name','filter', 'decorator'), - ('name',), + keyword, + attributes, + expressions, + ('name','filter', 'decorator'), + ('name',), **kwargs) name = attributes['name'] if re.match(r'^[\w_]+$',name): raise exceptions.CompileException( - "Missing parenthesis in %def", + "Missing parenthesis in %def", **self.exception_kwargs) - self.function_decl = ast.FunctionDecl("def " + name + ":pass", + self.function_decl = ast.FunctionDecl("def " + name + ":pass", **self.exception_kwargs) self.name = self.function_decl.funcname self.decorator = attributes.get('decorator', '') self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), + attributes.get('filter', ''), **self.exception_kwargs) is_anonymous = False @@ -428,40 +439,47 @@ class DefTag(Tag): for c in self.function_decl.defaults: res += list(ast.PythonCode(c, **self.exception_kwargs). undeclared_identifiers) - return res + list(self.filter_args.\ + return set(res).union( + self.filter_args.\ undeclared_identifiers.\ difference(filters.DEFAULT_ESCAPES.keys()) - ) + ).union( + self.expression_undeclared_identifiers + ).difference( + self.function_decl.argnames + ) class BlockTag(Tag): __keyword__ = 'block' def __init__(self, keyword, attributes, **kwargs): + expressions = ['buffered', 'cached', 'args'] + [ + c for c in attributes if c.startswith('cache_')] + super(BlockTag, self).__init__( - keyword, - attributes, - ('buffered', 'cached', 'cache_key', 'cache_timeout', - 'cache_type', 'cache_dir', 'cache_url', 'args'), - ('name','filter', 'decorator'), - (), + keyword, + attributes, + expressions, + ('name','filter', 'decorator'), + (), **kwargs) name = attributes.get('name') if name and not re.match(r'^[\w_]+$',name): raise exceptions.CompileException( - "%block may not specify an argument signature", - **self.exception_kwargs) + "%block may not specify an argument signature", + **self.exception_kwargs) if not name and attributes.get('args', None): raise exceptions.CompileException( "Only named %blocks may specify args", **self.exception_kwargs ) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), + self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs) self.name = name self.decorator = attributes.get('decorator', '') self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), + attributes.get('filter', ''), **self.exception_kwargs) @@ -482,17 +500,22 @@ class BlockTag(Tag): return self.body_decl.argnames def undeclared_identifiers(self): - return [] + return (self.filter_args.\ + undeclared_identifiers.\ + difference(filters.DEFAULT_ESCAPES.keys()) + ).union(self.expression_undeclared_identifiers) + + class CallTag(Tag): __keyword__ = 'call' def __init__(self, keyword, attributes, **kwargs): - super(CallTag, self).__init__(keyword, attributes, + super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs) self.expression = attributes['expr'] self.code = ast.PythonCode(self.expression, **self.exception_kwargs) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), + self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs) def declared_identifiers(self): @@ -506,23 +529,23 @@ class CallNamespaceTag(Tag): def __init__(self, namespace, defname, attributes, **kwargs): super(CallNamespaceTag, self).__init__( - namespace + ":" + defname, - attributes, - tuple(attributes.keys()) + ('args', ), - (), - (), + namespace + ":" + defname, + attributes, + tuple(attributes.keys()) + ('args', ), + (), + (), **kwargs) - + self.expression = "%s.%s(%s)" % ( - namespace, - defname, + namespace, + defname, ",".join(["%s=%s" % (k, v) for k, v in - self.parsed_attributes.iteritems() + self.parsed_attributes.iteritems() if k != 'args']) ) self.code = ast.PythonCode(self.expression, **self.exception_kwargs) self.body_decl = ast.FunctionArgs( - attributes.get('args', ''), + attributes.get('args', ''), **self.exception_kwargs) def declared_identifiers(self): @@ -537,23 +560,24 @@ class InheritTag(Tag): def __init__(self, keyword, attributes, **kwargs): super(InheritTag, self).__init__( - keyword, attributes, + keyword, attributes, ('file',), (), ('file',), **kwargs) class PageTag(Tag): __keyword__ = 'page' def __init__(self, keyword, attributes, **kwargs): + expressions = ['cached', 'args', 'expression_filter', 'enable_loop'] + [ + c for c in attributes if c.startswith('cache_')] + super(PageTag, self).__init__( - keyword, - attributes, - ('cached', 'cache_key', 'cache_timeout', - 'cache_type', 'cache_dir', 'cache_url', - 'args', 'expression_filter'), - (), - (), + keyword, + attributes, + expressions, + (), + (), **kwargs) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), + self.body_decl = ast.FunctionArgs(attributes.get('args', ''), **self.exception_kwargs) self.filter_args = ast.ArgumentList( attributes.get('expression_filter', ''), @@ -561,5 +585,5 @@ class PageTag(Tag): def declared_identifiers(self): return self.body_decl.argnames - - + + diff --git a/mako/pygen.py b/mako/pygen.py index 07f26670..e946de50 100644 --- a/mako/pygen.py +++ b/mako/pygen.py @@ -1,5 +1,5 @@ # mako/pygen.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -14,48 +14,48 @@ class PythonPrinter(object): def __init__(self, stream): # indentation counter self.indent = 0 - - # a stack storing information about why we incremented + + # a stack storing information about why we incremented # the indentation counter, to help us determine if we # should decrement it self.indent_detail = [] - + # the string of whitespace multiplied by the indent # counter to produce a line self.indentstring = " " - + # the stream we are writing to self.stream = stream - + # a list of lines that represents a buffered "block" of code, - # which can be later printed relative to an indent level + # which can be later printed relative to an indent level self.line_buffer = [] - + self.in_indent_lines = False - + self._reset_multi_line_flags() def write(self, text): self.stream.write(text) - + def write_indented_block(self, block): """print a line or lines of python which already contain indentation. - + The indentation of the total block of lines will be adjusted to that of - the current indent level.""" + the current indent level.""" self.in_indent_lines = False for l in re.split(r'\r?\n', block): self.line_buffer.append(l) - + def writelines(self, *lines): """print a series of lines of python.""" for line in lines: self.writeline(line) - + def writeline(self, line): """print a line of python, indenting it according to the current indent level. - + this also adjusts the indentation counter according to the content of the line. @@ -65,9 +65,7 @@ class PythonPrinter(object): self._flush_adjusted_lines() self.in_indent_lines = True - decreased_indent = False - - if (line is None or + if (line is None or re.match(r"^\s*#",line) or re.match(r"^\s*$", line) ): @@ -76,31 +74,30 @@ class PythonPrinter(object): hastext = True is_comment = line and len(line) and line[0] == '#' - + # see if this line should decrease the indentation level - if (not decreased_indent and - not is_comment and + if (not is_comment and (not hastext or self._is_unindentor(line)) ): - - if self.indent > 0: + + if self.indent > 0: self.indent -=1 # if the indent_detail stack is empty, the user # probably put extra closures - the resulting - # module wont compile. - if len(self.indent_detail) == 0: + # module wont compile. + if len(self.indent_detail) == 0: raise exceptions.SyntaxException( "Too many whitespace closures") self.indent_detail.pop() - + if line is None: return - + # write the line self.stream.write(self._indent_line(line) + "\n") - + # see if this line should increase the indentation level. - # note that a line can both decrase (before printing) and + # note that a line can both decrase (before printing) and # then increase (after printing) the indentation level. if re.search(r":[ \t]*(?:#.*)?$", line): @@ -108,7 +105,7 @@ class PythonPrinter(object): # keep track of what the keyword was that indented us, # if it is a python compound statement keyword # where we might have to look for an "unindent" keyword - match = re.match(r"^\s*(if|try|elif|while|for)", line) + match = re.match(r"^\s*(if|try|elif|while|for|with)", line) if match: # its a "compound" keyword, so we will check for "unindentors" indentor = match.group(1) @@ -119,7 +116,8 @@ class PythonPrinter(object): # its not a "compound" keyword. but lets also # test for valid Python keywords that might be indenting us, # else assume its a non-indenting line - m2 = re.match(r"^\s*(def|class|else|elif|except|finally)", line) + m2 = re.match(r"^\s*(def|class|else|elif|except|finally)", + line) if m2: self.indent += 1 self.indent_detail.append(indentor) @@ -127,53 +125,53 @@ class PythonPrinter(object): def close(self): """close this printer, flushing any remaining lines.""" self._flush_adjusted_lines() - + def _is_unindentor(self, line): - """return true if the given line is an 'unindentor', + """return true if the given line is an 'unindentor', relative to the last 'indent' event received. - + """ - + # no indentation detail has been pushed on; return False - if len(self.indent_detail) == 0: + if len(self.indent_detail) == 0: return False indentor = self.indent_detail[-1] - - # the last indent keyword we grabbed is not a + + # the last indent keyword we grabbed is not a # compound statement keyword; return False - if indentor is None: + if indentor is None: return False - + # if the current line doesnt have one of the "unindentor" keywords, # return False match = re.match(r"^\s*(else|elif|except|finally).*\:", line) - if not match: + if not match: return False - + # whitespace matches up, we have a compound indentor, # and this line has an unindentor, this # is probably good enough return True - + # should we decide that its not good enough, heres # more stuff to check. #keyword = match.group(1) - - # match the original indent keyword + + # match the original indent keyword #for crit in [ # (r'if|elif', r'else|elif'), # (r'try', r'except|finally|else'), # (r'while|for', r'else'), #]: - # if re.match(crit[0], indentor) and re.match(crit[1], keyword): + # if re.match(crit[0], indentor) and re.match(crit[1], keyword): # return True - + #return False - + def _indent_line(self, line, stripspace=''): """indent the given line according to the current indent level. - + stripspace is a string of space that will be truncated from the start of the line before indenting.""" @@ -185,7 +183,7 @@ class PythonPrinter(object): or triple-quoted section.""" self.backslashed, self.triplequoted = False, False - + def _in_multi_line(self, line): """return true if the given line is part of a multi-line block, via backslash or triple-quote.""" @@ -195,24 +193,24 @@ class PythonPrinter(object): # guard against the possibility of modifying the space inside of # a literal multiline string with unfortunately placed # whitespace - - current_state = (self.backslashed or self.triplequoted) - + + current_state = (self.backslashed or self.triplequoted) + if re.search(r"\\$", line): self.backslashed = True else: self.backslashed = False - + triples = len(re.findall(r"\"\"\"|\'\'\'", line)) if triples == 1 or triples % 2 != 0: self.triplequoted = not self.triplequoted - + return current_state def _flush_adjusted_lines(self): stripspace = None self._reset_multi_line_flags() - + for entry in self.line_buffer: if self._in_multi_line(entry): self.stream.write(entry + "\n") @@ -221,32 +219,32 @@ class PythonPrinter(object): if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry): stripspace = re.match(r"^([ \t]*)", entry).group(1) self.stream.write(self._indent_line(entry, stripspace) + "\n") - + self.line_buffer = [] self._reset_multi_line_flags() def adjust_whitespace(text): """remove the left-whitespace margin of a block of Python code.""" - + state = [False, False] (backslashed, triplequoted) = (0, 1) def in_multi_line(line): start_state = (state[backslashed] or state[triplequoted]) - + if re.search(r"\\$", line): state[backslashed] = True else: state[backslashed] = False - + def match(reg, t): m = re.match(reg, t) if m: return m, t[len(m.group(0)):] else: return None, t - + while line: if state[triplequoted]: m, line = match(r"%s" % state[triplequoted], line) @@ -258,14 +256,14 @@ def adjust_whitespace(text): m, line = match(r'#', line) if m: return start_state - + m, line = match(r"\"\"\"|\'\'\'", line) if m: state[triplequoted] = m.group(0) continue m, line = match(r".*?(?=\"\"\"|\'\'\'|#|$)", line) - + return start_state def _indent_line(line, stripspace = ''): diff --git a/mako/pyparser.py b/mako/pyparser.py index 953596af..1f39756e 100644 --- a/mako/pyparser.py +++ b/mako/pyparser.py @@ -1,5 +1,5 @@ # mako/pyparser.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -15,17 +15,17 @@ from mako import exceptions, util import operator if util.py3k: - # words that cannot be assigned to (notably + # words that cannot be assigned to (notably # smaller than the total keys in __builtins__) reserved = set(['True', 'False', 'None', 'print']) # the "id" attribute on a function node arg_id = operator.attrgetter('arg') else: - # words that cannot be assigned to (notably + # words that cannot be assigned to (notably # smaller than the total keys in __builtins__) reserved = set(['True', 'False', 'None']) - + # the "id" attribute on a function node arg_id = operator.attrgetter('id') @@ -42,7 +42,7 @@ except ImportError: def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" - + try: if _ast: @@ -54,8 +54,8 @@ def parse(code, mode='exec', **exception_kwargs): except Exception, e: raise exceptions.SyntaxException( "(%s) %s (%r)" % ( - e.__class__.__name__, - e, + e.__class__.__name__, + e, code[0:50] ), **exception_kwargs) @@ -66,13 +66,15 @@ if _ast: def __init__(self, listener, **exception_kwargs): self.in_function = False self.in_assign_targets = False - self.local_ident_stack = {} + self.local_ident_stack = set() self.listener = listener self.exception_kwargs = exception_kwargs def _add_declared(self, name): if not self.in_function: self.listener.declared_identifiers.add(name) + else: + self.local_ident_stack.add(name) def visit_ClassDef(self, node): self._add_declared(node.name) @@ -118,23 +120,20 @@ if _ast: # argument names in each function header so they arent # counted as "undeclared" - saved = {} inf = self.in_function self.in_function = True - for arg in node.args.args: - if arg_id(arg) in self.local_ident_stack: - saved[arg_id(arg)] = True - else: - self.local_ident_stack[arg_id(arg)] = True + + local_ident_stack = self.local_ident_stack + self.local_ident_stack = local_ident_stack.union([ + arg_id(arg) for arg in node.args.args + ]) if islambda: self.visit(node.body) else: for n in node.body: self.visit(n) self.in_function = inf - for arg in node.args.args: - if arg_id(arg) not in saved: - del self.local_ident_stack[arg_id(arg)] + self.local_ident_stack = local_ident_stack def visit_For(self, node): @@ -149,8 +148,10 @@ if _ast: def visit_Name(self, node): if isinstance(node.ctx, _ast.Store): + # this is eqiuvalent to visit_AssName in + # compiler self._add_declared(node.id) - if node.id not in reserved and node.id \ + elif node.id not in reserved and node.id \ not in self.listener.declared_identifiers and node.id \ not in self.local_ident_stack: self.listener.undeclared_identifiers.add(node.id) @@ -228,13 +229,15 @@ else: def __init__(self, listener, **exception_kwargs): self.in_function = False - self.local_ident_stack = {} + self.local_ident_stack = set() self.listener = listener self.exception_kwargs = exception_kwargs def _add_declared(self, name): if not self.in_function: self.listener.declared_identifiers.add(name) + else: + self.local_ident_stack.add(name) def visitClass(self, node, *args): self._add_declared(node.name) @@ -247,7 +250,6 @@ else: # flip around the visiting of Assign so the expression gets # evaluated first, in the case of a clause like "x=x+5" (x # is undeclared) - self.visit(node.expr, *args) for n in node.nodes: self.visit(n, *args) @@ -267,20 +269,18 @@ else: # argument names in each function header so they arent # counted as "undeclared" - saved = {} inf = self.in_function self.in_function = True - for arg in node.argnames: - if arg in self.local_ident_stack: - saved[arg] = True - else: - self.local_ident_stack[arg] = True + + local_ident_stack = self.local_ident_stack + self.local_ident_stack = local_ident_stack.union([ + arg for arg in node.argnames + ]) + for n in node.getChildNodes(): self.visit(n, *args) self.in_function = inf - for arg in node.argnames: - if arg not in saved: - del self.local_ident_stack[arg] + self.local_ident_stack = local_ident_stack def visitFor(self, node, *args): @@ -333,9 +333,11 @@ else: self.listener.codeargs.append(p) self.listener.args.append(ExpressionGenerator(n).value()) self.listener.declared_identifiers = \ - self.listener.declared_identifiers.union(p.declared_identifiers) + self.listener.declared_identifiers.union( + p.declared_identifiers) self.listener.undeclared_identifiers = \ - self.listener.undeclared_identifiers.union(p.undeclared_identifiers) + self.listener.undeclared_identifiers.union( + p.undeclared_identifiers) def visit(self, expr): visitor.walk(expr, self) # , walker=walker()) diff --git a/mako/runtime.py b/mako/runtime.py index dfd701aa..f890c809 100644 --- a/mako/runtime.py +++ b/mako/runtime.py @@ -1,5 +1,5 @@ # mako/runtime.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -10,65 +10,75 @@ Namespace, and various helper functions.""" from mako import exceptions, util import __builtin__, inspect, sys + class Context(object): """Provides runtime namespace, output buffer, and various callstacks for templates. - - See :ref:`runtime_toplevel` for detail on the usage of + + See :ref:`runtime_toplevel` for detail on the usage of :class:`.Context`. - + """ - + def __init__(self, buffer, **data): self._buffer_stack = [buffer] - + self._data = data + self._kwargs = data.copy() self._with_template = None self._outputting_as_unicode = None self.namespaces = {} - - # "capture" function which proxies to the + + # "capture" function which proxies to the # generic "capture" function self._data['capture'] = util.partial(capture, self) - + # "caller" stack used by def calls with content self.caller_stack = self._data['caller'] = CallerStack() - + + def _set_with_template(self, t): + self._with_template = t + illegal_names = t.reserved_names.intersection(self._data) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words passed to render(): %s" % + ", ".join(illegal_names)) + @property def lookup(self): - """Return the :class:`.TemplateLookup` associated + """Return the :class:`.TemplateLookup` associated with this :class:`.Context`. - + """ return self._with_template.lookup - + @property def kwargs(self): - """Return the dictionary of keyword argments associated with this + """Return the dictionary of keyword arguments associated with this :class:`.Context`. - + """ return self._kwargs.copy() - + def push_caller(self, caller): - """Pushes a 'caller' callable onto the callstack for + """Push a ``caller`` callable onto the callstack for this :class:`.Context`.""" - - + + self.caller_stack.append(caller) - + def pop_caller(self): - """Pops a 'caller' callable onto the callstack for this + """Pop a ``caller`` callable onto the callstack for this :class:`.Context`.""" del self.caller_stack[-1] - + def keys(self): """Return a list of all names established in this :class:`.Context`.""" return self._data.keys() - + def __getitem__(self, key): if key in self._data: return self._data[key] @@ -78,45 +88,45 @@ class Context(object): def _push_writer(self): """push a capturing buffer onto this Context and return the new writer function.""" - + buf = util.FastEncodingBuffer() self._buffer_stack.append(buf) return buf.write def _pop_buffer_and_writer(self): - """pop the most recent capturing buffer from this Context + """pop the most recent capturing buffer from this Context and return the current writer after the pop. - + """ buf = self._buffer_stack.pop() return buf, self._buffer_stack[-1].write - + def _push_buffer(self): """push a capturing buffer onto this Context.""" - + self._push_writer() - + def _pop_buffer(self): """pop the most recent capturing buffer from this Context.""" - + return self._buffer_stack.pop() - + def get(self, key, default=None): """Return a value from this :class:`.Context`.""" - - return self._data.get(key, + + return self._data.get(key, __builtin__.__dict__.get(key, default) ) - + def write(self, string): """Write a string to this :class:`.Context` object's underlying output buffer.""" - + self._buffer_stack[-1].write(string) - + def writer(self): - """Return the current writer function""" + """Return the current writer function.""" return self._buffer_stack[-1].write @@ -130,17 +140,17 @@ class Context(object): c.namespaces = self.namespaces c.caller_stack = self.caller_stack return c - + def locals_(self, d): - """create a new :class:`.Context` with a copy of this - :class:`Context`'s current state, updated with the given dictionary.""" - + """Create a new :class:`.Context` with a copy of this + :class:`.Context`'s current state, updated with the given dictionary.""" + if len(d) == 0: return self c = self._copy() c._data.update(d) return c - + def _clean_inheritance_tokens(self): """create a new copy of this :class:`.Context`. with tokens related to inheritance state removed.""" @@ -158,23 +168,27 @@ class CallerStack(list): def __nonzero__(self): return self._get_caller() and True or False def _get_caller(self): + # this method can be removed once + # codegen MAGIC_NUMBER moves past 7 return self[-1] def __getattr__(self, key): return getattr(self._get_caller(), key) def _push_frame(self): - self.append(self.nextcaller or None) + frame = self.nextcaller or None + self.append(frame) self.nextcaller = None + return frame def _pop_frame(self): self.nextcaller = self.pop() - - + + class Undefined(object): """Represents an undefined value in a template. - - All template modules have a constant value + + All template modules have a constant value ``UNDEFINED`` present which is an instance of this object. - + """ def __str__(self): raise NameError("Undefined") @@ -183,6 +197,110 @@ class Undefined(object): UNDEFINED = Undefined() +class LoopStack(object): + """a stack for LoopContexts that implements the context manager protocol + to automatically pop off the top of the stack on context exit + """ + + def __init__(self): + self.stack = [] + + def _enter(self, iterable): + self._push(iterable) + return self._top + + def _exit(self): + self._pop() + return self._top + + @property + def _top(self): + if self.stack: + return self.stack[-1] + else: + return self + + def _pop(self): + return self.stack.pop() + + def _push(self, iterable): + new = LoopContext(iterable) + if self.stack: + new.parent = self.stack[-1] + return self.stack.append(new) + + def __getattr__(self, key): + raise exceptions.RuntimeException("No loop context is established") + + def __iter__(self): + return iter(self._top) + + +class LoopContext(object): + """A magic loop variable. + Automatically accessible in any ``% for`` block. + + See the section :ref:`loop_context` for usage + notes. + + :attr:`parent` -> :class:`.LoopContext` or ``None`` + The parent loop, if one exists. + :attr:`index` -> `int` + The 0-based iteration count. + :attr:`reverse_index` -> `int` + The number of iterations remaining. + :attr:`first` -> `bool` + ``True`` on the first iteration, ``False`` otherwise. + :attr:`last` -> `bool` + ``True`` on the last iteration, ``False`` otherwise. + :attr:`even` -> `bool` + ``True`` when ``index`` is even. + :attr:`odd` -> `bool` + ``True`` when ``index`` is odd. + """ + + def __init__(self, iterable): + self._iterable = iterable + self.index = 0 + self.parent = None + + def __iter__(self): + for i in self._iterable: + yield i + self.index += 1 + + @util.memoized_instancemethod + def __len__(self): + return len(self._iterable) + + @property + def reverse_index(self): + return len(self) - self.index - 1 + + @property + def first(self): + return self.index == 0 + + @property + def last(self): + return self.index == len(self) - 1 + + @property + def even(self): + return not self.odd + + @property + def odd(self): + return bool(self.index % 2) + + def cycle(self, *values): + """Cycle through values as the loop progresses. + """ + if not values: + raise ValueError("You must provide values to cycle through") + return values[self.index % len(values)] + + class _NSAttr(object): def __init__(self, parent): self.__parent = parent @@ -193,24 +311,26 @@ class _NSAttr(object): return getattr(ns.module, key) else: ns = ns.inherits - raise AttributeError(key) - + raise AttributeError(key) + class Namespace(object): - """Provides access to collections of rendering methods, which + """Provides access to collections of rendering methods, which can be local, from other templates, or from imported modules. - - To access a particular rendering method referenced by a - :class:`.Namespace`, use plain attribute access:: - + + To access a particular rendering method referenced by a + :class:`.Namespace`, use plain attribute access: + + .. sourcecode:: mako + ${some_namespace.foo(x, y, z)} - - :class:`.Namespace` also contains several built-in attributes + + :class:`.Namespace` also contains several built-in attributes described here. - + """ - - def __init__(self, name, context, - callables=None, inherits=None, + + def __init__(self, name, context, + callables=None, inherits=None, populate_self=True, calling_uri=None): self.name = name self.context = context @@ -221,7 +341,7 @@ class Namespace(object): callables = () module = None - """The Python module referenced by this Namespace. + """The Python module referenced by this :class:`.Namespace`. If the namespace references a :class:`.Template`, then this module is the equivalent of ``template.module``, @@ -236,8 +356,8 @@ class Namespace(object): """ context = None - """The :class:`.Context` object for this namespace. - + """The :class:`.Context` object for this :class:`.Namespace`. + Namespaces are often created with copies of contexts that contain slightly different data, particularly in inheritance scenarios. Using the :class:`.Context` off of a :class:`.Namespace` one @@ -245,24 +365,24 @@ class Namespace(object): one-another. """ - + filename = None """The path of the filesystem file used for this - Namespace's module or template. + :class:`.Namespace`'s module or template. If this is a pure module-based - Namespace, this evaluates to ``module.__file__``. If a + :class:`.Namespace`, this evaluates to ``module.__file__``. If a template-based namespace, it evaluates to the original template file location. """ - + uri = None - """The uri for this Namespace's template. + """The URI for this :class:`.Namespace`'s template. I.e. whatever was sent to :meth:`.TemplateLookup.get_template()`. - This is the equivalent of :attr:`Template.uri`. + This is the equivalent of :attr:`.Template.uri`. """ @@ -270,8 +390,8 @@ class Namespace(object): @util.memoized_property def attr(self): - """Access module level attributes by name. - + """Access module level attributes by name. + This accessor allows templates to supply "scalar" attributes which are particularly handy in inheritance relationships. See the example in @@ -281,86 +401,72 @@ class Namespace(object): return _NSAttr(self) def get_namespace(self, uri): - """Return a :class:`.Namespace` corresponding to the given uri. - - If the given uri is a relative uri (i.e. it does not - contain ia leading slash ``/``), the uri is adjusted to - be relative to the uri of the namespace itself. This + """Return a :class:`.Namespace` corresponding to the given ``uri``. + + If the given ``uri`` is a relative URI (i.e. it does not + contain a leading slash ``/``), the ``uri`` is adjusted to + be relative to the ``uri`` of the namespace itself. This method is therefore mostly useful off of the built-in - ``local`` namespace, described in :ref:`namespace_local` + ``local`` namespace, described in :ref:`namespace_local`. In most cases, a template wouldn't need this function, and should instead use the ``<%namespace>`` tag to load namespaces. However, since all ``<%namespace>`` tags are - evaulated before the body of a template ever runs, + evaluated before the body of a template ever runs, this method can be used to locate namespaces using expressions that were generated within the body code of the template, or to conditionally use a particular namespace. - + """ key = (self, uri) if key in self.context.namespaces: return self.context.namespaces[key] else: - ns = TemplateNamespace(uri, self.context._copy(), - templateuri=uri, - calling_uri=self._templateuri) + ns = TemplateNamespace(uri, self.context._copy(), + templateuri=uri, + calling_uri=self._templateuri) self.context.namespaces[key] = ns return ns - + def get_template(self, uri): - """Return a :class:`.Template` from the given uri. - - The uri resolution is relative to the uri of this :class:`.Namespace` + """Return a :class:`.Template` from the given ``uri``. + + The ``uri`` resolution is relative to the ``uri`` of this :class:`.Namespace` object's :class:`.Template`. - + """ return _lookup_template(self.context, uri, self._templateuri) - + def get_cached(self, key, **kwargs): - """Return a value from the :class:`.Cache` referenced by this + """Return a value from the :class:`.Cache` referenced by this :class:`.Namespace` object's :class:`.Template`. - - The advantage to this method versus direct access to the + + The advantage to this method versus direct access to the :class:`.Cache` is that the configuration parameters declared in ``<%page>`` take effect here, thereby calling up the same configured backend as that configured by ``<%page>``. - + """ - - if self.template: - if not self.template.cache_enabled: - createfunc = kwargs.get('createfunc', None) - if createfunc: - return createfunc() - else: - return None - - if self.template.cache_dir: - kwargs.setdefault('data_dir', self.template.cache_dir) - if self.template.cache_type: - kwargs.setdefault('type', self.template.cache_type) - if self.template.cache_url: - kwargs.setdefault('url', self.template.cache_url) + return self.cache.get(key, **kwargs) - + @property def cache(self): - """Return the :class:`.Cache` object referenced - by this :class:`.Namespace` object's + """Return the :class:`.Cache` object referenced + by this :class:`.Namespace` object's :class:`.Template`. - + """ return self.template.cache - + def include_file(self, uri, **kwargs): - """Include a file at the given uri""" - + """Include a file at the given ``uri``.""" + _include_file(self.context, uri, self._templateuri, **kwargs) - + def _populate(self, d, l): for ident in l: if ident == '*': @@ -368,7 +474,7 @@ class Namespace(object): d[k] = v else: d[ident] = getattr(self, ident) - + def _get_star(self): if self.callables: for key in self.callables: @@ -381,7 +487,7 @@ class Namespace(object): val = getattr(self.inherits, key) else: raise AttributeError( - "Namespace '%s' has no member '%s'" % + "Namespace '%s' has no member '%s'" % (self.name, key)) setattr(self, key, val) return val @@ -389,8 +495,8 @@ class Namespace(object): class TemplateNamespace(Namespace): """A :class:`.Namespace` specific to a :class:`.Template` instance.""" - def __init__(self, name, context, template=None, templateuri=None, - callables=None, inherits=None, + def __init__(self, name, context, template=None, templateuri=None, + callables=None, inherits=None, populate_self=True, calling_uri=None): self.name = name self.context = context @@ -399,7 +505,7 @@ class TemplateNamespace(Namespace): self.callables = dict([(c.func_name, c) for c in callables]) if templateuri is not None: - self.template = _lookup_template(context, templateuri, + self.template = _lookup_template(context, templateuri, calling_uri) self._templateuri = self.template.module._template_uri elif template is not None: @@ -410,13 +516,13 @@ class TemplateNamespace(Namespace): if populate_self: lclcallable, lclcontext = \ - _populate_self_namespace(context, self.template, + _populate_self_namespace(context, self.template, self_ns=self) @property def module(self): - """The Python module referenced by this Namespace. - + """The Python module referenced by this :class:`.Namespace`. + If the namespace references a :class:`.Template`, then this module is the equivalent of ``template.module``, i.e. the generated module for the template. @@ -427,17 +533,17 @@ class TemplateNamespace(Namespace): @property def filename(self): """The path of the filesystem file used for this - Namespace's module or template. + :class:`.Namespace`'s module or template. """ return self.template.filename @property def uri(self): - """The uri for this Namespace's template. - + """The URI for this :class:`.Namespace`'s template. + I.e. whatever was sent to :meth:`.TemplateLookup.get_template()`. - - This is the equivalent of :attr:`Template.uri`. + + This is the equivalent of :attr:`.Template.uri`. """ return self.template.uri @@ -463,7 +569,7 @@ class TemplateNamespace(Namespace): else: raise AttributeError( - "Namespace '%s' has no member '%s'" % + "Namespace '%s' has no member '%s'" % (self.name, key)) setattr(self, key, val) return val @@ -471,8 +577,8 @@ class TemplateNamespace(Namespace): class ModuleNamespace(Namespace): """A :class:`.Namespace` specific to a Python module instance.""" - def __init__(self, name, context, module, - callables=None, inherits=None, + def __init__(self, name, context, module, + callables=None, inherits=None, populate_self=True, calling_uri=None): self.name = name self.context = context @@ -488,7 +594,7 @@ class ModuleNamespace(Namespace): @property def filename(self): """The path of the filesystem file used for this - Namespace's module or template. + :class:`.Namespace`'s module or template. """ return self.module.__file__ @@ -513,7 +619,7 @@ class ModuleNamespace(Namespace): val = getattr(self.inherits, key) else: raise AttributeError( - "Namespace '%s' has no member '%s'" % + "Namespace '%s' has no member '%s'" % (self.name, key)) setattr(self, key, val) return val @@ -521,11 +627,11 @@ class ModuleNamespace(Namespace): def supports_caller(func): """Apply a caller_stack compatibility decorator to a plain Python function. - + See the example in :ref:`namespaces_python_modules`. - + """ - + def wrap_stackframe(context, *args, **kwargs): context.caller_stack._push_frame() try: @@ -533,19 +639,19 @@ def supports_caller(func): finally: context.caller_stack._pop_frame() return wrap_stackframe - + def capture(context, callable_, *args, **kwargs): """Execute the given template def, capturing the output into a buffer. - + See the example in :ref:`namespaces_python_modules`. - + """ - + if not callable(callable_): raise exceptions.RuntimeException( - "capture() function expects a callable as " - "its argument (i.e. capture(func, *args, **kwargs))" + "capture() function expects a callable as " + "its argument (i.e. capture(func, *args, **kwargs))" ) context._push_buffer() try: @@ -567,7 +673,7 @@ def _decorate_toplevel(fn): return fn(y)(context, *args, **kw) return go return decorate_render - + def _decorate_inline(context, fn): def decorate_render(render_fn): dec = fn(render_fn) @@ -575,17 +681,17 @@ def _decorate_inline(context, fn): return dec(context, *args, **kw) return go return decorate_render - + def _include_file(context, uri, calling_uri, **kwargs): """locate the template from the given uri and include it in the current output.""" - + template = _lookup_template(context, uri, calling_uri) (callable_, ctx) = _populate_self_namespace( - context._clean_inheritance_tokens(), + context._clean_inheritance_tokens(), template) callable_(ctx, **_kwargs_for_include(callable_, context._data, **kwargs)) - + def _inherit_from(context, uri, calling_uri): """called by the _inherit method in template modules to set up the inheritance chain at the start of a template's @@ -599,9 +705,9 @@ def _inherit_from(context, uri, calling_uri): while ih.inherits is not None: ih = ih.inherits lclcontext = context.locals_({'next':ih}) - ih.inherits = TemplateNamespace("self:%s" % template.uri, - lclcontext, - template = template, + ih.inherits = TemplateNamespace("self:%s" % template.uri, + lclcontext, + template = template, populate_self=False) context._data['parent'] = lclcontext._data['local'] = ih.inherits callable_ = getattr(template.module, '_mako_inherit', None) @@ -619,7 +725,7 @@ def _lookup_template(context, uri, relativeto): lookup = context._with_template.lookup if lookup is None: raise exceptions.TemplateLookupException( - "Template '%s' has no TemplateLookup associated" % + "Template '%s' has no TemplateLookup associated" % context._with_template.uri) uri = lookup.adjust_uri(uri, relativeto) try: @@ -629,8 +735,8 @@ def _lookup_template(context, uri, relativeto): def _populate_self_namespace(context, template, self_ns=None): if self_ns is None: - self_ns = TemplateNamespace('self:%s' % template.uri, - context, template=template, + self_ns = TemplateNamespace('self:%s' % template.uri, + context, template=template, populate_self=False) context._data['self'] = context._data['local'] = self_ns if hasattr(template.module, '_mako_inherit'): @@ -640,7 +746,7 @@ def _populate_self_namespace(context, template, self_ns=None): return (template.callable_, context) def _render(template, callable_, args, data, as_unicode=False): - """create a Context and return the string + """create a Context and return the string output of the given template and template callable.""" if as_unicode: @@ -649,14 +755,14 @@ def _render(template, callable_, args, data, as_unicode=False): buf = util.StringIO() else: buf = util.FastEncodingBuffer( - unicode=as_unicode, - encoding=template.output_encoding, + unicode=as_unicode, + encoding=template.output_encoding, errors=template.encoding_errors) context = Context(buf, **data) context._outputting_as_unicode = as_unicode - context._with_template = template - - _render_context(template, callable_, context, *args, + context._set_with_template(template) + + _render_context(template, callable_, context, *args, **_kwargs_for_callable(callable_, data)) return context._pop_buffer().getvalue() @@ -665,7 +771,7 @@ def _kwargs_for_callable(callable_, data): # for normal pages, **pageargs is usually present if argspec[2]: return data - + # for rendering defs from the top level, figure out the args namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None] kwargs = {} @@ -681,10 +787,10 @@ def _kwargs_for_include(callable_, data, **kwargs): if arg != 'context' and arg in data and arg not in kwargs: kwargs[arg] = data[arg] return kwargs - + def _render_context(tmpl, callable_, context, *args, **kwargs): import mako.template as template - # create polymorphic 'self' namespace for this + # create polymorphic 'self' namespace for this # template with possibly updated context if not isinstance(tmpl, template.DefTemplate): # if main render method, call from the base of the inheritance stack @@ -694,7 +800,7 @@ def _render_context(tmpl, callable_, context, *args, **kwargs): # otherwise, call the actual rendering method specified (inherit, lclcontext) = _populate_self_namespace(context, tmpl.parent) _exec_template(callable_, context, args=args, kwargs=kwargs) - + def _exec_template(callable_, context, args=None, kwargs=None): """execute a rendering callable given the callable, a Context, and optional explicit arguments @@ -711,7 +817,7 @@ def _exec_template(callable_, context, args=None, kwargs=None): callable_(context, *args, **kwargs) except Exception, e: _render_error(template, context, e) - except: + except: e = sys.exc_info()[0] _render_error(template, context, e) else: @@ -730,6 +836,6 @@ def _render_error(template, context, error): context._buffer_stack[:] = [util.FastEncodingBuffer( error_template.output_encoding, error_template.encoding_errors)] - - context._with_template = error_template + + context._set_with_template(error_template) error_template.render_context(context, error=error) diff --git a/mako/template.py b/mako/template.py index 903dc425..b0691391 100644 --- a/mako/template.py +++ b/mako/template.py @@ -1,5 +1,5 @@ # mako/template.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,57 +8,74 @@ template strings, as well as template runtime operations.""" from mako.lexer import Lexer -from mako import runtime, util, exceptions, codegen -import imp, os, re, shutil, stat, sys, tempfile, time, types, weakref +from mako import runtime, util, exceptions, codegen, cache +import os, re, shutil, stat, sys, tempfile, types, weakref + - class Template(object): """Represents a compiled template. - + :class:`.Template` includes a reference to the original - template source (via the ``.source`` attribute) + template source (via the :attr:`.source` attribute) as well as the source code of the - generated Python module (i.e. the ``.code`` attribute), + generated Python module (i.e. the :attr:`.code` attribute), as well as a reference to an actual Python module. :class:`.Template` is constructed using either a literal string representing the template text, or a filename representing a filesystem path to a source file. - - :param text: textual template source. This argument is mutually - exclusive versus the "filename" parameter. - :param filename: filename of the source template. This argument is - mutually exclusive versus the "text" parameter. + :param text: textual template source. This argument is mutually + exclusive versus the ``filename`` parameter. + + :param filename: filename of the source template. This argument is + mutually exclusive versus the ``text`` parameter. :param buffer_filters: string list of filters to be applied - to the output of %defs which are buffered, cached, or otherwise + to the output of ``%def``\ s which are buffered, cached, or otherwise filtered, after all filters - defined with the %def itself have been applied. Allows the + defined with the ``%def`` itself have been applied. Allows the creation of default expression filters that let the output - of return-valued %defs "opt out" of that filtering via + of return-valued ``%def``\ s "opt out" of that filtering via passing special attributes or objects. - :param bytestring_passthrough: When True, and output_encoding is - set to None, and :meth:`.Template.render` is used to render, - the StringIO or cStringIO buffer will be used instead of the + :param bytestring_passthrough: When ``True``, and ``output_encoding`` is + set to ``None``, and :meth:`.Template.render` is used to render, + the `StringIO` or `cStringIO` buffer will be used instead of the default "fast" buffer. This allows raw bytestrings in the output stream, such as in expressions, to pass straight - through to the buffer. New in 0.4 to provide the same - behavior as that of the previous series. This flag is forced - to True if disable_unicode is also configured. + through to the buffer. This flag is forced + to ``True`` if ``disable_unicode`` is also configured. - :param cache_dir: Filesystem directory where cache files will be - placed. See :ref:`caching_toplevel`. + .. versionadded:: 0.4 + Added to provide the same behavior as that of the previous series. + + :param cache_args: Dictionary of cache configuration arguments that + will be passed to the :class:`.CacheImpl`. See :ref:`caching_toplevel`. + + :param cache_dir: + + .. deprecated:: 0.6 + Use the ``'dir'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. :param cache_enabled: Boolean flag which enables caching of this template. See :ref:`caching_toplevel`. - :param cache_type: Type of Beaker caching to be applied to the - template. See :ref:`caching_toplevel`. - - :param cache_url: URL of a memcached server with which to use - for caching. See :ref:`caching_toplevel`. + :param cache_impl: String name of a :class:`.CacheImpl` caching + implementation to use. Defaults to ``'beaker'``. + + :param cache_type: + + .. deprecated:: 0.6 + Use the ``'type'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. + + :param cache_url: + + .. deprecated:: 0.6 + Use the ``'url'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. :param default_filters: List of string filter names that will be applied to all expressions. See :ref:`filtering_default_filters`. @@ -66,9 +83,16 @@ class Template(object): :param disable_unicode: Disables all awareness of Python Unicode objects. See :ref:`unicode_disabled`. + :param enable_loop: When ``True``, enable the ``loop`` context variable. + This can be set to ``False`` to support templates that may + be making usage of the name "``loop``". Individual templates can + re-enable the "loop" context by placing the directive + ``enable_loop="True"`` inside the ``<%page>`` tag -- see + :ref:`migrating_loop`. + :param encoding_errors: Error parameter passed to ``encode()`` when string encoding is performed. See :ref:`usage_unicode`. - + :param error_handler: Python callable which is called whenever compile or runtime exceptions occur. The callable is passed the current context as well as the exception. If the @@ -76,13 +100,13 @@ class Template(object): be handled, else it is re-raised after the function completes. Is used to provide custom error-rendering functions. - + :param format_exceptions: if ``True``, exceptions which occur during the render phase of this template will be caught and formatted into an HTML error page, which then becomes the - rendered result of the :meth:`render` call. Otherwise, + rendered result of the :meth:`.render` call. Otherwise, runtime exceptions are propagated outwards. - + :param imports: String list of Python statements, typically individual "import" lines, which will be placed into the module level preamble of all generated Python modules. See the example @@ -92,66 +116,107 @@ class Template(object): be used in lieu of the coding comment. See :ref:`usage_unicode` as well as :ref:`unicode_toplevel` for details on source encoding. - + :param lookup: a :class:`.TemplateLookup` instance that will be used for all file lookups via the ``<%namespace>``, ``<%include>``, and ``<%inherit>`` tags. See :ref:`usage_templatelookup`. - - :param module_directory: Filesystem location where generated + + :param module_directory: Filesystem location where generated Python module files will be placed. - :param module_filename: Overrides the filename of the generated + :param module_filename: Overrides the filename of the generated Python module file. For advanced usage only. - - :param output_encoding: The encoding to use when :meth:`.render` - is called. + + :param module_writer: A callable which overrides how the Python + module is written entirely. The callable is passed the + encoded source content of the module and the destination + path to be written to. The default behavior of module writing + uses a tempfile in conjunction with a file move in order + to make the operation atomic. So a user-defined module + writing function that mimics the default behavior would be: + + .. sourcecode:: python + + import tempfile + import os + import shutil + + def module_writer(source, outputpath): + (dest, name) = \\ + tempfile.mkstemp( + dir=os.path.dirname(outputpath) + ) + + os.write(dest, source) + os.close(dest) + shutil.move(name, outputpath) + + from mako.template import Template + mytemplate = Template( + file="index.html", + module_directory="/path/to/modules", + module_writer=module_writer + ) + + The function is provided for unusual configurations where + certain platform-specific permissions or other special + steps are needed. + + :param output_encoding: The encoding to use when :meth:`.render` + is called. See :ref:`usage_unicode` as well as :ref:`unicode_toplevel`. - - :param preprocessor: Python callable which will be passed + + :param preprocessor: Python callable which will be passed the full template source before it is parsed. The return result of the callable will be used as the template source code. - - :param strict_undefined: Replaces the automatic usage of + + :param strict_undefined: Replaces the automatic usage of ``UNDEFINED`` for any undeclared variables not located in the :class:`.Context` with an immediate raise of ``NameError``. The advantage is immediate reporting of - missing variables which include the name. New in 0.3.6. - - :param uri: string uri or other identifier for this template. - If not provided, the uri is generated from the filesystem + missing variables which include the name. + + .. versionadded:: 0.3.6 + + :param uri: string URI or other identifier for this template. + If not provided, the ``uri`` is generated from the filesystem path, or from the in-memory identity of a non-file-based - template. The primary usage of the uri is to provide a key + template. The primary usage of the ``uri`` is to provide a key within :class:`.TemplateLookup`, as well as to generate the file path of the generated Python module file, if ``module_directory`` is specified. - + """ - - def __init__(self, - text=None, - filename=None, - uri=None, - format_exceptions=False, - error_handler=None, - lookup=None, - output_encoding=None, - encoding_errors='strict', - module_directory=None, - cache_type=None, - cache_dir=None, - cache_url=None, - module_filename=None, - input_encoding=None, + + def __init__(self, + text=None, + filename=None, + uri=None, + format_exceptions=False, + error_handler=None, + lookup=None, + output_encoding=None, + encoding_errors='strict', + module_directory=None, + cache_args=None, + cache_impl='beaker', + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + module_filename=None, + input_encoding=None, disable_unicode=False, - bytestring_passthrough=False, - default_filters=None, - buffer_filters=(), + module_writer=None, + bytestring_passthrough=False, + default_filters=None, + buffer_filters=(), strict_undefined=False, - imports=None, - preprocessor=None, - cache_enabled=True): + imports=None, + enable_loop=True, + preprocessor=None): if uri: self.module_id = re.sub(r'\W', "_", uri) self.uri = uri @@ -163,13 +228,25 @@ class Template(object): else: self.module_id = "memory:" + hex(id(self)) self.uri = self.module_id - + + u_norm = self.uri + if u_norm.startswith("/"): + u_norm = u_norm[1:] + u_norm = os.path.normpath(u_norm) + if u_norm.startswith(".."): + raise exceptions.TemplateLookupException( + "Template uri \"%s\" is invalid - " + "it cannot be relative outside " + "of the root path." % self.uri) + self.input_encoding = input_encoding self.output_encoding = output_encoding self.encoding_errors = encoding_errors self.disable_unicode = disable_unicode self.bytestring_passthrough = bytestring_passthrough or disable_unicode + self.enable_loop = enable_loop self.strict_undefined = strict_undefined + self.module_writer = module_writer if util.py3k and disable_unicode: raise exceptions.UnsupportedError( @@ -187,10 +264,10 @@ class Template(object): else: self.default_filters = default_filters self.buffer_filters = buffer_filters - + self.imports = imports self.preprocessor = preprocessor - + # if plain text, compile code in memory only if text is not None: (code, module) = _compile_text(self, text, filename) @@ -203,18 +280,14 @@ class Template(object): if module_filename is not None: path = module_filename elif module_directory is not None: - u = self.uri - if u[0] == '/': - u = u[1:] path = os.path.abspath( os.path.join( - os.path.normpath(module_directory), - os.path.normpath(u) + ".py" + os.path.normpath(module_directory), + u_norm + ".py" ) ) else: path = None - module = self._compile_from_file(path, filename) else: raise exceptions.RuntimeException( @@ -226,147 +299,192 @@ class Template(object): self.format_exceptions = format_exceptions self.error_handler = error_handler self.lookup = lookup - self.cache_type = cache_type - self.cache_dir = cache_dir - self.cache_url = cache_url + + self.module_directory = module_directory + + self._setup_cache_args( + cache_impl, cache_enabled, cache_args, + cache_type, cache_dir, cache_url + ) + + @util.memoized_property + def reserved_names(self): + if self.enable_loop: + return codegen.RESERVED_NAMES + else: + return codegen.RESERVED_NAMES.difference(['loop']) + + def _setup_cache_args(self, + cache_impl, cache_enabled, cache_args, + cache_type, cache_dir, cache_url): + self.cache_impl = cache_impl self.cache_enabled = cache_enabled - + if cache_args: + self.cache_args = cache_args + else: + self.cache_args = {} + + # transfer deprecated cache_* args + if cache_type: + self.cache_args['type'] = cache_type + if cache_dir: + self.cache_args['dir'] = cache_dir + if cache_url: + self.cache_args['url'] = cache_url + def _compile_from_file(self, path, filename): if path is not None: util.verify_directory(os.path.dirname(path)) filemtime = os.stat(filename)[stat.ST_MTIME] if not os.path.exists(path) or \ os.stat(path)[stat.ST_MTIME] < filemtime: + data = util.read_file(filename) _compile_module_file( - self, - open(filename, 'rb').read(), - filename, - path) - module = imp.load_source(self.module_id, path, open(path, 'rb')) + self, + data, + filename, + path, + self.module_writer) + module = util.load_module(self.module_id, path) del sys.modules[self.module_id] if module._magic_number != codegen.MAGIC_NUMBER: + data = util.read_file(filename) _compile_module_file( - self, - open(filename, 'rb').read(), - filename, - path) - module = imp.load_source(self.module_id, path, open(path, 'rb')) + self, + data, + filename, + path, + self.module_writer) + module = util.load_module(self.module_id, path) del sys.modules[self.module_id] ModuleInfo(module, path, self, filename, None, None) else: # template filename and no module directory, compile code # in memory + data = util.read_file(filename) code, module = _compile_text( - self, - open(filename, 'rb').read(), + self, + data, filename) self._source = None self._code = code ModuleInfo(module, None, self, filename, code, None) return module - + @property def source(self): - """return the template source code for this Template.""" - + """Return the template source code for this :class:`.Template`.""" + return _get_module_info_from_callable(self.callable_).source @property def code(self): - """return the module source code for this Template""" - + """Return the module source code for this :class:`.Template`.""" + return _get_module_info_from_callable(self.callable_).code - - @property + + @util.memoized_property def cache(self): - return self.module._template_cache - + return cache.Cache(self) + + @property + def cache_dir(self): + return self.cache_args['dir'] + @property + def cache_url(self): + return self.cache_args['url'] + @property + def cache_type(self): + return self.cache_args['type'] + def render(self, *args, **data): """Render the output of this template as a string. - - if the template specifies an output encoding, the string + + If the template specifies an output encoding, the string will be encoded accordingly, else the output is raw (raw - output uses cStringIO and can't handle multibyte - characters). a Context object is created corresponding - to the given data. Arguments that are explictly declared + output uses `cStringIO` and can't handle multibyte + characters). A :class:`.Context` object is created corresponding + to the given data. Arguments that are explicitly declared by this template's internal rendering method are also - pulled from the given \*args, \**data members. - + pulled from the given ``*args``, ``**data`` members. + """ return runtime._render(self, self.callable_, args, data) - + def render_unicode(self, *args, **data): - """render the output of this template as a unicode object.""" - - return runtime._render(self, - self.callable_, - args, - data, + """Render the output of this template as a unicode object.""" + + return runtime._render(self, + self.callable_, + args, + data, as_unicode=True) - + def render_context(self, context, *args, **kwargs): - """Render this Template with the given context. - - the data is written to the context's buffer. - + """Render this :class:`.Template` with the given context. + + The data is written to the context's buffer. + """ if getattr(context, '_with_template', None) is None: - context._with_template = self - runtime._render_context(self, - self.callable_, - context, - *args, + context._set_with_template(self) + runtime._render_context(self, + self.callable_, + context, + *args, **kwargs) - + def has_def(self, name): return hasattr(self.module, "render_%s" % name) - + def get_def(self, name): """Return a def of this template as a :class:`.DefTemplate`.""" - + return DefTemplate(self, getattr(self.module, "render_%s" % name)) def _get_def_callable(self, name): return getattr(self.module, "render_%s" % name) - + @property - def last_modified(self): - return self.module._modified_time - + def last_modified(self): + return self.module._modified_time + class ModuleTemplate(Template): """A Template which is constructed given an existing Python module. - + e.g.:: - + t = Template("this is a template") f = file("mymodule.py", "w") f.write(t.code) f.close() - + import mymodule - + t = ModuleTemplate(mymodule) print t.render() - + """ - - def __init__(self, module, - module_filename=None, - template=None, - template_filename=None, - module_source=None, + + def __init__(self, module, + module_filename=None, + template=None, + template_filename=None, + module_source=None, template_source=None, - output_encoding=None, + output_encoding=None, encoding_errors='strict', - disable_unicode=False, + disable_unicode=False, bytestring_passthrough=False, format_exceptions=False, - error_handler=None, - lookup=None, + error_handler=None, + lookup=None, + cache_args=None, + cache_impl='beaker', + cache_enabled=True, cache_type=None, - cache_dir=None, - cache_url=None, - cache_enabled=True + cache_dir=None, + cache_url=None, ): self.module_id = re.sub(r'\W', "_", module._template_uri) self.uri = module._template_uri @@ -375,6 +493,7 @@ class ModuleTemplate(Template): self.encoding_errors = encoding_errors self.disable_unicode = disable_unicode self.bytestring_passthrough = bytestring_passthrough or disable_unicode + self.enable_loop = module._enable_loop if util.py3k and disable_unicode: raise exceptions.UnsupportedError( @@ -387,26 +506,26 @@ class ModuleTemplate(Template): self.module = module self.filename = template_filename - ModuleInfo(module, - module_filename, - self, - template_filename, - module_source, + ModuleInfo(module, + module_filename, + self, + template_filename, + module_source, template_source) - + self.callable_ = self.module.render_body self.format_exceptions = format_exceptions self.error_handler = error_handler self.lookup = lookup - self.cache_type = cache_type - self.cache_dir = cache_dir - self.cache_url = cache_url - self.cache_enabled = cache_enabled - + self._setup_cache_args( + cache_impl, cache_enabled, cache_args, + cache_type, cache_dir, cache_url + ) + class DefTemplate(Template): - """a Template which represents a callable def in a parent + """A :class:`.Template` which represents a callable def in a parent template.""" - + def __init__(self, parent, callable_): self.parent = parent self.callable_ = callable_ @@ -415,6 +534,7 @@ class DefTemplate(Template): self.encoding_errors = parent.encoding_errors self.format_exceptions = parent.format_exceptions self.error_handler = parent.error_handler + self.enable_loop = parent.enable_loop self.lookup = parent.lookup self.bytestring_passthrough = parent.bytestring_passthrough @@ -425,16 +545,16 @@ class ModuleInfo(object): """Stores information about a module currently loaded into memory, provides reverse lookups of template source, module source code based on a module's identifier. - + """ _modules = weakref.WeakValueDictionary() - def __init__(self, - module, - module_filename, - template, - template_filename, - module_source, + def __init__(self, + module, + module_filename, + template, + template_filename, + module_source, template_source): self.module = module self.module_filename = module_filename @@ -444,14 +564,14 @@ class ModuleInfo(object): self._modules[module.__name__] = template._mmarker = self if module_filename: self._modules[module_filename] = self - + @property def code(self): if self.module_source is not None: return self.module_source else: - return open(self.module_filename).read() - + return util.read_file(self.module_filename) + @property def source(self): if self.template_source is not None: @@ -462,31 +582,37 @@ class ModuleInfo(object): else: return self.template_source else: + data = util.read_file(self.template_filename) if self.module._source_encoding: - return open(self.template_filename, 'rb').read().\ - decode(self.module._source_encoding) + return data.decode(self.module._source_encoding) else: - return open(self.template_filename).read() - -def _compile_text(template, text, filename): - identifier = template.module_id - lexer = Lexer(text, - filename, + return data + +def _compile(template, text, filename, generate_magic_comment): + lexer = Lexer(text, + filename, disable_unicode=template.disable_unicode, input_encoding=template.input_encoding, preprocessor=template.preprocessor) node = lexer.parse() - - source = codegen.compile(node, - template.uri, + source = codegen.compile(node, + template.uri, filename, default_filters=template.default_filters, - buffer_filters=template.buffer_filters, - imports=template.imports, + buffer_filters=template.buffer_filters, + imports=template.imports, source_encoding=lexer.encoding, - generate_magic_comment=template.disable_unicode, + generate_magic_comment=generate_magic_comment, disable_unicode=template.disable_unicode, - strict_undefined=template.strict_undefined) + strict_undefined=template.strict_undefined, + enable_loop=template.enable_loop, + reserved_names=template.reserved_names) + return source, lexer + +def _compile_text(template, text, filename): + identifier = template.module_id + source, lexer = _compile(template, text, filename, + generate_magic_comment=template.disable_unicode) cid = identifier if not util.py3k and isinstance(cid, unicode): @@ -496,41 +622,29 @@ def _compile_text(template, text, filename): exec code in module.__dict__, module.__dict__ return (source, module) -def _compile_module_file(template, text, filename, outputpath): +def _compile_module_file(template, text, filename, outputpath, module_writer): identifier = template.module_id - lexer = Lexer(text, - filename, - disable_unicode=template.disable_unicode, - input_encoding=template.input_encoding, - preprocessor=template.preprocessor) - - node = lexer.parse() - source = codegen.compile(node, - template.uri, - filename, - default_filters=template.default_filters, - buffer_filters=template.buffer_filters, - imports=template.imports, - source_encoding=lexer.encoding, - generate_magic_comment=True, - disable_unicode=template.disable_unicode, - strict_undefined=template.strict_undefined) - - # make tempfiles in the same location as the ultimate - # location. this ensures they're on the same filesystem, - # avoiding synchronization issues. - (dest, name) = tempfile.mkstemp(dir=os.path.dirname(outputpath)) - + source, lexer = _compile(template, text, filename, + generate_magic_comment=True) + if isinstance(source, unicode): source = source.encode(lexer.encoding or 'ascii') - - os.write(dest, source) - os.close(dest) - shutil.move(name, outputpath) + + if module_writer: + module_writer(source, outputpath) + else: + # make tempfiles in the same location as the ultimate + # location. this ensures they're on the same filesystem, + # avoiding synchronization issues. + (dest, name) = tempfile.mkstemp(dir=os.path.dirname(outputpath)) + + os.write(dest, source) + os.close(dest) + shutil.move(name, outputpath) def _get_module_info_from_callable(callable_): return _get_module_info(callable_.func_globals['__name__']) - + def _get_module_info(filename): return ModuleInfo._modules[filename] - + diff --git a/mako/util.py b/mako/util.py index 5518b4dd..df4bf4b7 100644 --- a/mako/util.py +++ b/mako/util.py @@ -1,13 +1,15 @@ # mako/util.py -# Copyright (C) 2006-2011 the Mako authors and contributors +# Copyright (C) 2006-2012 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php +import imp import sys py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) +py26 = sys.version_info >= (2, 6) py24 = sys.version_info >= (2, 4) and sys.version_info < (2, 5) jython = sys.platform.startswith('java') win32 = sys.platform.startswith('win') @@ -33,8 +35,8 @@ except ImportError: if win32 or jython: time_func = time.clock else: - time_func = time.time - + time_func = time.time + def function_named(fn, name): """Return a function with a given __name__. @@ -56,20 +58,57 @@ except: return newfunc if py24: + def all(iterable): + for i in iterable: + if not i: + return False + return True + def exception_name(exc): try: return exc.__class__.__name__ except AttributeError: return exc.__name__ else: + all = all + def exception_name(exc): return exc.__class__.__name__ - + + +class PluginLoader(object): + def __init__(self, group): + self.group = group + self.impls = {} + + def load(self, name): + if name in self.impls: + return self.impls[name]() + else: + import pkg_resources + for impl in pkg_resources.iter_entry_points( + self.group, + name): + self.impls[name] = impl.load + return impl.load() + else: + raise exceptions.RuntimeException( + "Can't load plugin %s %s" % + (self.group, name)) + + def register(self, name, modulepath, objname): + def load(): + mod = __import__(modulepath) + for token in modulepath.split(".")[1:]: + mod = getattr(mod, token) + return getattr(mod, objname) + self.impls[name] = load + def verify_directory(dir): """create and/or verify a filesystem directory.""" - + tries = 0 - + while not os.path.exists(dir): try: tries += 1 @@ -100,20 +139,47 @@ class memoized_property(object): obj.__dict__[self.__name__] = result = self.fget(obj) return result +class memoized_instancemethod(object): + """Decorate a method memoize its return value. + + Best applied to no-arg methods: memoization is not sensitive to + argument values, and will always return the same value even when + called with different arguments. + + """ + def __init__(self, fget, doc=None): + self.fget = fget + self.__doc__ = doc or fget.__doc__ + self.__name__ = fget.__name__ + + def __get__(self, obj, cls): + if obj is None: + return self + def oneshot(*args, **kw): + result = self.fget(obj, *args, **kw) + memo = lambda *a, **kw: result + memo.__name__ = self.__name__ + memo.__doc__ = self.__doc__ + obj.__dict__[self.__name__] = memo + return result + oneshot.__name__ = self.__name__ + oneshot.__doc__ = self.__doc__ + return oneshot + class SetLikeDict(dict): """a dictionary that has some setlike methods on it""" def union(self, other): """produce a 'union' of this dict and another (at the key level). - + values in the second dict take precedence over that of the first""" x = SetLikeDict(**self) x.update(other) return x class FastEncodingBuffer(object): - """a very rudimentary buffer that is faster than StringIO, + """a very rudimentary buffer that is faster than StringIO, but doesn't crash on unicode data like cStringIO.""" - + def __init__(self, encoding=None, errors='strict', unicode=False): self.data = collections.deque() self.encoding = encoding @@ -124,26 +190,27 @@ class FastEncodingBuffer(object): self.unicode = unicode self.errors = errors self.write = self.data.append - + def truncate(self): self.data = collections.deque() self.write = self.data.append - + def getvalue(self): if self.encoding: - return self.delim.join(self.data).encode(self.encoding, self.errors) + return self.delim.join(self.data).encode(self.encoding, + self.errors) else: return self.delim.join(self.data) class LRUCache(dict): - """A dictionary-like object that stores a limited number of items, discarding - lesser used items periodically. - + """A dictionary-like object that stores a limited number of items, + discarding lesser used items periodically. + this is a rewrite of LRUCache from Myghty to use a periodic timestamp-based - paradigm so that synchronization is not really needed. the size management + paradigm so that synchronization is not really needed. the size management is inexact. """ - + class _Item(object): def __init__(self, key, value): self.key = key @@ -151,26 +218,26 @@ class LRUCache(dict): self.timestamp = time_func() def __repr__(self): return repr(self.value) - + def __init__(self, capacity, threshold=.5): self.capacity = capacity self.threshold = threshold - + def __getitem__(self, key): item = dict.__getitem__(self, key) item.timestamp = time_func() return item.value - + def values(self): return [i.value for i in dict.values(self)] - + def setdefault(self, key, value): if key in self: return self[key] else: self[key] = value return value - + def __setitem__(self, key, value): item = dict.get(self, key) if item is None: @@ -179,17 +246,17 @@ class LRUCache(dict): else: item.value = value self._manage_size() - + def _manage_size(self): while len(self) > self.capacity + self.capacity * self.threshold: - bytime = sorted(dict.values(self), + bytime = sorted(dict.values(self), key=operator.attrgetter('timestamp'), reverse=True) for item in bytime[self.capacity:]: try: del self[item.key] except KeyError: - # if we couldnt find a key, most likely some other thread broke in - # on us. loop around and try again + # if we couldn't find a key, most likely some other thread + # broke in on us. loop around and try again break # Regexp to match python magic encoding line @@ -198,7 +265,8 @@ _PYTHON_MAGIC_COMMENT_re = re.compile( re.VERBOSE) def parse_encoding(fp): - """Deduce the encoding of a Python source file (binary mode) from magic comment. + """Deduce the encoding of a Python source file (binary mode) from magic + comment. It does this in the same way as the `Python interpreter`__ @@ -227,7 +295,8 @@ def parse_encoding(fp): pass else: line2 = fp.readline() - m = _PYTHON_MAGIC_COMMENT_re.match(line2.decode('ascii', 'ignore')) + m = _PYTHON_MAGIC_COMMENT_re.match( + line2.decode('ascii', 'ignore')) if has_bom: if m: @@ -244,14 +313,14 @@ def parse_encoding(fp): def sorted_dict_repr(d): """repr() a dictionary with the keys in order. - + Used by the lexer unit test to compare parse trees based on strings. - + """ keys = d.keys() keys.sort() return "{" + ", ".join(["%r: %r" % (k, d[k]) for k in keys]) + "}" - + def restore__ast(_ast): """Attempt to restore the required classes to the _ast module if it appears to be missing them @@ -350,3 +419,18 @@ except ImportError: import inspect def inspect_func_args(fn): return inspect.getargspec(fn) + +def read_file(path, mode='rb'): + fp = open(path, mode) + try: + data = fp.read() + return data + finally: + fp.close() + +def load_module(module_id, path): + fp = open(path, 'rb') + try: + return imp.load_source(module_id, path, fp) + finally: + fp.close()