Updated mako lib

mako/__init__.py

@@ -1,9 +1,9 @@
 # mako/__init__.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 
-__version__ = '0.7.2'
+__version__ = '0.9.1'
 

mako/_ast_util.py

@@ -1,5 +1,5 @@
 # mako/_ast_util.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -31,7 +31,7 @@
     :license: Python License.
 """
 from _ast import *
-
+from mako.compat import arg_stringname
 
 BOOLOP_SYMBOLS = {
     And:        'and',
@@ -403,10 +403,10 @@ class SourceGenerator(NodeVisitor):
                 self.visit(default)
         if node.vararg is not None:
             write_comma()
-            self.write('*' + node.vararg)
+            self.write('*' + arg_stringname(node.vararg))
         if node.kwarg is not None:
             write_comma()
-            self.write('**' + node.kwarg)
+            self.write('**' + arg_stringname(node.kwarg))
 
     def decorators(self, node):
         for decorator in node.decorator_list:
@@ -659,6 +659,12 @@ class SourceGenerator(NodeVisitor):
     def visit_Name(self, node):
         self.write(node.id)
 
+    def visit_NameConstant(self, node):
+        self.write(str(node.value))
+
+    def visit_arg(self, node):
+        self.write(node.arg)
+
     def visit_Str(self, node):
         self.write(repr(node.s))
 

mako/ast.py

@@ -1,5 +1,5 @@
 # mako/ast.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,7 +7,8 @@
 """utilities for analyzing expressions and blocks of Python
 code, as well as generating Python from AST nodes"""
 
-from mako import exceptions, pyparser, util
+from mako import exceptions, pyparser, compat
+from mako.compat import arg_stringname
 import re
 
 class PythonCode(object):
@@ -33,7 +34,7 @@ class PythonCode(object):
         # - AST is less likely to break with version changes
         # (for example, the behavior of co_names changed a little bit
         # in python version 2.5)
-        if isinstance(code, basestring):
+        if isinstance(code, compat.string_types):
             expr = pyparser.parse(code.lstrip(), "exec", **exception_kwargs)
         else:
             expr = code
@@ -48,7 +49,7 @@ class ArgumentList(object):
         self.args = []
         self.declared_identifiers = set()
         self.undeclared_identifiers = set()
-        if isinstance(code, basestring):
+        if isinstance(code, compat.string_types):
             if re.match(r"\S", code) and not re.match(r",\s*$", code):
                 # if theres text and no trailing comma, insure its parsed
                 # as a tuple by adding a trailing comma
@@ -126,10 +127,10 @@ class FunctionDecl(object):
         for arg in argnames:
             default = None
             if kwargs:
-                arg = "**" + arg
+                arg = "**" + arg_stringname(arg)
                 kwargs = False
             elif varargs:
-                arg = "*" + arg
+                arg = "*" + arg_stringname(arg)
                 varargs = False
             else:
                 default = len(defaults) and defaults.pop() or None

mako/cache.py

@@ -1,10 +1,10 @@
 # mako/cache.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-from mako import exceptions, util
+from mako import compat, util
 
 _cache_plugins = util.PluginLoader("mako.cache")
 
@@ -64,7 +64,7 @@ class Cache(object):
     def __init__(self, template, *args):
         # check for a stale template calling the
         # constructor
-        if isinstance(template, basestring) and args:
+        if isinstance(template, compat.string_types) and args:
             return
         self.template = template
         self.id = template.module.__name__

mako/codegen.py

@@ -1,5 +1,5 @@
 # mako/codegen.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -11,8 +11,10 @@ import time
 import re
 from mako.pygen import PythonPrinter
 from mako import util, ast, parsetree, filters, exceptions
+from mako import compat
 
-MAGIC_NUMBER = 8
+
+MAGIC_NUMBER = 9
 
 # names which are hardwired into the
 # template and are not accessed via the
@@ -25,12 +27,13 @@ def compile(node,
                 default_filters=None,
                 buffer_filters=None,
                 imports=None,
+                future_imports=None,
                 source_encoding=None,
                 generate_magic_comment=True,
                 disable_unicode=False,
                 strict_undefined=False,
                 enable_loop=True,
-                reserved_names=()):
+                reserved_names=frozenset()):
 
     """Generate module source code given a parsetree node,
       uri, and optional source filename"""
@@ -39,7 +42,7 @@ def compile(node,
     # a bytestring itself, as we will be embedding it into
     # the generated source and we don't want to coerce the
     # result into a unicode object, in "disable_unicode" mode
-    if not util.py3k and isinstance(source_encoding, unicode):
+    if not compat.py3k and isinstance(source_encoding, compat.text_type):
         source_encoding = source_encoding.encode(source_encoding)
 
 
@@ -52,6 +55,7 @@ def compile(node,
                                             default_filters,
                                             buffer_filters,
                                             imports,
+                                            future_imports,
                                             source_encoding,
                                             generate_magic_comment,
                                             disable_unicode,
@@ -68,6 +72,7 @@ class _CompileContext(object):
                     default_filters,
                     buffer_filters,
                     imports,
+                    future_imports,
                     source_encoding,
                     generate_magic_comment,
                     disable_unicode,
@@ -79,6 +84,7 @@ class _CompileContext(object):
         self.default_filters = default_filters
         self.buffer_filters = buffer_filters
         self.imports = imports
+        self.future_imports = future_imports
         self.source_encoding = source_encoding
         self.generate_magic_comment = generate_magic_comment
         self.disable_unicode = disable_unicode
@@ -97,7 +103,6 @@ class _GenerateRenderMethod(object):
         self.compiler = compiler
         self.node = node
         self.identifier_stack = [None]
-
         self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag))
 
         if self.in_def:
@@ -153,7 +158,6 @@ class _GenerateRenderMethod(object):
         inherit = []
         namespaces = {}
         module_code = []
-        encoding =[None]
 
         self.compiler.pagetag = None
 
@@ -184,9 +188,12 @@ class _GenerateRenderMethod(object):
         # module-level names, python code
         if self.compiler.generate_magic_comment and \
             self.compiler.source_encoding:
-            self.printer.writeline("# -*- encoding:%s -*-" %
+            self.printer.writeline("# -*- coding:%s -*-" %
                                     self.compiler.source_encoding)
 
+        if self.compiler.future_imports:
+            self.printer.writeline("from __future__ import %s" %
+                                   (", ".join(self.compiler.future_imports),))
         self.printer.writeline("from mako import runtime, filters, cache")
         self.printer.writeline("UNDEFINED = runtime.UNDEFINED")
         self.printer.writeline("__M_dict_builtin = dict")
@@ -236,7 +243,7 @@ class _GenerateRenderMethod(object):
         elif len(namespaces):
             self.write_namespaces(namespaces)
 
-        return main_identifiers.topleveldefs.values()
+        return list(main_identifiers.topleveldefs.values())
 
     def write_render_callable(self, node, name, args, buffered, filtered,
             cached):
@@ -315,13 +322,13 @@ class _GenerateRenderMethod(object):
                 "except KeyError:",
                     "_mako_generate_namespaces(context)",
                 "return context.namespaces[(__name__, name)]",
-            None,None
+            None, None
             )
         self.printer.writeline("def _mako_generate_namespaces(context):")
 
 
         for node in namespaces.values():
-            if node.attributes.has_key('import'):
+            if 'import' in node.attributes:
                 self.compiler.has_ns_imports = True
             self.write_source_comment(node)
             if len(node.nodes):
@@ -456,8 +463,8 @@ class _GenerateRenderMethod(object):
         if toplevel and getattr(self.compiler, 'has_ns_imports', False):
             self.printer.writeline("_import_ns = {}")
             self.compiler.has_imports = True
-            for ident, ns in self.compiler.namespaces.iteritems():
-                if ns.attributes.has_key('import'):
+            for ident, ns in self.compiler.namespaces.items():
+                if 'import' in ns.attributes:
                     self.printer.writeline(
                             "_mako_get_namespace(context, %r)."\
                                     "_populate(_import_ns, %r)" %
@@ -541,7 +548,7 @@ class _GenerateRenderMethod(object):
         if not self.in_def and (
                                 len(self.identifiers.locally_assigned) > 0 or
                                 len(self.identifiers.argument_declared) > 0):
-            nameargs.insert(0, 'context.locals_(__M_locals)')
+            nameargs.insert(0, 'context._locals(__M_locals)')
         else:
             nameargs.insert(0, 'context')
         self.printer.writeline("def %s(%s):" % (funcname, ",".join(namedecls)))
@@ -691,21 +698,21 @@ class _GenerateRenderMethod(object):
                 "cache._ctx_get_or_create("\
                 "%s, lambda:__M_%s(%s),  context, %s__M_defname=%r)" % \
                             (cachekey, name, ','.join(pass_args),
-                            ''.join(["%s=%s, " % (k,v)
+                            ''.join(["%s=%s, " % (k, v)
                             for k, v in cache_args.items()]),
                             name
                             )
             # apply buffer_filters
             s = self.create_filter_callable(self.compiler.buffer_filters, s,
                                             False)
-            self.printer.writelines("return " + s,None)
+            self.printer.writelines("return " + s, None)
         else:
             self.printer.writelines(
                     "__M_writer(context.get('local')."
                     "cache._ctx_get_or_create("\
                     "%s, lambda:__M_%s(%s), context, %s__M_defname=%r))" %
                     (cachekey, name, ','.join(pass_args),
-                    ''.join(["%s=%s, " % (k,v)
+                    ''.join(["%s=%s, " % (k, v)
                         for k, v in cache_args.items()]),
                     name,
                     ),
@@ -784,10 +791,10 @@ class _GenerateRenderMethod(object):
             #          and end control lines, and
             #    3) any control line with no content other than comments
             if not children or (
-                    util.all(isinstance(c, (parsetree.Comment,
+                    compat.all(isinstance(c, (parsetree.Comment,
                                             parsetree.ControlLine))
                              for c in children) and
-                    util.all((node.is_ternary(c.keyword) or c.isend)
+                    compat.all((node.is_ternary(c.keyword) or c.isend)
                              for c in children
                              if isinstance(c, parsetree.ControlLine))):
                 self.printer.writeline("pass")
@@ -1115,7 +1122,7 @@ class _Identifiers(object):
                         % (node.name, ), **node.exception_kwargs)
 
         for ident in node.undeclared_identifiers():
-            if ident != 'context' and\
+            if ident != 'context' and \
                        ident not in self.declared.union(self.locally_declared):
                 self.undeclared.add(ident)
 
@@ -1129,6 +1136,12 @@ class _Identifiers(object):
         for n in node.nodes:
             n.accept_visitor(self)
 
+    def visitTextTag(self, node):
+        for ident in node.undeclared_identifiers():
+            if ident != 'context' and \
+                ident not in self.declared.union(self.locally_declared):
+                self.undeclared.add(ident)
+
     def visitIncludeTag(self, node):
         self.check_declared(node)
 

mako/compat.py

@@ -0,0 +1,167 @@
+import sys
+import time
+
+py3k = sys.version_info >= (3, 0)
+py33 = sys.version_info >= (3, 3)
+py26 = sys.version_info >= (2, 6)
+py25 = sys.version_info >= (2, 5)
+jython = sys.platform.startswith('java')
+win32 = sys.platform.startswith('win')
+pypy = hasattr(sys, 'pypy_version_info')
+
+if py3k:
+    from io import StringIO
+    import builtins as compat_builtins
+    from urllib.parse import quote_plus, unquote_plus
+    from html.entities import codepoint2name, name2codepoint
+    string_types = str,
+    binary_type = bytes
+    text_type = str
+
+    from io import BytesIO as byte_buffer
+
+    def u(s):
+        return s
+
+    def octal(lit):
+        return eval("0o" + lit)
+
+else:
+    import __builtin__ as compat_builtins
+    try:
+        from cStringIO import StringIO
+    except:
+        from StringIO import StringIO
+
+    byte_buffer = StringIO
+
+    from urllib import quote_plus, unquote_plus
+    from htmlentitydefs import codepoint2name, name2codepoint
+    string_types = basestring,
+    binary_type = str
+    text_type = unicode
+
+    def u(s):
+        return unicode(s, "utf-8")
+
+    def octal(lit):
+        return eval("0" + lit)
+
+
+if py33:
+    from importlib import machinery
+    def load_module(module_id, path):
+        return machinery.SourceFileLoader(module_id, path).load_module()
+else:
+    import imp
+    def load_module(module_id, path):
+        fp = open(path, 'rb')
+        try:
+            return imp.load_source(module_id, path, fp)
+        finally:
+            fp.close()
+
+
+def exception_as():
+    return sys.exc_info()[1]
+
+try:
+    import threading
+    if py3k:
+        import _thread as thread
+    else:
+        import thread
+except ImportError:
+    import dummy_threading as threading
+    if py3k:
+        import _dummy_thread as thread
+    else:
+        import dummy_thread as thread
+
+if win32 or jython:
+    time_func = time.clock
+else:
+    time_func = time.time
+
+try:
+    from functools import partial
+except:
+    def partial(func, *args, **keywords):
+        def newfunc(*fargs, **fkeywords):
+            newkeywords = keywords.copy()
+            newkeywords.update(fkeywords)
+            return func(*(args + fargs), **newkeywords)
+        return newfunc
+
+if not py25:
+    def all(iterable):
+        for i in iterable:
+            if not i:
+                return False
+        return True
+
+    def exception_name(exc):
+        try:
+            return exc.__class__.__name__
+        except AttributeError:
+            return exc.__name__
+else:
+    all = all
+
+    def exception_name(exc):
+        return exc.__class__.__name__
+
+try:
+    from inspect import CO_VARKEYWORDS, CO_VARARGS
+    def inspect_func_args(fn):
+        if py3k:
+            co = fn.__code__
+        else:
+            co = fn.func_code
+
+        nargs = co.co_argcount
+        names = co.co_varnames
+        args = list(names[:nargs])
+
+        varargs = None
+        if co.co_flags & CO_VARARGS:
+            varargs = co.co_varnames[nargs]
+            nargs = nargs + 1
+        varkw = None
+        if co.co_flags & CO_VARKEYWORDS:
+            varkw = co.co_varnames[nargs]
+
+        if py3k:
+            return args, varargs, varkw, fn.__defaults__
+        else:
+            return args, varargs, varkw, fn.func_defaults
+except ImportError:
+    import inspect
+    def inspect_func_args(fn):
+        return inspect.getargspec(fn)
+
+if py3k:
+    def callable(fn):
+        return hasattr(fn, '__call__')
+else:
+    callable = callable
+
+
+################################################
+# cross-compatible metaclass implementation
+# Copyright (c) 2010-2012 Benjamin Peterson
+def with_metaclass(meta, base=object):
+    """Create a base class with a metaclass."""
+    return meta("%sBase" % meta.__name__, (base,), {})
+################################################
+
+
+def arg_stringname(func_arg):
+    """Gets the string name of a kwarg or vararg
+    In Python3.4 a function's args are
+    of _ast.arg type not _ast.name
+    """
+    if hasattr(func_arg, 'arg'):
+        return func_arg.arg
+    else:
+        return str(func_arg)

mako/exceptions.py

@@ -1,13 +1,15 @@
 # mako/exceptions.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 """exception classes"""
 
-import traceback, sys, re
-from mako import util
+import traceback
+import sys
+import re
+from mako import util, compat
 
 class MakoException(Exception):
     pass
@@ -84,12 +86,12 @@ class RichTraceback(object):
 
     @property
     def errorname(self):
-        return util.exception_name(self.error)
+        return compat.exception_name(self.error)
 
     def _init_message(self):
         """Find a unicode representation of self.error"""
         try:
-            self.message = unicode(self.error)
+            self.message = compat.text_type(self.error)
         except UnicodeError:
             try:
                 self.message = str(self.error)
@@ -97,8 +99,8 @@ class RichTraceback(object):
                 # Fallback to args as neither unicode nor
                 # str(Exception(u'\xe6')) work in Python < 2.6
                 self.message = self.error.args[0]
-        if not isinstance(self.message, unicode):
-            self.message = unicode(self.message, 'ascii', 'replace')
+        if not isinstance(self.message, compat.text_type):
+            self.message = compat.text_type(self.message, 'ascii', 'replace')
 
     def _get_reformatted_records(self, records):
         for rec in records:
@@ -150,7 +152,7 @@ class RichTraceback(object):
                     template_filename = info.template_filename or filename
                 except KeyError:
                     # A normal .py file (not a Template)
-                    if not util.py3k:
+                    if not compat.py3k:
                         try:
                             fp = open(filename, 'rb')
                             encoding = util.parse_encoding(fp)
@@ -233,15 +235,25 @@ ${tback.errorname}: ${tback.message}
 """)
 
 
-try:
+def _install_pygments():
+    global syntax_highlight, pygments_html_formatter
     from mako.ext.pygmentplugin import syntax_highlight,\
             pygments_html_formatter
-except ImportError:
+
+def _install_fallback():
+    global syntax_highlight, pygments_html_formatter
     from mako.filters import html_escape
     pygments_html_formatter = None
     def syntax_highlight(filename='', language=None):
         return html_escape
 
+def _install_highlighting():
+    try:
+        _install_pygments()
+    except ImportError:
+        _install_fallback()
+_install_highlighting()
+
 def html_error_template():
     """Provides a template that renders a stack trace in an HTML format,
     providing an excerpt of code as well as substituting source template

mako/ext/autohandler.py

@@ -1,5 +1,5 @@
 # ext/autohandler.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php

mako/ext/babelplugin.py

@@ -1,14 +1,13 @@
 # ext/babelplugin.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 """gettext message extraction via Babel: http://babel.edgewall.org/"""
-from StringIO import StringIO
-
 from babel.messages.extract import extract_python
-
+from mako.compat import StringIO
+from mako import compat
 from mako import lexer, parsetree
 
 def extract(fileobj, keywords, comment_tags, options):
@@ -77,46 +76,41 @@ def extract_nodes(nodes, keywords, comment_tags, options):
         elif isinstance(node, parsetree.PageTag):
             code = node.body_decl.code
         elif isinstance(node, parsetree.CallNamespaceTag):
-            attribs = ', '.join(['%s=%s' % (key, val)
-                                 for key, val in node.attributes.iteritems()])
-            code = '{%s}' % attribs
+            code = node.expression
             child_nodes = node.nodes
         elif isinstance(node, parsetree.ControlLine):
             if node.isend:
-                translator_comments = []
                 in_translator_comments = False
                 continue
             code = node.text
         elif isinstance(node, parsetree.Code):
-            # <% and <%! blocks would provide their own translator comments
-            translator_comments = []
             in_translator_comments = False
-
             code = node.code.code
         elif isinstance(node, parsetree.Expression):
             code = node.code.code
         else:
-            translator_comments = []
-            in_translator_comments = False
             continue
 
         # Comments don't apply unless they immediately preceed the message
         if translator_comments and \
                 translator_comments[-1][0] < node.lineno - 1:
             translator_comments = []
-        else:
-            translator_comments = \
-                [comment[1] for comment in translator_comments]
 
-        if isinstance(code, unicode):
+        translator_strings = [comment[1] for comment in translator_comments]
+
+        if isinstance(code, compat.text_type):
             code = code.encode('ascii', 'backslashreplace')
-        code = StringIO(code)
+
+        used_translator_comments = False
+        code = compat.byte_buffer(code)
         for lineno, funcname, messages, python_translator_comments \
                 in extract_python(code, keywords, comment_tags, options):
             yield (node.lineno + (lineno - 1), funcname, messages,
-                   translator_comments + python_translator_comments)
+                   translator_strings + python_translator_comments)
+            used_translator_comments = True
 
-        translator_comments = []
+        if used_translator_comments:
+            translator_comments = []
         in_translator_comments = False
 
         if child_nodes:

mako/ext/beaker_cache.py

@@ -7,11 +7,11 @@ from mako.cache import CacheImpl
 _beaker_cache = None
 class BeakerCacheImpl(CacheImpl):
     """A :class:`.CacheImpl` provided for the Beaker caching system.
-    
+
     This plugin is used by default, based on the default
     value of ``'beaker'`` for the ``cache_impl`` parameter of the
     :class:`.Template` or :class:`.TemplateLookup` classes.
-    
+
     """
 
     def __init__(self, cache):
@@ -19,7 +19,7 @@ class BeakerCacheImpl(CacheImpl):
         if _beaker_cache is None:
             try:
                 from beaker import cache as beaker_cache
-            except ImportError, e:
+            except ImportError:
                 raise exceptions.RuntimeException(
                             "the Beaker package is required to use cache "
                             "functionality.")
@@ -60,11 +60,11 @@ class BeakerCacheImpl(CacheImpl):
     def put(self, key, value, **kw):
         cache, kw = self._get_cache(**kw)
         cache.put(key, value, **kw)
- 
+
     def get(self, key, **kw):
         cache, kw = self._get_cache(**kw)
         return cache.get(key, **kw)
- 
+
     def invalidate(self, key, **kw):
         cache, kw = self._get_cache(**kw)
         cache.remove_value(key, **kw)

mako/ext/preprocessors.py

@@ -1,5 +1,5 @@
 # ext/preprocessors.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php

mako/ext/pygmentplugin.py

@@ -1,5 +1,5 @@
 # ext/pygmentplugin.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -13,7 +13,7 @@ from pygments.token import \
      Text, Comment, Operator, Keyword, Name, String, Other
 from pygments.formatters.html import HtmlFormatter
 from pygments import highlight
-from mako import util
+from mako import compat
 
 class MakoLexer(RegexLexer):
     name = 'Mako'
@@ -110,7 +110,7 @@ pygments_html_formatter = HtmlFormatter(cssclass='syntax-highlighted',
                                         linenos=True)
 def syntax_highlight(filename='', language=None):
     mako_lexer = MakoLexer()
-    if util.py3k:
+    if compat.py3k:
         python_lexer = Python3Lexer()
     else:
         python_lexer = PythonLexer()

mako/ext/turbogears.py

@@ -1,10 +1,11 @@
 # ext/turbogears.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-import re, inspect
+import inspect
+from mako import compat
 from mako.lookup import TemplateLookup
 from mako.template import Template
 
@@ -19,13 +20,13 @@ class TGPlugin(object):
 
         # Pull the options out and initialize the lookup
         lookup_options = {}
-        for k, v in options.iteritems():
+        for k, v in options.items():
             if k.startswith('mako.'):
                 lookup_options[k[5:]] = v
             elif k in ['directories', 'filesystem_checks', 'module_directory']:
                 lookup_options[k] = v
         self.lookup = TemplateLookup(**lookup_options)
- 
+
         self.tmpl_options = {}
         # transfer lookup args to template args, based on those available
         # in getargspec
@@ -46,7 +47,7 @@ class TGPlugin(object):
         return self.lookup.get_template(templatename)
 
     def render(self, info, format="html", fragment=False, template=None):
-        if isinstance(template, basestring):
+        if isinstance(template, compat.string_types):
             template = self.load_template(template)
 
         # Load extra vars func if provided

mako/filters.py

@@ -1,29 +1,38 @@
 # mako/filters.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 
-import re, urllib, htmlentitydefs, codecs
-from StringIO import StringIO
-from mako import util
+import re
+import codecs
+
+from mako.compat import quote_plus, unquote_plus, codepoint2name, \
+        name2codepoint
+
+from mako import compat
 
 xml_escapes = {
-    '&' : '&amp;',
-    '>' : '&gt;',
-    '<' : '&lt;',
-    '"' : '&#34;',   # also &quot; in html-only
-    "'" : '&#39;'    # also &apos; in html-only
+    '&': '&amp;',
+    '>': '&gt;',
+    '<': '&lt;',
+    '"': '&#34;',   # also &quot; in html-only
+    "'": '&#39;'    # also &apos; in html-only
 }
 
 # XXX: &quot; is valid in HTML and XML
 #      &apos; is not valid HTML, but is valid XML
 
-def legacy_html_escape(string):
+def legacy_html_escape(s):
     """legacy HTML escape for non-unicode mode."""
+    s = s.replace("&", "&amp;")
+    s = s.replace(">", "&gt;")
+    s = s.replace("<", "&lt;")
+    s = s.replace('"', "&#34;")
+    s = s.replace("'", "&#39;")
+    return s
 
-    return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string)
 
 try:
     import markupsafe
@@ -31,17 +40,16 @@ try:
 except ImportError:
     html_escape = legacy_html_escape
 
-
 def xml_escape(string):
     return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string)
 
 def url_escape(string):
     # convert into a list of octets
     string = string.encode("utf8")
-    return urllib.quote_plus(string)
+    return quote_plus(string)
 
 def url_unescape(string):
-    text = urllib.unquote_plus(string)
+    text = unquote_plus(string)
     if not is_ascii_str(text):
         text = text.decode("utf8")
     return text
@@ -53,12 +61,12 @@ def trim(string):
 class Decode(object):
     def __getattr__(self, key):
         def decode(x):
-            if isinstance(x, unicode):
+            if isinstance(x, compat.text_type):
                 return x
-            elif not isinstance(x, str):
-                return unicode(str(x), encoding=key)
+            elif not isinstance(x, compat.binary_type):
+                return compat.text_type(str(x), encoding=key)
             else:
-                return unicode(x, encoding=key)
+                return compat.text_type(x, encoding=key)
         return decode
 decode = Decode()
 
@@ -72,8 +80,8 @@ def is_ascii_str(text):
 
 class XMLEntityEscaper(object):
     def __init__(self, codepoint2name, name2codepoint):
-        self.codepoint2entity = dict([(c, u'&%s;' % n)
-                                      for c,n in codepoint2name.iteritems()])
+        self.codepoint2entity = dict([(c, compat.text_type('&%s;' % n))
+                                      for c, n in codepoint2name.items()])
         self.name2codepoint = name2codepoint
 
     def escape_entities(self, text):
@@ -81,7 +89,7 @@ class XMLEntityEscaper(object):
 
         Only characters corresponding to a named entity are replaced.
         """
-        return unicode(text).translate(self.codepoint2entity)
+        return compat.text_type(text).translate(self.codepoint2entity)
 
     def __escape(self, m):
         codepoint = ord(m.group())
@@ -102,7 +110,7 @@ class XMLEntityEscaper(object):
 
         The return value is guaranteed to be ASCII.
         """
-        return self.__escapable.sub(self.__escape, unicode(text)
+        return self.__escapable.sub(self.__escape, compat.text_type(text)
                                     ).encode('ascii')
 
     # XXX: This regexp will not match all valid XML entity names__.
@@ -127,7 +135,7 @@ class XMLEntityEscaper(object):
             # U+FFFD = "REPLACEMENT CHARACTER"
         if codepoint < 128:
             return chr(codepoint)
-        return unichr(codepoint)
+        return chr(codepoint)
 
     def unescape(self, text):
         """Unescape character references.
@@ -138,8 +146,7 @@ class XMLEntityEscaper(object):
         return self.__characterrefs.sub(self.__unescape, text)
 
 
-_html_entities_escaper = XMLEntityEscaper(htmlentitydefs.codepoint2name,
-                                          htmlentitydefs.name2codepoint)
+_html_entities_escaper = XMLEntityEscaper(codepoint2name, name2codepoint)
 
 html_entities_escape = _html_entities_escaper.escape_entities
 html_entities_unescape = _html_entities_escaper.unescape
@@ -159,7 +166,7 @@ def htmlentityreplace_errors(ex):
         # Handle encoding errors
         bad_text = ex.object[ex.start:ex.end]
         text = _html_entities_escaper.escape(bad_text)
-        return (unicode(text), ex.end)
+        return (compat.text_type(text), ex.end)
     raise ex
 
 codecs.register_error('htmlentityreplace', htmlentityreplace_errors)
@@ -168,20 +175,20 @@ codecs.register_error('htmlentityreplace', htmlentityreplace_errors)
 # TODO: options to make this dynamic per-compilation will be added in a later
 # release
 DEFAULT_ESCAPES = {
-    'x':'filters.xml_escape',
-    'h':'filters.html_escape',
-    'u':'filters.url_escape',
-    'trim':'filters.trim',
-    'entity':'filters.html_entities_escape',
-    'unicode':'unicode',
-    'decode':'decode',
-    'str':'str',
-    'n':'n'
+    'x': 'filters.xml_escape',
+    'h': 'filters.html_escape',
+    'u': 'filters.url_escape',
+    'trim': 'filters.trim',
+    'entity': 'filters.html_entities_escape',
+    'unicode': 'unicode',
+    'decode': 'decode',
+    'str': 'str',
+    'n': 'n'
 }
 
-if util.py3k:
+if compat.py3k:
     DEFAULT_ESCAPES.update({
-        'unicode':'str'
+        'unicode': 'str'
     })
 
 NON_UNICODE_ESCAPES = DEFAULT_ESCAPES.copy()

mako/lexer.py

@@ -1,13 +1,14 @@
 # mako/lexer.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 """provides the Lexer class for parsing template strings into parse trees."""
 
-import re, codecs
-from mako import parsetree, exceptions, util
+import re
+import codecs
+from mako import parsetree, exceptions, compat
 from mako.pygen import adjust_whitespace
 
 _regexp_cache = {}
@@ -29,7 +30,7 @@ class Lexer(object):
         self.disable_unicode = disable_unicode
         self.encoding = input_encoding
 
-        if util.py3k and disable_unicode:
+        if compat.py3k and disable_unicode:
             raise exceptions.UnsupportedError(
                                     "Mako for Python 3 does not "
                                     "support disabling Unicode")
@@ -173,7 +174,7 @@ class Lexer(object):
            or raw if decode_raw=False
 
         """
-        if isinstance(text, unicode):
+        if isinstance(text, compat.text_type):
             m = self._coding_re.match(text)
             encoding = m and m.group(1) or known_encoding or 'ascii'
             return encoding, text
@@ -198,7 +199,7 @@ class Lexer(object):
         if decode_raw:
             try:
                 text = text.decode(parsed_encoding)
-            except UnicodeDecodeError, e:
+            except UnicodeDecodeError:
                 raise exceptions.CompileException(
                            "Unicode decode operation of encoding '%s' failed" %
                            parsed_encoding,
@@ -344,8 +345,6 @@ class Lexer(object):
                  |
                  (?=\${)      # an expression
                  |
-                 (?=\#\*)     # multiline comment
-                 |
                  (?=</?[%&])  # a substitution or block or call start or end
                               # - don't consume
                  |

mako/lookup.py

@@ -1,5 +1,5 @@
 # mako/lookup.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -169,9 +169,11 @@ class TemplateLookup(TemplateCollection):
                         buffer_filters=(),
                         strict_undefined=False,
                         imports=None,
+                        future_imports=None,
                         enable_loop=True,
                         input_encoding=None,
-                        preprocessor=None):
+                        preprocessor=None,
+                        lexer_cls=None):
 
         self.directories = [posixpath.normpath(d) for d in
                             util.to_list(directories, ())
@@ -208,8 +210,11 @@ class TemplateLookup(TemplateCollection):
             'buffer_filters':buffer_filters,
             'strict_undefined':strict_undefined,
             'imports':imports,
+            'future_imports':future_imports,
             'enable_loop':enable_loop,
-            'preprocessor':preprocessor}
+            'preprocessor':preprocessor,
+            'lexer_cls':lexer_cls
+        }
 
         if collection_size == -1:
             self._collection = {}

mako/parsetree.py

@@ -1,12 +1,12 @@
 # mako/parsetree.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 """defines the parse tree components for Mako templates."""
 
-from mako import exceptions, ast, util, filters
+from mako import exceptions, ast, util, filters, compat
 import re
 
 class Node(object):
@@ -20,8 +20,8 @@ class Node(object):
 
     @property
     def exception_kwargs(self):
-        return {'source':self.source, 'lineno':self.lineno,
-                'pos':self.pos, 'filename':self.filename}
+        return {'source': self.source, 'lineno': self.lineno,
+                'pos': self.pos, 'filename': self.filename}
 
     def get_children(self):
         return []
@@ -204,9 +204,9 @@ class _TagMeta(type):
     _classmap = {}
 
     def __init__(cls, clsname, bases, dict):
-        if cls.__keyword__ is not None:
+        if getattr(cls, '__keyword__', None) is not None:
             cls._classmap[cls.__keyword__] = cls
-            super(_TagMeta, cls).__init__(clsname, bases, dict)
+        super(_TagMeta, cls).__init__(clsname, bases, dict)
 
     def __call__(cls, keyword, attributes, **kwargs):
         if ":" in keyword:
@@ -226,7 +226,7 @@ class _TagMeta(type):
             )
         return type.__call__(cls, keyword, attributes, **kwargs)
 
-class Tag(Node):
+class Tag(compat.with_metaclass(_TagMeta, Node)):
     """abstract base class for tags.
 
     <%sometag/>
@@ -236,8 +236,6 @@ class Tag(Node):
     </%someothertag>
 
     """
-
-    __metaclass__ = _TagMeta
     __keyword__ = None
 
     def __init__(self, keyword, attributes, expressions,
@@ -393,6 +391,13 @@ class TextTag(Tag):
                                     attributes.get('filter', ''),
                                     **self.exception_kwargs)
 
+    def undeclared_identifiers(self):
+        return self.filter_args.\
+                            undeclared_identifiers.\
+                            difference(filters.DEFAULT_ESCAPES.keys()).union(
+                        self.expression_undeclared_identifiers
+                    )
+
 class DefTag(Tag):
     __keyword__ = 'def'
 
@@ -405,11 +410,11 @@ class DefTag(Tag):
                 keyword,
                 attributes,
                 expressions,
-                ('name','filter', 'decorator'),
+                ('name', 'filter', 'decorator'),
                 ('name',),
                 **kwargs)
         name = attributes['name']
-        if re.match(r'^[\w_]+$',name):
+        if re.match(r'^[\w_]+$', name):
             raise exceptions.CompileException(
                                 "Missing parenthesis in %def",
                                 **self.exception_kwargs)
@@ -540,7 +545,7 @@ class CallNamespaceTag(Tag):
                                 namespace,
                                 defname,
                                 ",".join(["%s=%s" % (k, v) for k, v in
-                                            self.parsed_attributes.iteritems()
+                                            self.parsed_attributes.items()
                                             if k != 'args'])
                             )
         self.code = ast.PythonCode(self.expression, **self.exception_kwargs)

mako/pygen.py

@@ -1,13 +1,12 @@
 # mako/pygen.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 """utilities for generating and formatting literal Python code."""
 
-import re, string
-from StringIO import StringIO
+import re
 from mako import exceptions
 
 class PythonPrinter(object):

mako/pyparser.py

@@ -1,5 +1,5 @@
 # mako/pyparser.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -10,11 +10,11 @@ Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
 module is used.
 """
 
-from StringIO import StringIO
-from mako import exceptions, util
+from mako import exceptions, util, compat
+from mako.compat import StringIO, arg_stringname
 import operator
 
-if util.py3k:
+if compat.py3k:
     # words that cannot be assigned to (notably
     # smaller than the total keys in __builtins__)
     reserved = set(['True', 'False', 'None', 'print'])
@@ -33,13 +33,14 @@ else:
 try:
     import _ast
     util.restore__ast(_ast)
-    import _ast_util
+    from mako import _ast_util
 except ImportError:
     _ast = None
     from compiler import parse as compiler_parse
     from compiler import visitor
 
 
+
 def parse(code, mode='exec', **exception_kwargs):
     """Parse an expression into AST"""
 
@@ -48,14 +49,14 @@ def parse(code, mode='exec', **exception_kwargs):
         if _ast:
             return _ast_util.parse(code, '<unknown>', mode)
         else:
-            if isinstance(code, unicode):
+            if isinstance(code, compat.text_type):
                 code = code.encode('ascii', 'backslashreplace')
             return compiler_parse(code, mode)
-    except Exception, e:
+    except Exception:
         raise exceptions.SyntaxException(
                     "(%s) %s (%r)" % (
-                        e.__class__.__name__,
-                        e,
+                        compat.exception_as().__class__.__name__,
+                        compat.exception_as(),
                         code[0:50]
                     ), **exception_kwargs)
 
@@ -92,7 +93,7 @@ if _ast:
                 self.visit(n)
             self.in_assign_targets = in_a
 
-        if util.py3k:
+        if compat.py3k:
 
             # ExceptHandler is in Python 2, but this block only works in
             # Python 3 (and is required there)
@@ -112,6 +113,14 @@ if _ast:
             self._add_declared(node.name)
             self._visit_function(node, False)
 
+        def _expand_tuples(self, args):
+            for arg in args:
+                if isinstance(arg, _ast.Tuple):
+                    for n in arg.elts:
+                        yield n
+                else:
+                    yield arg
+
         def _visit_function(self, node, islambda):
 
             # push function state onto stack.  dont log any more
@@ -125,7 +134,7 @@ if _ast:
 
             local_ident_stack = self.local_ident_stack
             self.local_ident_stack = local_ident_stack.union([
-                arg_id(arg) for arg in node.args.args
+                arg_id(arg) for arg in self._expand_tuples(node.args.args)
             ])
             if islambda:
                 self.visit(node.body)
@@ -148,7 +157,7 @@ if _ast:
 
         def visit_Name(self, node):
             if isinstance(node.ctx, _ast.Store):
-                # this is eqiuvalent to visit_AssName in 
+                # this is eqiuvalent to visit_AssName in
                 # compiler
                 self._add_declared(node.id)
             elif node.id not in reserved and node.id \
@@ -207,15 +216,14 @@ if _ast:
             self.listener.funcname = node.name
             argnames = [arg_id(arg) for arg in node.args.args]
             if node.args.vararg:
-                argnames.append(node.args.vararg)
+                argnames.append(arg_stringname(node.args.vararg))
             if node.args.kwarg:
-                argnames.append(node.args.kwarg)
+                argnames.append(arg_stringname(node.args.kwarg))
             self.listener.argnames = argnames
             self.listener.defaults = node.args.defaults  # ast
             self.listener.varargs = node.args.vararg
             self.listener.kwargs = node.args.kwarg
 
-
     class ExpressionGenerator(object):
 
         def __init__(self, astnode):
@@ -261,6 +269,14 @@ else:
             self._add_declared(node.name)
             self._visit_function(node, args)
 
+        def _expand_tuples(self, args):
+            for arg in args:
+                if isinstance(arg, tuple):
+                    for n in arg:
+                        yield n
+                else:
+                    yield arg
+
         def _visit_function(self, node, args):
 
             # push function state onto stack.  dont log any more
@@ -274,7 +290,7 @@ else:
 
             local_ident_stack = self.local_ident_stack
             self.local_ident_stack = local_ident_stack.union([
-                arg for arg in node.argnames
+                arg for arg in self._expand_tuples(node.argnames)
             ])
 
             for n in node.getChildNodes():
@@ -524,11 +540,32 @@ else:
                     self.visit(a)
             self.buf.write(')')
 
+        def visitLambda(self, node, *args):
+            self.buf.write('lambda ')
+
+            argnames = list(node.argnames)
+
+            kw = arg = None
+            if node.kwargs > 0:
+                kw = argnames.pop(-1)
+            if node.varargs > 0:
+                arg = argnames.pop(-1)
+
+            if arg:
+                argnames.append("*%s" % arg)
+            if kw:
+                argnames.append("**%s" % kw)
+
+            self.buf.write(", ".join(argnames))
+
+            self.buf.write(': ')
+            self.visit(node.code)
+
 
     class walker(visitor.ASTVisitor):
 
         def dispatch(self, node, *args):
-            print 'Node:', str(node)
+            print('Node:', str(node))
 
             # print "dir:", dir(node)
 

mako/runtime.py

@@ -1,5 +1,5 @@
 # mako/runtime.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,8 +7,11 @@
 """provides runtime services for templates, including Context,
 Namespace, and various helper functions."""
 
-from mako import exceptions, util
-import __builtin__, inspect, sys
+from mako import exceptions, util, compat
+from mako.compat import compat_builtins
+import inspect
+import sys
+import collections
 
 
 class Context(object):
@@ -32,7 +35,7 @@ class Context(object):
 
         # "capture" function which proxies to the
         # generic "capture" function
-        self._data['capture'] = util.partial(capture, self)
+        self._data['capture'] = compat.partial(capture, self)
 
         # "caller" stack used by def calls with content
         self.caller_stack = self._data['caller'] = CallerStack()
@@ -55,8 +58,22 @@ class Context(object):
 
     @property
     def kwargs(self):
-        """Return the dictionary of keyword arguments associated with this
-        :class:`.Context`.
+        """Return the dictionary of top level keyword arguments associated
+        with this :class:`.Context`.
+
+        This dictionary only includes the top-level arguments passed to
+        :meth:`.Template.render`.  It does not include names produced within
+        the template execution such as local variable names or special names
+        such as ``self``, ``next``, etc.
+
+        The purpose of this dictionary is primarily for the case that
+        a :class:`.Template` accepts arguments via its ``<%page>`` tag,
+        which are normally expected to be passed via :meth:`.Template.render`,
+        except the template is being called in an inheritance context,
+        using the ``body()`` method.   :attr:`.Context.kwargs` can then be
+        used to propagate these arguments to the inheriting template::
+
+            ${next.body(**context.kwargs)}
 
         """
         return self._kwargs.copy()
@@ -77,13 +94,13 @@ class Context(object):
     def keys(self):
         """Return a list of all names established in this :class:`.Context`."""
 
-        return self._data.keys()
+        return list(self._data.keys())
 
     def __getitem__(self, key):
         if key in self._data:
             return self._data[key]
         else:
-            return __builtin__.__dict__[key]
+            return compat_builtins.__dict__[key]
 
     def _push_writer(self):
         """push a capturing buffer onto this Context and return
@@ -116,7 +133,7 @@ class Context(object):
         """Return a value from this :class:`.Context`."""
 
         return self._data.get(key,
-                __builtin__.__dict__.get(key, default)
+                compat_builtins.__dict__.get(key, default)
                 )
 
     def write(self, string):
@@ -141,11 +158,18 @@ class Context(object):
         c.caller_stack = self.caller_stack
         return c
 
-    def locals_(self, d):
+    def _locals(self, d):
         """Create a new :class:`.Context` with a copy of this
-        :class:`.Context`'s current state, updated with the given dictionary."""
+        :class:`.Context`'s current state,
+        updated with the given dictionary.
 
-        if len(d) == 0:
+        The :attr:`.Context.kwargs` collection remains
+        unaffected.
+
+
+        """
+
+        if not d:
             return self
         c = self._copy()
         c._data.update(d)
@@ -165,19 +189,27 @@ class Context(object):
 class CallerStack(list):
     def __init__(self):
         self.nextcaller = None
+
     def __nonzero__(self):
-        return self._get_caller() and True or False
+        return self.__bool__()
+
+    def __bool__(self):
+        return len(self) and self._get_caller() and True or False
+
     def _get_caller(self):
         # this method can be removed once
         # codegen MAGIC_NUMBER moves past 7
         return self[-1]
+
     def __getattr__(self, key):
         return getattr(self._get_caller(), key)
+
     def _push_frame(self):
         frame = self.nextcaller or None
         self.append(frame)
         self.nextcaller = None
         return frame
+
     def _pop_frame(self):
         self.nextcaller = self.pop()
 
@@ -192,7 +224,11 @@ class Undefined(object):
     """
     def __str__(self):
         raise NameError("Undefined")
+
     def __nonzero__(self):
+        return self.__bool__()
+
+    def __bool__(self):
         return False
 
 UNDEFINED = Undefined()
@@ -336,7 +372,7 @@ class Namespace(object):
         self.context = context
         self.inherits = inherits
         if callables is not None:
-            self.callables = dict([(c.func_name, c) for c in callables])
+            self.callables = dict([(c.__name__, c) for c in callables])
 
     callables = ()
 
@@ -394,8 +430,13 @@ class Namespace(object):
 
         This accessor allows templates to supply "scalar"
         attributes which are particularly handy in inheritance
-        relationships. See the example in
-        :ref:`inheritance_toplevel`.
+        relationships.
+
+        .. seealso::
+
+            :ref:`inheritance_attr`
+
+            :ref:`namespace_attr_for_includes`
 
         """
         return _NSAttr(self)
@@ -502,7 +543,7 @@ class TemplateNamespace(Namespace):
         self.context = context
         self.inherits = inherits
         if callables is not None:
-            self.callables = dict([(c.func_name, c) for c in callables])
+            self.callables = dict([(c.__name__, c) for c in callables])
 
         if templateuri is not None:
             self.template = _lookup_template(context, templateuri,
@@ -554,7 +595,7 @@ class TemplateNamespace(Namespace):
                 yield (key, self.callables[key])
         def get(key):
             callable_ = self.template._get_def_callable(key)
-            return util.partial(callable_, self.context)
+            return compat.partial(callable_, self.context)
         for k in self.template.module._exports:
             yield (k, get(k))
 
@@ -563,7 +604,7 @@ class TemplateNamespace(Namespace):
             val = self.callables[key]
         elif self.template.has_def(key):
             callable_ = self.template._get_def_callable(key)
-            val = util.partial(callable_, self.context)
+            val = compat.partial(callable_, self.context)
         elif self.inherits:
             val = getattr(self.inherits, key)
 
@@ -584,7 +625,7 @@ class ModuleNamespace(Namespace):
         self.context = context
         self.inherits = inherits
         if callables is not None:
-            self.callables = dict([(c.func_name, c) for c in callables])
+            self.callables = dict([(c.__name__, c) for c in callables])
 
         mod = __import__(module)
         for token in module.split('.')[1:]:
@@ -602,19 +643,19 @@ class ModuleNamespace(Namespace):
         if self.callables:
             for key in self.callables:
                 yield (key, self.callables[key])
-        def get(key):
-            callable_ = getattr(self.module, key)
-            return util.partial(callable_, self.context)
-        for k in dir(self.module):
-            if k[0] != '_':
-                yield (k, get(k))
+        for key in dir(self.module):
+            if key[0] != '_':
+                callable_ = getattr(self.module, key)
+                if compat.callable(callable_):
+                    yield key, compat.partial(callable_, self.context)
+
 
     def __getattr__(self, key):
         if key in self.callables:
             val = self.callables[key]
         elif hasattr(self.module, key):
             callable_ = getattr(self.module, key)
-            val = util.partial(callable_, self.context)
+            val = compat.partial(callable_, self.context)
         elif self.inherits:
             val = getattr(self.inherits, key)
         else:
@@ -648,7 +689,7 @@ def capture(context, callable_, *args, **kwargs):
 
     """
 
-    if not callable(callable_):
+    if not compat.callable(callable_):
         raise exceptions.RuntimeException(
                            "capture() function expects a callable as "
                            "its argument (i.e. capture(func, *args, **kwargs))"
@@ -704,10 +745,10 @@ def _inherit_from(context, uri, calling_uri):
     ih = self_ns
     while ih.inherits is not None:
         ih = ih.inherits
-    lclcontext = context.locals_({'next':ih})
+    lclcontext = context._locals({'next': ih})
     ih.inherits = TemplateNamespace("self:%s" % template.uri,
                                 lclcontext,
-                                template = template,
+                                template=template,
                                 populate_self=False)
     context._data['parent'] = lclcontext._data['local'] = ih.inherits
     callable_ = getattr(template.module, '_mako_inherit', None)
@@ -730,8 +771,8 @@ def _lookup_template(context, uri, relativeto):
     uri = lookup.adjust_uri(uri, relativeto)
     try:
         return lookup.get_template(uri)
-    except exceptions.TopLevelLookupException, e:
-        raise exceptions.TemplateLookupException(str(e))
+    except exceptions.TopLevelLookupException:
+        raise exceptions.TemplateLookupException(str(compat.exception_as()))
 
 def _populate_self_namespace(context, template, self_ns=None):
     if self_ns is None:
@@ -750,12 +791,12 @@ def _render(template, callable_, args, data, as_unicode=False):
     output of the given template and template callable."""
 
     if as_unicode:
-        buf = util.FastEncodingBuffer(unicode=True)
+        buf = util.FastEncodingBuffer(as_unicode=True)
     elif template.bytestring_passthrough:
-        buf = util.StringIO()
+        buf = compat.StringIO()
     else:
         buf = util.FastEncodingBuffer(
-                        unicode=as_unicode,
+                        as_unicode=as_unicode,
                         encoding=template.output_encoding,
                         errors=template.encoding_errors)
     context = Context(buf, **data)
@@ -767,7 +808,7 @@ def _render(template, callable_, args, data, as_unicode=False):
     return context._pop_buffer().getvalue()
 
 def _kwargs_for_callable(callable_, data):
-    argspec = util.inspect_func_args(callable_)
+    argspec = compat.inspect_func_args(callable_)
     # for normal pages, **pageargs is usually present
     if argspec[2]:
         return data
@@ -781,7 +822,7 @@ def _kwargs_for_callable(callable_, data):
     return kwargs
 
 def _kwargs_for_include(callable_, data, **kwargs):
-    argspec = util.inspect_func_args(callable_)
+    argspec = compat.inspect_func_args(callable_)
     namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None]
     for arg in namedargs:
         if arg != 'context' and arg in data and arg not in kwargs:
@@ -815,8 +856,8 @@ def _exec_template(callable_, context, args=None, kwargs=None):
         error = None
         try:
             callable_(context, *args, **kwargs)
-        except Exception, e:
-            _render_error(template, context, e)
+        except Exception:
+            _render_error(template, context, compat.exception_as())
         except:
             e = sys.exc_info()[0]
             _render_error(template, context, e)
@@ -831,7 +872,7 @@ def _render_error(template, context, error):
     else:
         error_template = exceptions.html_error_template()
         if context._outputting_as_unicode:
-            context._buffer_stack[:] = [util.FastEncodingBuffer(unicode=True)]
+            context._buffer_stack[:] = [util.FastEncodingBuffer(as_unicode=True)]
         else:
             context._buffer_stack[:] = [util.FastEncodingBuffer(
                                             error_template.output_encoding,

mako/template.py

@@ -1,5 +1,5 @@
 # mako/template.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -8,8 +8,15 @@
 template strings, as well as template runtime operations."""
 
 from mako.lexer import Lexer
-from mako import runtime, util, exceptions, codegen, cache
-import os, re, shutil, stat, sys, tempfile, types, weakref
+from mako import runtime, util, exceptions, codegen, cache, compat
+import os
+import re
+import shutil
+import stat
+import sys
+import tempfile
+import types
+import weakref
 
 
 class Template(object):
@@ -112,6 +119,15 @@ class Template(object):
      preamble of all generated Python modules. See the example
      in :ref:`filtering_default_filters`.
 
+    :param future_imports: String list of names to import from `__future__`.
+     These will be concatenated into a comma-separated string and inserted
+     into the beginning of the template, e.g. ``futures_imports=['FOO',
+     'BAR']`` results in ``from __future__ import FOO, BAR``.  If you're
+     interested in using features like the new division operator, you must
+     use future_imports to convey that to the renderer, as otherwise the
+     import will not appear as the first executed statement in the generated
+     code and will therefore not have the desired effect.
+
     :param input_encoding: Encoding of the template's source code.  Can
      be used in lieu of the coding comment. See
      :ref:`usage_unicode` as well as :ref:`unicode_toplevel` for
@@ -154,7 +170,7 @@ class Template(object):
 
          from mako.template import Template
          mytemplate = Template(
-                         file="index.html",
+                         filename="index.html",
                          module_directory="/path/to/modules",
                          module_writer=module_writer
                      )
@@ -172,6 +188,12 @@ class Template(object):
      result of the callable will be used as the template source
      code.
 
+    :param lexer_cls: A :class:`.Lexer` class used to parse
+     the template.   The :class:`.Lexer` class is used by
+     default.
+
+     .. versionadded:: 0.7.4
+
     :param strict_undefined: Replaces the automatic usage of
      ``UNDEFINED`` for any undeclared variables not located in
      the :class:`.Context` with an immediate raise of
@@ -190,6 +212,8 @@ class Template(object):
 
     """
 
+    lexer_cls = Lexer
+
     def __init__(self,
                     text=None,
                     filename=None,
@@ -215,8 +239,10 @@ class Template(object):
                     buffer_filters=(),
                     strict_undefined=False,
                     imports=None,
+                    future_imports=None,
                     enable_loop=True,
-                    preprocessor=None):
+                    preprocessor=None,
+                    lexer_cls=None):
         if uri:
             self.module_id = re.sub(r'\W', "_", uri)
             self.uri = uri
@@ -248,7 +274,7 @@ class Template(object):
         self.strict_undefined = strict_undefined
         self.module_writer = module_writer
 
-        if util.py3k and disable_unicode:
+        if compat.py3k and disable_unicode:
             raise exceptions.UnsupportedError(
                                     "Mako for Python 3 does not "
                                     "support disabling Unicode")
@@ -257,7 +283,7 @@ class Template(object):
                                     "output_encoding must be set to "
                                     "None when disable_unicode is used.")
         if default_filters is None:
-            if util.py3k or self.disable_unicode:
+            if compat.py3k or self.disable_unicode:
                 self.default_filters = ['str']
             else:
                 self.default_filters = ['unicode']
@@ -266,8 +292,12 @@ class Template(object):
         self.buffer_filters = buffer_filters
 
         self.imports = imports
+        self.future_imports = future_imports
         self.preprocessor = preprocessor
 
+        if lexer_cls is not None:
+            self.lexer_cls = lexer_cls
+
         # if plain text, compile code in memory only
         if text is not None:
             (code, module) = _compile_text(self, text, filename)
@@ -307,6 +337,7 @@ class Template(object):
             cache_type, cache_dir, cache_url
         )
 
+
     @util.memoized_property
     def reserved_names(self):
         if self.enable_loop:
@@ -345,7 +376,7 @@ class Template(object):
                             filename,
                             path,
                             self.module_writer)
-            module = util.load_module(self.module_id, path)
+            module = compat.load_module(self.module_id, path)
             del sys.modules[self.module_id]
             if module._magic_number != codegen.MAGIC_NUMBER:
                 data = util.read_file(filename)
@@ -355,7 +386,7 @@ class Template(object):
                             filename,
                             path,
                             self.module_writer)
-                module = util.load_module(self.module_id, path)
+                module = compat.load_module(self.module_id, path)
                 del sys.modules[self.module_id]
             ModuleInfo(module, path, self, filename, None, None)
         else:
@@ -495,7 +526,7 @@ class ModuleTemplate(Template):
         self.bytestring_passthrough = bytestring_passthrough or disable_unicode
         self.enable_loop = module._enable_loop
 
-        if util.py3k and disable_unicode:
+        if compat.py3k and disable_unicode:
             raise exceptions.UnsupportedError(
                                     "Mako for Python 3 does not "
                                     "support disabling Unicode")
@@ -570,13 +601,13 @@ class ModuleInfo(object):
         if self.module_source is not None:
             return self.module_source
         else:
-            return util.read_file(self.module_filename)
+            return util.read_python_file(self.module_filename)
 
     @property
     def source(self):
         if self.template_source is not None:
             if self.module._source_encoding and \
-                    not isinstance(self.template_source, unicode):
+                    not isinstance(self.template_source, compat.text_type):
                 return self.template_source.decode(
                                 self.module._source_encoding)
             else:
@@ -589,11 +620,11 @@ class ModuleInfo(object):
                 return data
 
 def _compile(template, text, filename, generate_magic_comment):
-    lexer = Lexer(text,
-                    filename,
-                    disable_unicode=template.disable_unicode,
-                    input_encoding=template.input_encoding,
-                    preprocessor=template.preprocessor)
+    lexer = template.lexer_cls(text,
+                           filename,
+                           disable_unicode=template.disable_unicode,
+                           input_encoding=template.input_encoding,
+                           preprocessor=template.preprocessor)
     node = lexer.parse()
     source = codegen.compile(node,
                             template.uri,
@@ -601,6 +632,7 @@ def _compile(template, text, filename, generate_magic_comment):
                             default_filters=template.default_filters,
                             buffer_filters=template.buffer_filters,
                             imports=template.imports,
+                            future_imports=template.future_imports,
                             source_encoding=lexer.encoding,
                             generate_magic_comment=generate_magic_comment,
                             disable_unicode=template.disable_unicode,
@@ -615,19 +647,20 @@ def _compile_text(template, text, filename):
                         generate_magic_comment=template.disable_unicode)
 
     cid = identifier
-    if not util.py3k and isinstance(cid, unicode):
+    if not compat.py3k and isinstance(cid, compat.text_type):
         cid = cid.encode()
     module = types.ModuleType(cid)
     code = compile(source, cid, 'exec')
-    exec code in module.__dict__, module.__dict__
+
+    # this exec() works for 2.4->3.3.
+    exec(code, module.__dict__, module.__dict__)
     return (source, module)
 
 def _compile_module_file(template, text, filename, outputpath, module_writer):
-    identifier = template.module_id
     source, lexer = _compile(template, text, filename,
                         generate_magic_comment=True)
 
-    if isinstance(source, unicode):
+    if isinstance(source, compat.text_type):
         source = source.encode(lexer.encoding or 'ascii')
 
     if module_writer:
@@ -643,7 +676,10 @@ def _compile_module_file(template, text, filename, outputpath, module_writer):
         shutil.move(name, outputpath)
 
 def _get_module_info_from_callable(callable_):
-    return _get_module_info(callable_.func_globals['__name__'])
+    if compat.py3k:
+        return _get_module_info(callable_.__globals__['__name__'])
+    else:
+        return _get_module_info(callable_.func_globals['__name__'])
 
 def _get_module_info(filename):
     return ModuleInfo._modules[filename]

mako/util.py

@@ -1,41 +1,15 @@
 # mako/util.py
-# Copyright (C) 2006-2012 the Mako authors and contributors <see AUTHORS file>
+# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
 #
 # This module is part of Mako and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-import imp
-import sys
-
-
-py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0)
-py26 = sys.version_info >= (2, 6)
-py24 = sys.version_info >= (2, 4) and sys.version_info < (2, 5)
-jython = sys.platform.startswith('java')
-win32 = sys.platform.startswith('win')
-
-if py3k:
-    from io import StringIO
-else:
-    try:
-        from cStringIO import StringIO
-    except:
-        from StringIO import StringIO
-
-import codecs, re, weakref, os, time, operator
+import re
 import collections
-
-try:
-    import threading
-    import thread
-except ImportError:
-    import dummy_threading as threading
-    import dummy_thread as thread
-
-if win32 or jython:
-    time_func = time.clock
-else:
-    time_func = time.time
+import codecs
+import os
+from mako import compat
+import operator
 
 def function_named(fn, name):
     """Return a function with a given __name__.
@@ -47,34 +21,6 @@ def function_named(fn, name):
     fn.__name__ = name
     return fn
 
-try:
-    from functools import partial
-except:
-    def partial(func, *args, **keywords):
-        def newfunc(*fargs, **fkeywords):
-            newkeywords = keywords.copy()
-            newkeywords.update(fkeywords)
-            return func(*(args + fargs), **newkeywords)
-        return newfunc
-
-if py24:
-    def all(iterable):
-        for i in iterable:
-            if not i:
-                return False
-        return True
-
-    def exception_name(exc):
-        try:
-            return exc.__class__.__name__
-        except AttributeError:
-            return exc.__name__
-else:
-    all = all
-
-    def exception_name(exc):
-        return exc.__class__.__name__
-
 
 class PluginLoader(object):
     def __init__(self, group):
@@ -83,7 +29,7 @@ class PluginLoader(object):
 
     def load(self, name):
         if name in self.impls:
-             return self.impls[name]()
+            return self.impls[name]()
         else:
             import pkg_resources
             for impl in pkg_resources.iter_entry_points(
@@ -92,6 +38,7 @@ class PluginLoader(object):
                 self.impls[name] = impl.load
                 return impl.load()
             else:
+                from mako import exceptions
                 raise exceptions.RuntimeException(
                         "Can't load plugin %s %s" %
                         (self.group, name))
@@ -112,7 +59,7 @@ def verify_directory(dir):
     while not os.path.exists(dir):
         try:
             tries += 1
-            os.makedirs(dir, 0775)
+            os.makedirs(dir, compat.octal("0775"))
         except:
             if tries > 5:
                 raise
@@ -180,14 +127,14 @@ class FastEncodingBuffer(object):
     """a very rudimentary buffer that is faster than StringIO,
     but doesn't crash on unicode data like cStringIO."""
 
-    def __init__(self, encoding=None, errors='strict', unicode=False):
+    def __init__(self, encoding=None, errors='strict', as_unicode=False):
         self.data = collections.deque()
         self.encoding = encoding
-        if unicode:
-            self.delim = u''
+        if as_unicode:
+            self.delim = compat.u('')
         else:
             self.delim = ''
-        self.unicode = unicode
+        self.as_unicode = as_unicode
         self.errors = errors
         self.write = self.data.append
 
@@ -215,7 +162,7 @@ class LRUCache(dict):
         def __init__(self, key, value):
             self.key = key
             self.value = value
-            self.timestamp = time_func()
+            self.timestamp = compat.time_func()
         def __repr__(self):
             return repr(self.value)
 
@@ -225,7 +172,7 @@ class LRUCache(dict):
 
     def __getitem__(self, key):
         item = dict.__getitem__(self, key)
-        item.timestamp = time_func()
+        item.timestamp = compat.time_func()
         return item.value
 
     def values(self):
@@ -300,9 +247,8 @@ def parse_encoding(fp):
 
         if has_bom:
             if m:
-                raise SyntaxError, \
-                      "python refuses to compile code with both a UTF8" \
-                      " byte-order-mark and a magic encoding comment"
+                raise SyntaxError("python refuses to compile code with both a UTF8" \
+                      " byte-order-mark and a magic encoding comment")
             return 'utf_8'
         elif m:
             return m.group(1)
@@ -317,7 +263,7 @@ def sorted_dict_repr(d):
     Used by the lexer unit test to compare parse trees based on strings.
 
     """
-    keys = d.keys()
+    keys = list(d.keys())
     keys.sort()
     return "{" + ", ".join(["%r: %r" % (k, d[k]) for k in keys]) + "}"
 
@@ -397,28 +343,6 @@ mako in baz not in mako""", '<unknown>', 'exec', _ast.PyCF_ONLY_AST)
     _ast.NotIn = type(m.body[12].value.ops[1])
 
 
-try:
-    from inspect import CO_VARKEYWORDS, CO_VARARGS
-    def inspect_func_args(fn):
-        co = fn.func_code
-
-        nargs = co.co_argcount
-        names = co.co_varnames
-        args = list(names[:nargs])
-
-        varargs = None
-        if co.co_flags & CO_VARARGS:
-            varargs = co.co_varnames[nargs]
-            nargs = nargs + 1
-        varkw = None
-        if co.co_flags & CO_VARKEYWORDS:
-            varkw = co.co_varnames[nargs]
-
-        return args, varargs, varkw, fn.func_defaults
-except ImportError:
-    import inspect
-    def inspect_func_args(fn):
-        return inspect.getargspec(fn)
 
 def read_file(path, mode='rb'):
     fp = open(path, mode)
@@ -428,9 +352,14 @@ def read_file(path, mode='rb'):
     finally:
         fp.close()
 
-def load_module(module_id, path):
-    fp = open(path, 'rb')
+def read_python_file(path):
+    fp = open(path, "rb")
     try:
-        return imp.load_source(module_id, path, fp)
+        encoding = parse_encoding(fp)
+        data = fp.read()
+        if encoding:
+            data = data.decode(encoding)
+        return data
     finally:
         fp.close()
+