diff --git a/venv/Lib/site-packages/jinja2/meta.py b/venv/Lib/site-packages/jinja2/meta.py new file mode 100644 index 0000000..298499e --- /dev/null +++ b/venv/Lib/site-packages/jinja2/meta.py @@ -0,0 +1,112 @@ +"""Functions that expose information about templates that might be +interesting for introspection. +""" + +import typing as t + +from . import nodes +from .compiler import CodeGenerator +from .compiler import Frame + +if t.TYPE_CHECKING: + from .environment import Environment + + +class TrackingCodeGenerator(CodeGenerator): + """We abuse the code generator for introspection.""" + + def __init__(self, environment: "Environment") -> None: + super().__init__(environment, "", "") + self.undeclared_identifiers: t.Set[str] = set() + + def write(self, x: str) -> None: + """Don't write.""" + + def enter_frame(self, frame: Frame) -> None: + """Remember all undeclared identifiers.""" + super().enter_frame(frame) + + for _, (action, param) in frame.symbols.loads.items(): + if action == "resolve" and param not in self.environment.globals: + self.undeclared_identifiers.add(param) + + +def find_undeclared_variables(ast: nodes.Template) -> t.Set[str]: + """Returns a set of all variables in the AST that will be looked up from + the context at runtime. Because at compile time it's not known which + variables will be used depending on the path the execution takes at + runtime, all variables are returned. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}') + >>> meta.find_undeclared_variables(ast) == {'bar'} + True + + .. admonition:: Implementation + + Internally the code generator is used for finding undeclared variables. + This is good to know because the code generator might raise a + :exc:`TemplateAssertionError` during compilation and as a matter of + fact this function can currently raise that exception as well. + """ + codegen = TrackingCodeGenerator(ast.environment) # type: ignore + codegen.visit(ast) + return codegen.undeclared_identifiers + + +_ref_types = (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include) +_RefType = t.Union[nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include] + + +def find_referenced_templates(ast: nodes.Template) -> t.Iterator[t.Optional[str]]: + """Finds all the referenced templates from the AST. This will return an + iterator over all the hardcoded template extensions, inclusions and + imports. If dynamic inheritance or inclusion is used, `None` will be + yielded. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}') + >>> list(meta.find_referenced_templates(ast)) + ['layout.html', None] + + This function is useful for dependency tracking. For example if you want + to rebuild parts of the website after a layout template has changed. + """ + template_name: t.Any + + for node in ast.find_all(_ref_types): + template: nodes.Expr = node.template # type: ignore + + if not isinstance(template, nodes.Const): + # a tuple with some non consts in there + if isinstance(template, (nodes.Tuple, nodes.List)): + for template_name in template.items: + # something const, only yield the strings and ignore + # non-string consts that really just make no sense + if isinstance(template_name, nodes.Const): + if isinstance(template_name.value, str): + yield template_name.value + # something dynamic in there + else: + yield None + # something dynamic we don't know about here + else: + yield None + continue + # constant is a basestring, direct template name + if isinstance(template.value, str): + yield template.value + # a tuple or list (latter *should* not happen) made of consts, + # yield the consts that are strings. We could warn here for + # non string values + elif isinstance(node, nodes.Include) and isinstance( + template.value, (tuple, list) + ): + for template_name in template.value: + if isinstance(template_name, str): + yield template_name + # something else we don't care about, we could warn here + else: + yield None diff --git a/venv/Lib/site-packages/jinja2/nativetypes.py b/venv/Lib/site-packages/jinja2/nativetypes.py new file mode 100644 index 0000000..71db8cc --- /dev/null +++ b/venv/Lib/site-packages/jinja2/nativetypes.py @@ -0,0 +1,130 @@ +import typing as t +from ast import literal_eval +from ast import parse +from itertools import chain +from itertools import islice +from types import GeneratorType + +from . import nodes +from .compiler import CodeGenerator +from .compiler import Frame +from .compiler import has_safe_repr +from .environment import Environment +from .environment import Template + + +def native_concat(values: t.Iterable[t.Any]) -> t.Optional[t.Any]: + """Return a native Python type from the list of compiled nodes. If + the result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, + the string is returned. + + :param values: Iterable of outputs to concatenate. + """ + head = list(islice(values, 2)) + + if not head: + return None + + if len(head) == 1: + raw = head[0] + if not isinstance(raw, str): + return raw + else: + if isinstance(values, GeneratorType): + values = chain(head, values) + raw = "".join([str(v) for v in values]) + + try: + return literal_eval( + # In Python 3.10+ ast.literal_eval removes leading spaces/tabs + # from the given string. For backwards compatibility we need to + # parse the string ourselves without removing leading spaces/tabs. + parse(raw, mode="eval") + ) + except (ValueError, SyntaxError, MemoryError): + return raw + + +class NativeCodeGenerator(CodeGenerator): + """A code generator which renders Python types by not adding + ``str()`` around output nodes. + """ + + @staticmethod + def _default_finalize(value: t.Any) -> t.Any: + return value + + def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: + return repr("".join([str(v) for v in group])) + + def _output_child_to_const( + self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo + ) -> t.Any: + const = node.as_const(frame.eval_ctx) + + if not has_safe_repr(const): + raise nodes.Impossible() + + if isinstance(node, nodes.TemplateData): + return const + + return finalize.const(const) # type: ignore + + def _output_child_pre( + self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo + ) -> None: + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post( + self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo + ) -> None: + if finalize.src is not None: + self.write(")") + + +class NativeEnvironment(Environment): + """An environment that renders templates to native Python types.""" + + code_generator_class = NativeCodeGenerator + concat = staticmethod(native_concat) # type: ignore + + +class NativeTemplate(Template): + environment_class = NativeEnvironment + + def render(self, *args: t.Any, **kwargs: t.Any) -> t.Any: + """Render the template to produce a native Python type. If the + result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed + with :func:`ast.literal_eval`, the parsed value is returned. + Otherwise, the string is returned. + """ + ctx = self.new_context(dict(*args, **kwargs)) + + try: + return self.environment_class.concat( # type: ignore + self.root_render_func(ctx) + ) + except Exception: + return self.environment.handle_exception() + + async def render_async(self, *args: t.Any, **kwargs: t.Any) -> t.Any: + if not self.environment.is_async: + raise RuntimeError( + "The environment was not created with async mode enabled." + ) + + ctx = self.new_context(dict(*args, **kwargs)) + + try: + return self.environment_class.concat( # type: ignore + [n async for n in self.root_render_func(ctx)] # type: ignore + ) + except Exception: + return self.environment.handle_exception() + + +NativeEnvironment.template_class = NativeTemplate diff --git a/venv/Lib/site-packages/jinja2/nodes.py b/venv/Lib/site-packages/jinja2/nodes.py new file mode 100644 index 0000000..2f93b90 --- /dev/null +++ b/venv/Lib/site-packages/jinja2/nodes.py @@ -0,0 +1,1206 @@ +"""AST nodes generated by the parser for the compiler. Also provides +some node tree helper functions used by the parser and compiler in order +to normalize nodes. +""" + +import inspect +import operator +import typing as t +from collections import deque + +from markupsafe import Markup + +from .utils import _PassArg + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + +_NodeBound = t.TypeVar("_NodeBound", bound="Node") + +_binop_to_func: t.Dict[str, t.Callable[[t.Any, t.Any], t.Any]] = { + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + "+": operator.add, + "-": operator.sub, +} + +_uaop_to_func: t.Dict[str, t.Callable[[t.Any], t.Any]] = { + "not": operator.not_, + "+": operator.pos, + "-": operator.neg, +} + +_cmpop_to_func: t.Dict[str, t.Callable[[t.Any, t.Any], t.Any]] = { + "eq": operator.eq, + "ne": operator.ne, + "gt": operator.gt, + "gteq": operator.ge, + "lt": operator.lt, + "lteq": operator.le, + "in": lambda a, b: a in b, + "notin": lambda a, b: a not in b, +} + + +class Impossible(Exception): + """Raised if the node could not perform a requested action.""" + + +class NodeType(type): + """A metaclass for nodes that handles the field and attribute + inheritance. fields and attributes from the parent class are + automatically forwarded to the child.""" + + def __new__(mcs, name, bases, d): # type: ignore + for attr in "fields", "attributes": + storage: t.List[t.Tuple[str, ...]] = [] + storage.extend(getattr(bases[0] if bases else object, attr, ())) + storage.extend(d.get(attr, ())) + assert len(bases) <= 1, "multiple inheritance not allowed" + assert len(storage) == len(set(storage)), "layout conflict" + d[attr] = tuple(storage) + d.setdefault("abstract", False) + return type.__new__(mcs, name, bases, d) + + +class EvalContext: + """Holds evaluation time information. Custom attributes can be attached + to it in extensions. + """ + + def __init__( + self, environment: "Environment", template_name: t.Optional[str] = None + ) -> None: + self.environment = environment + if callable(environment.autoescape): + self.autoescape = environment.autoescape(template_name) + else: + self.autoescape = environment.autoescape + self.volatile = False + + def save(self) -> t.Mapping[str, t.Any]: + return self.__dict__.copy() + + def revert(self, old: t.Mapping[str, t.Any]) -> None: + self.__dict__.clear() + self.__dict__.update(old) + + +def get_eval_context(node: "Node", ctx: t.Optional[EvalContext]) -> EvalContext: + if ctx is None: + if node.environment is None: + raise RuntimeError( + "if no eval context is passed, the node must have an" + " attached environment." + ) + return EvalContext(node.environment) + return ctx + + +class Node(metaclass=NodeType): + """Baseclass for all Jinja nodes. There are a number of nodes available + of different types. There are four major types: + + - :class:`Stmt`: statements + - :class:`Expr`: expressions + - :class:`Helper`: helper nodes + - :class:`Template`: the outermost wrapper node + + All nodes have fields and attributes. Fields may be other nodes, lists, + or arbitrary values. Fields are passed to the constructor as regular + positional arguments, attributes as keyword arguments. Each node has + two attributes: `lineno` (the line number of the node) and `environment`. + The `environment` attribute is set at the end of the parsing process for + all nodes automatically. + """ + + fields: t.Tuple[str, ...] = () + attributes: t.Tuple[str, ...] = ("lineno", "environment") + abstract = True + + lineno: int + environment: t.Optional["Environment"] + + def __init__(self, *fields: t.Any, **attributes: t.Any) -> None: + if self.abstract: + raise TypeError("abstract nodes are not instantiable") + if fields: + if len(fields) != len(self.fields): + if not self.fields: + raise TypeError(f"{type(self).__name__!r} takes 0 arguments") + raise TypeError( + f"{type(self).__name__!r} takes 0 or {len(self.fields)}" + f" argument{'s' if len(self.fields) != 1 else ''}" + ) + for name, arg in zip(self.fields, fields): + setattr(self, name, arg) + for attr in self.attributes: + setattr(self, attr, attributes.pop(attr, None)) + if attributes: + raise TypeError(f"unknown attribute {next(iter(attributes))!r}") + + def iter_fields( + self, + exclude: t.Optional[t.Container[str]] = None, + only: t.Optional[t.Container[str]] = None, + ) -> t.Iterator[t.Tuple[str, t.Any]]: + """This method iterates over all fields that are defined and yields + ``(key, value)`` tuples. Per default all fields are returned, but + it's possible to limit that to some fields by providing the `only` + parameter or to exclude some using the `exclude` parameter. Both + should be sets or tuples of field names. + """ + for name in self.fields: + if ( + (exclude is None and only is None) + or (exclude is not None and name not in exclude) + or (only is not None and name in only) + ): + try: + yield name, getattr(self, name) + except AttributeError: + pass + + def iter_child_nodes( + self, + exclude: t.Optional[t.Container[str]] = None, + only: t.Optional[t.Container[str]] = None, + ) -> t.Iterator["Node"]: + """Iterates over all direct child nodes of the node. This iterates + over all fields and yields the values of they are nodes. If the value + of a field is a list all the nodes in that list are returned. + """ + for _, item in self.iter_fields(exclude, only): + if isinstance(item, list): + for n in item: + if isinstance(n, Node): + yield n + elif isinstance(item, Node): + yield item + + def find(self, node_type: t.Type[_NodeBound]) -> t.Optional[_NodeBound]: + """Find the first node of a given type. If no such node exists the + return value is `None`. + """ + for result in self.find_all(node_type): + return result + + return None + + def find_all( + self, node_type: t.Union[t.Type[_NodeBound], t.Tuple[t.Type[_NodeBound], ...]] + ) -> t.Iterator[_NodeBound]: + """Find all the nodes of a given type. If the type is a tuple, + the check is performed for any of the tuple items. + """ + for child in self.iter_child_nodes(): + if isinstance(child, node_type): + yield child # type: ignore + yield from child.find_all(node_type) + + def set_ctx(self, ctx: str) -> "Node": + """Reset the context of a node and all child nodes. Per default the + parser will all generate nodes that have a 'load' context as it's the + most common one. This method is used in the parser to set assignment + targets and other nodes to a store context. + """ + todo = deque([self]) + while todo: + node = todo.popleft() + if "ctx" in node.fields: + node.ctx = ctx # type: ignore + todo.extend(node.iter_child_nodes()) + return self + + def set_lineno(self, lineno: int, override: bool = False) -> "Node": + """Set the line numbers of the node and children.""" + todo = deque([self]) + while todo: + node = todo.popleft() + if "lineno" in node.attributes: + if node.lineno is None or override: + node.lineno = lineno + todo.extend(node.iter_child_nodes()) + return self + + def set_environment(self, environment: "Environment") -> "Node": + """Set the environment for all nodes.""" + todo = deque([self]) + while todo: + node = todo.popleft() + node.environment = environment + todo.extend(node.iter_child_nodes()) + return self + + def __eq__(self, other: t.Any) -> bool: + if type(self) is not type(other): + return NotImplemented + + return tuple(self.iter_fields()) == tuple(other.iter_fields()) + + __hash__ = object.__hash__ + + def __repr__(self) -> str: + args_str = ", ".join(f"{a}={getattr(self, a, None)!r}" for a in self.fields) + return f"{type(self).__name__}({args_str})" + + def dump(self) -> str: + def _dump(node: t.Union[Node, t.Any]) -> None: + if not isinstance(node, Node): + buf.append(repr(node)) + return + + buf.append(f"nodes.{type(node).__name__}(") + if not node.fields: + buf.append(")") + return + for idx, field in enumerate(node.fields): + if idx: + buf.append(", ") + value = getattr(node, field) + if isinstance(value, list): + buf.append("[") + for idx, item in enumerate(value): + if idx: + buf.append(", ") + _dump(item) + buf.append("]") + else: + _dump(value) + buf.append(")") + + buf: t.List[str] = [] + _dump(self) + return "".join(buf) + + +class Stmt(Node): + """Base node for all statements.""" + + abstract = True + + +class Helper(Node): + """Nodes that exist in a specific context only.""" + + abstract = True + + +class Template(Node): + """Node that represents a template. This must be the outermost node that + is passed to the compiler. + """ + + fields = ("body",) + body: t.List[Node] + + +class Output(Stmt): + """A node that holds multiple expressions which are then printed out. + This is used both for the `print` statement and the regular template data. + """ + + fields = ("nodes",) + nodes: t.List["Expr"] + + +class Extends(Stmt): + """Represents an extends statement.""" + + fields = ("template",) + template: "Expr" + + +class For(Stmt): + """The for loop. `target` is the target for the iteration (usually a + :class:`Name` or :class:`Tuple`), `iter` the iterable. `body` is a list + of nodes that are used as loop-body, and `else_` a list of nodes for the + `else` block. If no else node exists it has to be an empty list. + + For filtered nodes an expression can be stored as `test`, otherwise `None`. + """ + + fields = ("target", "iter", "body", "else_", "test", "recursive") + target: Node + iter: Node + body: t.List[Node] + else_: t.List[Node] + test: t.Optional[Node] + recursive: bool + + +class If(Stmt): + """If `test` is true, `body` is rendered, else `else_`.""" + + fields = ("test", "body", "elif_", "else_") + test: Node + body: t.List[Node] + elif_: t.List["If"] + else_: t.List[Node] + + +class Macro(Stmt): + """A macro definition. `name` is the name of the macro, `args` a list of + arguments and `defaults` a list of defaults if there are any. `body` is + a list of nodes for the macro body. + """ + + fields = ("name", "args", "defaults", "body") + name: str + args: t.List["Name"] + defaults: t.List["Expr"] + body: t.List[Node] + + +class CallBlock(Stmt): + """Like a macro without a name but a call instead. `call` is called with + the unnamed macro as `caller` argument this node holds. + """ + + fields = ("call", "args", "defaults", "body") + call: "Call" + args: t.List["Name"] + defaults: t.List["Expr"] + body: t.List[Node] + + +class FilterBlock(Stmt): + """Node for filter sections.""" + + fields = ("body", "filter") + body: t.List[Node] + filter: "Filter" + + +class With(Stmt): + """Specific node for with statements. In older versions of Jinja the + with statement was implemented on the base of the `Scope` node instead. + + .. versionadded:: 2.9.3 + """ + + fields = ("targets", "values", "body") + targets: t.List["Expr"] + values: t.List["Expr"] + body: t.List[Node] + + +class Block(Stmt): + """A node that represents a block. + + .. versionchanged:: 3.0.0 + the `required` field was added. + """ + + fields = ("name", "body", "scoped", "required") + name: str + body: t.List[Node] + scoped: bool + required: bool + + +class Include(Stmt): + """A node that represents the include tag.""" + + fields = ("template", "with_context", "ignore_missing") + template: "Expr" + with_context: bool + ignore_missing: bool + + +class Import(Stmt): + """A node that represents the import tag.""" + + fields = ("template", "target", "with_context") + template: "Expr" + target: str + with_context: bool + + +class FromImport(Stmt): + """A node that represents the from import tag. It's important to not + pass unsafe names to the name attribute. The compiler translates the + attribute lookups directly into getattr calls and does *not* use the + subscript callback of the interface. As exported variables may not + start with double underscores (which the parser asserts) this is not a + problem for regular Jinja code, but if this node is used in an extension + extra care must be taken. + + The list of names may contain tuples if aliases are wanted. + """ + + fields = ("template", "names", "with_context") + template: "Expr" + names: t.List[t.Union[str, t.Tuple[str, str]]] + with_context: bool + + +class ExprStmt(Stmt): + """A statement that evaluates an expression and discards the result.""" + + fields = ("node",) + node: Node + + +class Assign(Stmt): + """Assigns an expression to a target.""" + + fields = ("target", "node") + target: "Expr" + node: Node + + +class AssignBlock(Stmt): + """Assigns a block to a target.""" + + fields = ("target", "filter", "body") + target: "Expr" + filter: t.Optional["Filter"] + body: t.List[Node] + + +class Expr(Node): + """Baseclass for all expressions.""" + + abstract = True + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + """Return the value of the expression as constant or raise + :exc:`Impossible` if this was not possible. + + An :class:`EvalContext` can be provided, if none is given + a default context is created which requires the nodes to have + an attached environment. + + .. versionchanged:: 2.4 + the `eval_ctx` parameter was added. + """ + raise Impossible() + + def can_assign(self) -> bool: + """Check if it's possible to assign something to this node.""" + return False + + +class BinExpr(Expr): + """Baseclass for all binary expressions.""" + + fields = ("left", "right") + left: Expr + right: Expr + operator: str + abstract = True + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + + # intercepted operators cannot be folded at compile time + if ( + eval_ctx.environment.sandboxed + and self.operator in eval_ctx.environment.intercepted_binops # type: ignore + ): + raise Impossible() + f = _binop_to_func[self.operator] + try: + return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx)) + except Exception as e: + raise Impossible() from e + + +class UnaryExpr(Expr): + """Baseclass for all unary expressions.""" + + fields = ("node",) + node: Expr + operator: str + abstract = True + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + + # intercepted operators cannot be folded at compile time + if ( + eval_ctx.environment.sandboxed + and self.operator in eval_ctx.environment.intercepted_unops # type: ignore + ): + raise Impossible() + f = _uaop_to_func[self.operator] + try: + return f(self.node.as_const(eval_ctx)) + except Exception as e: + raise Impossible() from e + + +class Name(Expr): + """Looks up a name or stores a value in a name. + The `ctx` of the node can be one of the following values: + + - `store`: store a value in the name + - `load`: load that name + - `param`: like `store` but if the name was defined as function parameter. + """ + + fields = ("name", "ctx") + name: str + ctx: str + + def can_assign(self) -> bool: + return self.name not in {"true", "false", "none", "True", "False", "None"} + + +class NSRef(Expr): + """Reference to a namespace value assignment""" + + fields = ("name", "attr") + name: str + attr: str + + def can_assign(self) -> bool: + # We don't need any special checks here; NSRef assignments have a + # runtime check to ensure the target is a namespace object which will + # have been checked already as it is created using a normal assignment + # which goes through a `Name` node. + return True + + +class Literal(Expr): + """Baseclass for literals.""" + + abstract = True + + +class Const(Literal): + """All constant values. The parser will return this node for simple + constants such as ``42`` or ``"foo"`` but it can be used to store more + complex values such as lists too. Only constants with a safe + representation (objects where ``eval(repr(x)) == x`` is true). + """ + + fields = ("value",) + value: t.Any + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + return self.value + + @classmethod + def from_untrusted( + cls, + value: t.Any, + lineno: t.Optional[int] = None, + environment: "t.Optional[Environment]" = None, + ) -> "Const": + """Return a const object if the value is representable as + constant value in the generated code, otherwise it will raise + an `Impossible` exception. + """ + from .compiler import has_safe_repr + + if not has_safe_repr(value): + raise Impossible() + return cls(value, lineno=lineno, environment=environment) + + +class TemplateData(Literal): + """A constant template string.""" + + fields = ("data",) + data: str + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> str: + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + if eval_ctx.autoescape: + return Markup(self.data) + return self.data + + +class Tuple(Literal): + """For loop unpacking and some other things like multiple arguments + for subscripts. Like for :class:`Name` `ctx` specifies if the tuple + is used for loading the names or storing. + """ + + fields = ("items", "ctx") + items: t.List[Expr] + ctx: str + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Tuple[t.Any, ...]: + eval_ctx = get_eval_context(self, eval_ctx) + return tuple(x.as_const(eval_ctx) for x in self.items) + + def can_assign(self) -> bool: + for item in self.items: + if not item.can_assign(): + return False + return True + + +class List(Literal): + """Any list literal such as ``[1, 2, 3]``""" + + fields = ("items",) + items: t.List[Expr] + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.List[t.Any]: + eval_ctx = get_eval_context(self, eval_ctx) + return [x.as_const(eval_ctx) for x in self.items] + + +class Dict(Literal): + """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of + :class:`Pair` nodes. + """ + + fields = ("items",) + items: t.List["Pair"] + + def as_const( + self, eval_ctx: t.Optional[EvalContext] = None + ) -> t.Dict[t.Any, t.Any]: + eval_ctx = get_eval_context(self, eval_ctx) + return dict(x.as_const(eval_ctx) for x in self.items) + + +class Pair(Helper): + """A key, value pair for dicts.""" + + fields = ("key", "value") + key: Expr + value: Expr + + def as_const( + self, eval_ctx: t.Optional[EvalContext] = None + ) -> t.Tuple[t.Any, t.Any]: + eval_ctx = get_eval_context(self, eval_ctx) + return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx) + + +class Keyword(Helper): + """A key, value pair for keyword arguments where key is a string.""" + + fields = ("key", "value") + key: str + value: Expr + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Tuple[str, t.Any]: + eval_ctx = get_eval_context(self, eval_ctx) + return self.key, self.value.as_const(eval_ctx) + + +class CondExpr(Expr): + """A conditional expression (inline if expression). (``{{ + foo if bar else baz }}``) + """ + + fields = ("test", "expr1", "expr2") + test: Expr + expr1: Expr + expr2: t.Optional[Expr] + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + if self.test.as_const(eval_ctx): + return self.expr1.as_const(eval_ctx) + + # if we evaluate to an undefined object, we better do that at runtime + if self.expr2 is None: + raise Impossible() + + return self.expr2.as_const(eval_ctx) + + +def args_as_const( + node: t.Union["_FilterTestCommon", "Call"], eval_ctx: t.Optional[EvalContext] +) -> t.Tuple[t.List[t.Any], t.Dict[t.Any, t.Any]]: + args = [x.as_const(eval_ctx) for x in node.args] + kwargs = dict(x.as_const(eval_ctx) for x in node.kwargs) + + if node.dyn_args is not None: + try: + args.extend(node.dyn_args.as_const(eval_ctx)) + except Exception as e: + raise Impossible() from e + + if node.dyn_kwargs is not None: + try: + kwargs.update(node.dyn_kwargs.as_const(eval_ctx)) + except Exception as e: + raise Impossible() from e + + return args, kwargs + + +class _FilterTestCommon(Expr): + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + node: Expr + name: str + args: t.List[Expr] + kwargs: t.List[Pair] + dyn_args: t.Optional[Expr] + dyn_kwargs: t.Optional[Expr] + abstract = True + _is_filter = True + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + + if eval_ctx.volatile: + raise Impossible() + + if self._is_filter: + env_map = eval_ctx.environment.filters + else: + env_map = eval_ctx.environment.tests + + func = env_map.get(self.name) + pass_arg = _PassArg.from_obj(func) # type: ignore + + if func is None or pass_arg is _PassArg.context: + raise Impossible() + + if eval_ctx.environment.is_async and ( + getattr(func, "jinja_async_variant", False) is True + or inspect.iscoroutinefunction(func) + ): + raise Impossible() + + args, kwargs = args_as_const(self, eval_ctx) + args.insert(0, self.node.as_const(eval_ctx)) + + if pass_arg is _PassArg.eval_context: + args.insert(0, eval_ctx) + elif pass_arg is _PassArg.environment: + args.insert(0, eval_ctx.environment) + + try: + return func(*args, **kwargs) + except Exception as e: + raise Impossible() from e + + +class Filter(_FilterTestCommon): + """Apply a filter to an expression. ``name`` is the name of the + filter, the other fields are the same as :class:`Call`. + + If ``node`` is ``None``, the filter is being used in a filter block + and is applied to the content of the block. + """ + + node: t.Optional[Expr] # type: ignore + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + if self.node is None: + raise Impossible() + + return super().as_const(eval_ctx=eval_ctx) + + +class Test(_FilterTestCommon): + """Apply a test to an expression. ``name`` is the name of the test, + the other field are the same as :class:`Call`. + + .. versionchanged:: 3.0 + ``as_const`` shares the same logic for filters and tests. Tests + check for volatile, async, and ``@pass_context`` etc. + decorators. + """ + + _is_filter = False + + +class Call(Expr): + """Calls an expression. `args` is a list of arguments, `kwargs` a list + of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args` + and `dyn_kwargs` has to be either `None` or a node that is used as + node for dynamic positional (``*args``) or keyword (``**kwargs``) + arguments. + """ + + fields = ("node", "args", "kwargs", "dyn_args", "dyn_kwargs") + node: Expr + args: t.List[Expr] + kwargs: t.List[Keyword] + dyn_args: t.Optional[Expr] + dyn_kwargs: t.Optional[Expr] + + +class Getitem(Expr): + """Get an attribute or item from an expression and prefer the item.""" + + fields = ("node", "arg", "ctx") + node: Expr + arg: Expr + ctx: str + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + if self.ctx != "load": + raise Impossible() + + eval_ctx = get_eval_context(self, eval_ctx) + + try: + return eval_ctx.environment.getitem( + self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx) + ) + except Exception as e: + raise Impossible() from e + + +class Getattr(Expr): + """Get an attribute or item from an expression that is a ascii-only + bytestring and prefer the attribute. + """ + + fields = ("node", "attr", "ctx") + node: Expr + attr: str + ctx: str + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + if self.ctx != "load": + raise Impossible() + + eval_ctx = get_eval_context(self, eval_ctx) + + try: + return eval_ctx.environment.getattr(self.node.as_const(eval_ctx), self.attr) + except Exception as e: + raise Impossible() from e + + +class Slice(Expr): + """Represents a slice object. This must only be used as argument for + :class:`Subscript`. + """ + + fields = ("start", "stop", "step") + start: t.Optional[Expr] + stop: t.Optional[Expr] + step: t.Optional[Expr] + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> slice: + eval_ctx = get_eval_context(self, eval_ctx) + + def const(obj: t.Optional[Expr]) -> t.Optional[t.Any]: + if obj is None: + return None + return obj.as_const(eval_ctx) + + return slice(const(self.start), const(self.stop), const(self.step)) + + +class Concat(Expr): + """Concatenates the list of expressions provided after converting + them to strings. + """ + + fields = ("nodes",) + nodes: t.List[Expr] + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> str: + eval_ctx = get_eval_context(self, eval_ctx) + return "".join(str(x.as_const(eval_ctx)) for x in self.nodes) + + +class Compare(Expr): + """Compares an expression with some other expressions. `ops` must be a + list of :class:`Operand`\\s. + """ + + fields = ("expr", "ops") + expr: Expr + ops: t.List["Operand"] + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + result = value = self.expr.as_const(eval_ctx) + + try: + for op in self.ops: + new_value = op.expr.as_const(eval_ctx) + result = _cmpop_to_func[op.op](value, new_value) + + if not result: + return False + + value = new_value + except Exception as e: + raise Impossible() from e + + return result + + +class Operand(Helper): + """Holds an operator and an expression.""" + + fields = ("op", "expr") + op: str + expr: Expr + + +class Mul(BinExpr): + """Multiplies the left with the right node.""" + + operator = "*" + + +class Div(BinExpr): + """Divides the left by the right node.""" + + operator = "/" + + +class FloorDiv(BinExpr): + """Divides the left by the right node and converts the + result into an integer by truncating. + """ + + operator = "//" + + +class Add(BinExpr): + """Add the left to the right node.""" + + operator = "+" + + +class Sub(BinExpr): + """Subtract the right from the left node.""" + + operator = "-" + + +class Mod(BinExpr): + """Left modulo right.""" + + operator = "%" + + +class Pow(BinExpr): + """Left to the power of right.""" + + operator = "**" + + +class And(BinExpr): + """Short circuited AND.""" + + operator = "and" + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx) + + +class Or(BinExpr): + """Short circuited OR.""" + + operator = "or" + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx) + + +class Not(UnaryExpr): + """Negate the expression.""" + + operator = "not" + + +class Neg(UnaryExpr): + """Make the expression negative.""" + + operator = "-" + + +class Pos(UnaryExpr): + """Make the expression positive (noop for most expressions)""" + + operator = "+" + + +# Helpers for extensions + + +class EnvironmentAttribute(Expr): + """Loads an attribute from the environment object. This is useful for + extensions that want to call a callback stored on the environment. + """ + + fields = ("name",) + name: str + + +class ExtensionAttribute(Expr): + """Returns the attribute of an extension bound to the environment. + The identifier is the identifier of the :class:`Extension`. + + This node is usually constructed by calling the + :meth:`~jinja2.ext.Extension.attr` method on an extension. + """ + + fields = ("identifier", "name") + identifier: str + name: str + + +class ImportedName(Expr): + """If created with an import name the import name is returned on node + access. For example ``ImportedName('cgi.escape')`` returns the `escape` + function from the cgi module on evaluation. Imports are optimized by the + compiler so there is no need to assign them to local variables. + """ + + fields = ("importname",) + importname: str + + +class InternalName(Expr): + """An internal name in the compiler. You cannot create these nodes + yourself but the parser provides a + :meth:`~jinja2.parser.Parser.free_identifier` method that creates + a new identifier for you. This identifier is not available from the + template and is not treated specially by the compiler. + """ + + fields = ("name",) + name: str + + def __init__(self) -> None: + raise TypeError( + "Can't create internal names. Use the " + "`free_identifier` method on a parser." + ) + + +class MarkSafe(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`).""" + + fields = ("expr",) + expr: Expr + + def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> Markup: + eval_ctx = get_eval_context(self, eval_ctx) + return Markup(self.expr.as_const(eval_ctx)) + + +class MarkSafeIfAutoescape(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`) but + only if autoescaping is active. + + .. versionadded:: 2.5 + """ + + fields = ("expr",) + expr: Expr + + def as_const( + self, eval_ctx: t.Optional[EvalContext] = None + ) -> t.Union[Markup, t.Any]: + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + expr = self.expr.as_const(eval_ctx) + if eval_ctx.autoescape: + return Markup(expr) + return expr + + +class ContextReference(Expr): + """Returns the current template context. It can be used like a + :class:`Name` node, with a ``'load'`` ctx and will return the + current :class:`~jinja2.runtime.Context` object. + + Here an example that assigns the current template name to a + variable named `foo`:: + + Assign(Name('foo', ctx='store'), + Getattr(ContextReference(), 'name')) + + This is basically equivalent to using the + :func:`~jinja2.pass_context` decorator when using the high-level + API, which causes a reference to the context to be passed as the + first argument to a function. + """ + + +class DerivedContextReference(Expr): + """Return the current template context including locals. Behaves + exactly like :class:`ContextReference`, but includes local + variables, such as from a ``for`` loop. + + .. versionadded:: 2.11 + """ + + +class Continue(Stmt): + """Continue a loop.""" + + +class Break(Stmt): + """Break a loop.""" + + +class Scope(Stmt): + """An artificial scope.""" + + fields = ("body",) + body: t.List[Node] + + +class OverlayScope(Stmt): + """An overlay scope for extensions. This is a largely unoptimized scope + that however can be used to introduce completely arbitrary variables into + a sub scope from a dictionary or dictionary like object. The `context` + field has to evaluate to a dictionary object. + + Example usage:: + + OverlayScope(context=self.call_method('get_context'), + body=[...]) + + .. versionadded:: 2.10 + """ + + fields = ("context", "body") + context: Expr + body: t.List[Node] + + +class EvalContextModifier(Stmt): + """Modifies the eval context. For each option that should be modified, + a :class:`Keyword` has to be added to the :attr:`options` list. + + Example to change the `autoescape` setting:: + + EvalContextModifier(options=[Keyword('autoescape', Const(True))]) + """ + + fields = ("options",) + options: t.List[Keyword] + + +class ScopedEvalContextModifier(EvalContextModifier): + """Modifies the eval context and reverts it later. Works exactly like + :class:`EvalContextModifier` but will only modify the + :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`. + """ + + fields = ("body",) + body: t.List[Node] + + +# make sure nobody creates custom nodes +def _failing_new(*args: t.Any, **kwargs: t.Any) -> "te.NoReturn": + raise TypeError("can't create custom node types") + + +NodeType.__new__ = staticmethod(_failing_new) # type: ignore +del _failing_new diff --git a/venv/Lib/site-packages/jinja2/optimizer.py b/venv/Lib/site-packages/jinja2/optimizer.py new file mode 100644 index 0000000..32d1c71 --- /dev/null +++ b/venv/Lib/site-packages/jinja2/optimizer.py @@ -0,0 +1,48 @@ +"""The optimizer tries to constant fold expressions and modify the AST +in place so that it should be faster to evaluate. + +Because the AST does not contain all the scoping information and the +compiler has to find that out, we cannot do all the optimizations we +want. For example, loop unrolling doesn't work because unrolled loops +would have a different scope. The solution would be a second syntax tree +that stored the scoping rules. +""" + +import typing as t + +from . import nodes +from .visitor import NodeTransformer + +if t.TYPE_CHECKING: + from .environment import Environment + + +def optimize(node: nodes.Node, environment: "Environment") -> nodes.Node: + """The context hint can be used to perform an static optimization + based on the context given.""" + optimizer = Optimizer(environment) + return t.cast(nodes.Node, optimizer.visit(node)) + + +class Optimizer(NodeTransformer): + def __init__(self, environment: "t.Optional[Environment]") -> None: + self.environment = environment + + def generic_visit( + self, node: nodes.Node, *args: t.Any, **kwargs: t.Any + ) -> nodes.Node: + node = super().generic_visit(node, *args, **kwargs) + + # Do constant folding. Some other nodes besides Expr have + # as_const, but folding them causes errors later on. + if isinstance(node, nodes.Expr): + try: + return nodes.Const.from_untrusted( + node.as_const(args[0] if args else None), + lineno=node.lineno, + environment=self.environment, + ) + except nodes.Impossible: + pass + + return node diff --git a/venv/Lib/site-packages/jinja2/parser.py b/venv/Lib/site-packages/jinja2/parser.py new file mode 100644 index 0000000..f411775 --- /dev/null +++ b/venv/Lib/site-packages/jinja2/parser.py @@ -0,0 +1,1049 @@ +"""Parse tokens from the lexer into nodes for the compiler.""" + +import typing +import typing as t + +from . import nodes +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .lexer import describe_token +from .lexer import describe_token_expr + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + +_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include) +_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock) + +_statement_keywords = frozenset( + [ + "for", + "if", + "block", + "extends", + "print", + "macro", + "include", + "from", + "import", + "set", + "with", + "autoescape", + ] +) +_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) + +_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = { + "add": nodes.Add, + "sub": nodes.Sub, + "mul": nodes.Mul, + "div": nodes.Div, + "floordiv": nodes.FloorDiv, + "mod": nodes.Mod, +} + + +class Parser: + """This is the central parsing class Jinja uses. It's passed to + extensions and can be used to parse expressions or statements. + """ + + def __init__( + self, + environment: "Environment", + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + state: t.Optional[str] = None, + ) -> None: + self.environment = environment + self.stream = environment._tokenize(source, name, filename, state) + self.name = name + self.filename = filename + self.closed = False + self.extensions: t.Dict[ + str, t.Callable[[Parser], t.Union[nodes.Node, t.List[nodes.Node]]] + ] = {} + for extension in environment.iter_extensions(): + for tag in extension.tags: + self.extensions[tag] = extension.parse + self._last_identifier = 0 + self._tag_stack: t.List[str] = [] + self._end_token_stack: t.List[t.Tuple[str, ...]] = [] + + def fail( + self, + msg: str, + lineno: t.Optional[int] = None, + exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError, + ) -> "te.NoReturn": + """Convenience method that raises `exc` with the message, passed + line number or last line number as well as the current name and + filename. + """ + if lineno is None: + lineno = self.stream.current.lineno + raise exc(msg, lineno, self.name, self.filename) + + def _fail_ut_eof( + self, + name: t.Optional[str], + end_token_stack: t.List[t.Tuple[str, ...]], + lineno: t.Optional[int], + ) -> "te.NoReturn": + expected: t.Set[str] = set() + for exprs in end_token_stack: + expected.update(map(describe_token_expr, exprs)) + if end_token_stack: + currently_looking: t.Optional[str] = " or ".join( + map(repr, map(describe_token_expr, end_token_stack[-1])) + ) + else: + currently_looking = None + + if name is None: + message = ["Unexpected end of template."] + else: + message = [f"Encountered unknown tag {name!r}."] + + if currently_looking: + if name is not None and name in expected: + message.append( + "You probably made a nesting mistake. Jinja is expecting this tag," + f" but currently looking for {currently_looking}." + ) + else: + message.append( + f"Jinja was looking for the following tags: {currently_looking}." + ) + + if self._tag_stack: + message.append( + "The innermost block that needs to be closed is" + f" {self._tag_stack[-1]!r}." + ) + + self.fail(" ".join(message), lineno) + + def fail_unknown_tag( + self, name: str, lineno: t.Optional[int] = None + ) -> "te.NoReturn": + """Called if the parser encounters an unknown tag. Tries to fail + with a human readable error message that could help to identify + the problem. + """ + self._fail_ut_eof(name, self._end_token_stack, lineno) + + def fail_eof( + self, + end_tokens: t.Optional[t.Tuple[str, ...]] = None, + lineno: t.Optional[int] = None, + ) -> "te.NoReturn": + """Like fail_unknown_tag but for end of template situations.""" + stack = list(self._end_token_stack) + if end_tokens is not None: + stack.append(end_tokens) + self._fail_ut_eof(None, stack, lineno) + + def is_tuple_end( + self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None + ) -> bool: + """Are we at the end of a tuple?""" + if self.stream.current.type in ("variable_end", "block_end", "rparen"): + return True + elif extra_end_rules is not None: + return self.stream.current.test_any(extra_end_rules) # type: ignore + return False + + def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName: + """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" + self._last_identifier += 1 + rv = object.__new__(nodes.InternalName) + nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) + return rv + + def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]: + """Parse a single statement.""" + token = self.stream.current + if token.type != "name": + self.fail("tag name expected", token.lineno) + self._tag_stack.append(token.value) + pop_tag = True + try: + if token.value in _statement_keywords: + f = getattr(self, f"parse_{self.stream.current.value}") + return f() # type: ignore + if token.value == "call": + return self.parse_call_block() + if token.value == "filter": + return self.parse_filter_block() + ext = self.extensions.get(token.value) + if ext is not None: + return ext(self) + + # did not work out, remove the token we pushed by accident + # from the stack so that the unknown tag fail function can + # produce a proper error message. + self._tag_stack.pop() + pop_tag = False + self.fail_unknown_tag(token.value, token.lineno) + finally: + if pop_tag: + self._tag_stack.pop() + + def parse_statements( + self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False + ) -> t.List[nodes.Node]: + """Parse multiple statements into a list until one of the end tokens + is reached. This is used to parse the body of statements as it also + parses template data if appropriate. The parser checks first if the + current token is a colon and skips it if there is one. Then it checks + for the block end and parses until if one of the `end_tokens` is + reached. Per default the active token in the stream at the end of + the call is the matched end token. If this is not wanted `drop_needle` + can be set to `True` and the end token is removed. + """ + # the first token may be a colon for python compatibility + self.stream.skip_if("colon") + + # in the future it would be possible to add whole code sections + # by adding some sort of end of statement token and parsing those here. + self.stream.expect("block_end") + result = self.subparse(end_tokens) + + # we reached the end of the template too early, the subparser + # does not check for this, so we do that now + if self.stream.current.type == "eof": + self.fail_eof(end_tokens) + + if drop_needle: + next(self.stream) + return result + + def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]: + """Parse an assign statement.""" + lineno = next(self.stream).lineno + target = self.parse_assign_target(with_namespace=True) + if self.stream.skip_if("assign"): + expr = self.parse_tuple() + return nodes.Assign(target, expr, lineno=lineno) + filter_node = self.parse_filter(None) + body = self.parse_statements(("name:endset",), drop_needle=True) + return nodes.AssignBlock(target, filter_node, body, lineno=lineno) + + def parse_for(self) -> nodes.For: + """Parse a for loop.""" + lineno = self.stream.expect("name:for").lineno + target = self.parse_assign_target(extra_end_rules=("name:in",)) + self.stream.expect("name:in") + iter = self.parse_tuple( + with_condexpr=False, extra_end_rules=("name:recursive",) + ) + test = None + if self.stream.skip_if("name:if"): + test = self.parse_expression() + recursive = self.stream.skip_if("name:recursive") + body = self.parse_statements(("name:endfor", "name:else")) + if next(self.stream).value == "endfor": + else_ = [] + else: + else_ = self.parse_statements(("name:endfor",), drop_needle=True) + return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) + + def parse_if(self) -> nodes.If: + """Parse an if construct.""" + node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) + while True: + node.test = self.parse_tuple(with_condexpr=False) + node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) + node.elif_ = [] + node.else_ = [] + token = next(self.stream) + if token.test("name:elif"): + node = nodes.If(lineno=self.stream.current.lineno) + result.elif_.append(node) + continue + elif token.test("name:else"): + result.else_ = self.parse_statements(("name:endif",), drop_needle=True) + break + return result + + def parse_with(self) -> nodes.With: + node = nodes.With(lineno=next(self.stream).lineno) + targets: t.List[nodes.Expr] = [] + values: t.List[nodes.Expr] = [] + while self.stream.current.type != "block_end": + if targets: + self.stream.expect("comma") + target = self.parse_assign_target() + target.set_ctx("param") + targets.append(target) + self.stream.expect("assign") + values.append(self.parse_expression()) + node.targets = targets + node.values = values + node.body = self.parse_statements(("name:endwith",), drop_needle=True) + return node + + def parse_autoescape(self) -> nodes.Scope: + node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) + node.options = [nodes.Keyword("autoescape", self.parse_expression())] + node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) + return nodes.Scope([node]) + + def parse_block(self) -> nodes.Block: + node = nodes.Block(lineno=next(self.stream).lineno) + node.name = self.stream.expect("name").value + node.scoped = self.stream.skip_if("name:scoped") + node.required = self.stream.skip_if("name:required") + + # common problem people encounter when switching from django + # to jinja. we do not support hyphens in block names, so let's + # raise a nicer error message in that case. + if self.stream.current.type == "sub": + self.fail( + "Block names in Jinja have to be valid Python identifiers and may not" + " contain hyphens, use an underscore instead." + ) + + node.body = self.parse_statements(("name:endblock",), drop_needle=True) + + # enforce that required blocks only contain whitespace or comments + # by asserting that the body, if not empty, is just TemplateData nodes + # with whitespace data + if node.required: + for body_node in node.body: + if not isinstance(body_node, nodes.Output) or any( + not isinstance(output_node, nodes.TemplateData) + or not output_node.data.isspace() + for output_node in body_node.nodes + ): + self.fail("Required blocks can only contain comments or whitespace") + + self.stream.skip_if("name:" + node.name) + return node + + def parse_extends(self) -> nodes.Extends: + node = nodes.Extends(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + return node + + def parse_import_context( + self, node: _ImportInclude, default: bool + ) -> _ImportInclude: + if self.stream.current.test_any( + "name:with", "name:without" + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + else: + node.with_context = default + return node + + def parse_include(self) -> nodes.Include: + node = nodes.Include(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + if self.stream.current.test("name:ignore") and self.stream.look().test( + "name:missing" + ): + node.ignore_missing = True + self.stream.skip(2) + else: + node.ignore_missing = False + return self.parse_import_context(node, True) + + def parse_import(self) -> nodes.Import: + node = nodes.Import(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:as") + node.target = self.parse_assign_target(name_only=True).name + return self.parse_import_context(node, False) + + def parse_from(self) -> nodes.FromImport: + node = nodes.FromImport(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:import") + node.names = [] + + def parse_context() -> bool: + if self.stream.current.value in { + "with", + "without", + } and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + return True + return False + + while True: + if node.names: + self.stream.expect("comma") + if self.stream.current.type == "name": + if parse_context(): + break + target = self.parse_assign_target(name_only=True) + if target.name.startswith("_"): + self.fail( + "names starting with an underline can not be imported", + target.lineno, + exc=TemplateAssertionError, + ) + if self.stream.skip_if("name:as"): + alias = self.parse_assign_target(name_only=True) + node.names.append((target.name, alias.name)) + else: + node.names.append(target.name) + if parse_context() or self.stream.current.type != "comma": + break + else: + self.stream.expect("name") + if not hasattr(node, "with_context"): + node.with_context = False + return node + + def parse_signature(self, node: _MacroCall) -> None: + args = node.args = [] + defaults = node.defaults = [] + self.stream.expect("lparen") + while self.stream.current.type != "rparen": + if args: + self.stream.expect("comma") + arg = self.parse_assign_target(name_only=True) + arg.set_ctx("param") + if self.stream.skip_if("assign"): + defaults.append(self.parse_expression()) + elif defaults: + self.fail("non-default argument follows default argument") + args.append(arg) + self.stream.expect("rparen") + + def parse_call_block(self) -> nodes.CallBlock: + node = nodes.CallBlock(lineno=next(self.stream).lineno) + if self.stream.current.type == "lparen": + self.parse_signature(node) + else: + node.args = [] + node.defaults = [] + + call_node = self.parse_expression() + if not isinstance(call_node, nodes.Call): + self.fail("expected call", node.lineno) + node.call = call_node + node.body = self.parse_statements(("name:endcall",), drop_needle=True) + return node + + def parse_filter_block(self) -> nodes.FilterBlock: + node = nodes.FilterBlock(lineno=next(self.stream).lineno) + node.filter = self.parse_filter(None, start_inline=True) # type: ignore + node.body = self.parse_statements(("name:endfilter",), drop_needle=True) + return node + + def parse_macro(self) -> nodes.Macro: + node = nodes.Macro(lineno=next(self.stream).lineno) + node.name = self.parse_assign_target(name_only=True).name + self.parse_signature(node) + node.body = self.parse_statements(("name:endmacro",), drop_needle=True) + return node + + def parse_print(self) -> nodes.Output: + node = nodes.Output(lineno=next(self.stream).lineno) + node.nodes = [] + while self.stream.current.type != "block_end": + if node.nodes: + self.stream.expect("comma") + node.nodes.append(self.parse_expression()) + return node + + @typing.overload + def parse_assign_target( + self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ... + ) -> nodes.Name: ... + + @typing.overload + def parse_assign_target( + self, + with_tuple: bool = True, + name_only: bool = False, + extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, + with_namespace: bool = False, + ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ... + + def parse_assign_target( + self, + with_tuple: bool = True, + name_only: bool = False, + extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, + with_namespace: bool = False, + ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: + """Parse an assignment target. As Jinja allows assignments to + tuples, this function can parse all allowed assignment targets. Per + default assignments to tuples are parsed, that can be disable however + by setting `with_tuple` to `False`. If only assignments to names are + wanted `name_only` can be set to `True`. The `extra_end_rules` + parameter is forwarded to the tuple parsing function. If + `with_namespace` is enabled, a namespace assignment may be parsed. + """ + target: nodes.Expr + + if name_only: + token = self.stream.expect("name") + target = nodes.Name(token.value, "store", lineno=token.lineno) + else: + if with_tuple: + target = self.parse_tuple( + simplified=True, + extra_end_rules=extra_end_rules, + with_namespace=with_namespace, + ) + else: + target = self.parse_primary(with_namespace=with_namespace) + + target.set_ctx("store") + + if not target.can_assign(): + self.fail( + f"can't assign to {type(target).__name__.lower()!r}", target.lineno + ) + + return target # type: ignore + + def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr: + """Parse an expression. Per default all expressions are parsed, if + the optional `with_condexpr` parameter is set to `False` conditional + expressions are not parsed. + """ + if with_condexpr: + return self.parse_condexpr() + return self.parse_or() + + def parse_condexpr(self) -> nodes.Expr: + lineno = self.stream.current.lineno + expr1 = self.parse_or() + expr3: t.Optional[nodes.Expr] + + while self.stream.skip_if("name:if"): + expr2 = self.parse_or() + if self.stream.skip_if("name:else"): + expr3 = self.parse_condexpr() + else: + expr3 = None + expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) + lineno = self.stream.current.lineno + return expr1 + + def parse_or(self) -> nodes.Expr: + lineno = self.stream.current.lineno + left = self.parse_and() + while self.stream.skip_if("name:or"): + right = self.parse_and() + left = nodes.Or(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_and(self) -> nodes.Expr: + lineno = self.stream.current.lineno + left = self.parse_not() + while self.stream.skip_if("name:and"): + right = self.parse_not() + left = nodes.And(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_not(self) -> nodes.Expr: + if self.stream.current.test("name:not"): + lineno = next(self.stream).lineno + return nodes.Not(self.parse_not(), lineno=lineno) + return self.parse_compare() + + def parse_compare(self) -> nodes.Expr: + lineno = self.stream.current.lineno + expr = self.parse_math1() + ops = [] + while True: + token_type = self.stream.current.type + if token_type in _compare_operators: + next(self.stream) + ops.append(nodes.Operand(token_type, self.parse_math1())) + elif self.stream.skip_if("name:in"): + ops.append(nodes.Operand("in", self.parse_math1())) + elif self.stream.current.test("name:not") and self.stream.look().test( + "name:in" + ): + self.stream.skip(2) + ops.append(nodes.Operand("notin", self.parse_math1())) + else: + break + lineno = self.stream.current.lineno + if not ops: + return expr + return nodes.Compare(expr, ops, lineno=lineno) + + def parse_math1(self) -> nodes.Expr: + lineno = self.stream.current.lineno + left = self.parse_concat() + while self.stream.current.type in ("add", "sub"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_concat() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_concat(self) -> nodes.Expr: + lineno = self.stream.current.lineno + args = [self.parse_math2()] + while self.stream.current.type == "tilde": + next(self.stream) + args.append(self.parse_math2()) + if len(args) == 1: + return args[0] + return nodes.Concat(args, lineno=lineno) + + def parse_math2(self) -> nodes.Expr: + lineno = self.stream.current.lineno + left = self.parse_pow() + while self.stream.current.type in ("mul", "div", "floordiv", "mod"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_pow() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_pow(self) -> nodes.Expr: + lineno = self.stream.current.lineno + left = self.parse_unary() + while self.stream.current.type == "pow": + next(self.stream) + right = self.parse_unary() + left = nodes.Pow(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_unary(self, with_filter: bool = True) -> nodes.Expr: + token_type = self.stream.current.type + lineno = self.stream.current.lineno + node: nodes.Expr + + if token_type == "sub": + next(self.stream) + node = nodes.Neg(self.parse_unary(False), lineno=lineno) + elif token_type == "add": + next(self.stream) + node = nodes.Pos(self.parse_unary(False), lineno=lineno) + else: + node = self.parse_primary() + node = self.parse_postfix(node) + if with_filter: + node = self.parse_filter_expr(node) + return node + + def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: + """Parse a name or literal value. If ``with_namespace`` is enabled, also + parse namespace attr refs, for use in assignments.""" + token = self.stream.current + node: nodes.Expr + if token.type == "name": + next(self.stream) + if token.value in ("true", "false", "True", "False"): + node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) + elif token.value in ("none", "None"): + node = nodes.Const(None, lineno=token.lineno) + elif with_namespace and self.stream.current.type == "dot": + # If namespace attributes are allowed at this point, and the next + # token is a dot, produce a namespace reference. + next(self.stream) + attr = self.stream.expect("name") + node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) + else: + node = nodes.Name(token.value, "load", lineno=token.lineno) + elif token.type == "string": + next(self.stream) + buf = [token.value] + lineno = token.lineno + while self.stream.current.type == "string": + buf.append(self.stream.current.value) + next(self.stream) + node = nodes.Const("".join(buf), lineno=lineno) + elif token.type in ("integer", "float"): + next(self.stream) + node = nodes.Const(token.value, lineno=token.lineno) + elif token.type == "lparen": + next(self.stream) + node = self.parse_tuple(explicit_parentheses=True) + self.stream.expect("rparen") + elif token.type == "lbracket": + node = self.parse_list() + elif token.type == "lbrace": + node = self.parse_dict() + else: + self.fail(f"unexpected {describe_token(token)!r}", token.lineno) + return node + + def parse_tuple( + self, + simplified: bool = False, + with_condexpr: bool = True, + extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, + explicit_parentheses: bool = False, + with_namespace: bool = False, + ) -> t.Union[nodes.Tuple, nodes.Expr]: + """Works like `parse_expression` but if multiple expressions are + delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. + This method could also return a regular expression instead of a tuple + if no commas where found. + + The default parsing mode is a full tuple. If `simplified` is `True` + only names and literals are parsed; ``with_namespace`` allows namespace + attr refs as well. The `no_condexpr` parameter is forwarded to + :meth:`parse_expression`. + + Because tuples do not require delimiters and may end in a bogus comma + an extra hint is needed that marks the end of a tuple. For example + for loops support tuples between `for` and `in`. In that case the + `extra_end_rules` is set to ``['name:in']``. + + `explicit_parentheses` is true if the parsing was triggered by an + expression in parentheses. This is used to figure out if an empty + tuple is a valid expression or not. + """ + lineno = self.stream.current.lineno + if simplified: + + def parse() -> nodes.Expr: + return self.parse_primary(with_namespace=with_namespace) + + else: + + def parse() -> nodes.Expr: + return self.parse_expression(with_condexpr=with_condexpr) + + args: t.List[nodes.Expr] = [] + is_tuple = False + + while True: + if args: + self.stream.expect("comma") + if self.is_tuple_end(extra_end_rules): + break + args.append(parse()) + if self.stream.current.type == "comma": + is_tuple = True + else: + break + lineno = self.stream.current.lineno + + if not is_tuple: + if args: + return args[0] + + # if we don't have explicit parentheses, an empty tuple is + # not a valid expression. This would mean nothing (literally + # nothing) in the spot of an expression would be an empty + # tuple. + if not explicit_parentheses: + self.fail( + "Expected an expression," + f" got {describe_token(self.stream.current)!r}" + ) + + return nodes.Tuple(args, "load", lineno=lineno) + + def parse_list(self) -> nodes.List: + token = self.stream.expect("lbracket") + items: t.List[nodes.Expr] = [] + while self.stream.current.type != "rbracket": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbracket": + break + items.append(self.parse_expression()) + self.stream.expect("rbracket") + return nodes.List(items, lineno=token.lineno) + + def parse_dict(self) -> nodes.Dict: + token = self.stream.expect("lbrace") + items: t.List[nodes.Pair] = [] + while self.stream.current.type != "rbrace": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbrace": + break + key = self.parse_expression() + self.stream.expect("colon") + value = self.parse_expression() + items.append(nodes.Pair(key, value, lineno=key.lineno)) + self.stream.expect("rbrace") + return nodes.Dict(items, lineno=token.lineno) + + def parse_postfix(self, node: nodes.Expr) -> nodes.Expr: + while True: + token_type = self.stream.current.type + if token_type == "dot" or token_type == "lbracket": + node = self.parse_subscript(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr: + while True: + token_type = self.stream.current.type + if token_type == "pipe": + node = self.parse_filter(node) # type: ignore + elif token_type == "name" and self.stream.current.value == "is": + node = self.parse_test(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_subscript( + self, node: nodes.Expr + ) -> t.Union[nodes.Getattr, nodes.Getitem]: + token = next(self.stream) + arg: nodes.Expr + + if token.type == "dot": + attr_token = self.stream.current + next(self.stream) + if attr_token.type == "name": + return nodes.Getattr( + node, attr_token.value, "load", lineno=token.lineno + ) + elif attr_token.type != "integer": + self.fail("expected name or number", attr_token.lineno) + arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + if token.type == "lbracket": + args: t.List[nodes.Expr] = [] + while self.stream.current.type != "rbracket": + if args: + self.stream.expect("comma") + args.append(self.parse_subscribed()) + self.stream.expect("rbracket") + if len(args) == 1: + arg = args[0] + else: + arg = nodes.Tuple(args, "load", lineno=token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + self.fail("expected subscript expression", token.lineno) + + def parse_subscribed(self) -> nodes.Expr: + lineno = self.stream.current.lineno + args: t.List[t.Optional[nodes.Expr]] + + if self.stream.current.type == "colon": + next(self.stream) + args = [None] + else: + node = self.parse_expression() + if self.stream.current.type != "colon": + return node + next(self.stream) + args = [node] + + if self.stream.current.type == "colon": + args.append(None) + elif self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + + if self.stream.current.type == "colon": + next(self.stream) + if self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + else: + args.append(None) + + return nodes.Slice(lineno=lineno, *args) # noqa: B026 + + def parse_call_args( + self, + ) -> t.Tuple[ + t.List[nodes.Expr], + t.List[nodes.Keyword], + t.Optional[nodes.Expr], + t.Optional[nodes.Expr], + ]: + token = self.stream.expect("lparen") + args = [] + kwargs = [] + dyn_args = None + dyn_kwargs = None + require_comma = False + + def ensure(expr: bool) -> None: + if not expr: + self.fail("invalid syntax for function call expression", token.lineno) + + while self.stream.current.type != "rparen": + if require_comma: + self.stream.expect("comma") + + # support for trailing comma + if self.stream.current.type == "rparen": + break + + if self.stream.current.type == "mul": + ensure(dyn_args is None and dyn_kwargs is None) + next(self.stream) + dyn_args = self.parse_expression() + elif self.stream.current.type == "pow": + ensure(dyn_kwargs is None) + next(self.stream) + dyn_kwargs = self.parse_expression() + else: + if ( + self.stream.current.type == "name" + and self.stream.look().type == "assign" + ): + # Parsing a kwarg + ensure(dyn_kwargs is None) + key = self.stream.current.value + self.stream.skip(2) + value = self.parse_expression() + kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) + else: + # Parsing an arg + ensure(dyn_args is None and dyn_kwargs is None and not kwargs) + args.append(self.parse_expression()) + + require_comma = True + + self.stream.expect("rparen") + return args, kwargs, dyn_args, dyn_kwargs + + def parse_call(self, node: nodes.Expr) -> nodes.Call: + # The lparen will be expected in parse_call_args, but the lineno + # needs to be recorded before the stream is advanced. + token = self.stream.current + args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() + return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) + + def parse_filter( + self, node: t.Optional[nodes.Expr], start_inline: bool = False + ) -> t.Optional[nodes.Expr]: + while self.stream.current.type == "pipe" or start_inline: + if not start_inline: + next(self.stream) + token = self.stream.expect("name") + name = token.value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() + else: + args = [] + kwargs = [] + dyn_args = dyn_kwargs = None + node = nodes.Filter( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + start_inline = False + return node + + def parse_test(self, node: nodes.Expr) -> nodes.Expr: + token = next(self.stream) + if self.stream.current.test("name:not"): + next(self.stream) + negated = True + else: + negated = False + name = self.stream.expect("name").value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + dyn_args = dyn_kwargs = None + kwargs: t.List[nodes.Keyword] = [] + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() + elif self.stream.current.type in { + "name", + "string", + "integer", + "float", + "lparen", + "lbracket", + "lbrace", + } and not self.stream.current.test_any("name:else", "name:or", "name:and"): + if self.stream.current.test("name:is"): + self.fail("You cannot chain multiple tests with is") + arg_node = self.parse_primary() + arg_node = self.parse_postfix(arg_node) + args = [arg_node] + else: + args = [] + node = nodes.Test( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + if negated: + node = nodes.Not(node, lineno=token.lineno) + return node + + def subparse( + self, end_tokens: t.Optional[t.Tuple[str, ...]] = None + ) -> t.List[nodes.Node]: + body: t.List[nodes.Node] = [] + data_buffer: t.List[nodes.Node] = [] + add_data = data_buffer.append + + if end_tokens is not None: + self._end_token_stack.append(end_tokens) + + def flush_data() -> None: + if data_buffer: + lineno = data_buffer[0].lineno + body.append(nodes.Output(data_buffer[:], lineno=lineno)) + del data_buffer[:] + + try: + while self.stream: + token = self.stream.current + if token.type == "data": + if token.value: + add_data(nodes.TemplateData(token.value, lineno=token.lineno)) + next(self.stream) + elif token.type == "variable_begin": + next(self.stream) + add_data(self.parse_tuple(with_condexpr=True)) + self.stream.expect("variable_end") + elif token.type == "block_begin": + flush_data() + next(self.stream) + if end_tokens is not None and self.stream.current.test_any( + *end_tokens + ): + return body + rv = self.parse_statement() + if isinstance(rv, list): + body.extend(rv) + else: + body.append(rv) + self.stream.expect("block_end") + else: + raise AssertionError("internal parsing error") + + flush_data() + finally: + if end_tokens is not None: + self._end_token_stack.pop() + return body + + def parse(self) -> nodes.Template: + """Parse the whole template into a `Template` node.""" + result = nodes.Template(self.subparse(), lineno=1) + result.set_environment(self.environment) + return result