diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d2d792..892da79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +### Templates + +- A cycle of includes no longer stops the collection of the filters a render reaches + (#284). A template reached again on the way contributes what it reaches, its names + mapped through each include, as Django renders it when the project's conditions let + the cycle turn: a template including itself with other bindings, a recursive tree, or + a parent including the child that extends it. A filter reached this way may be + reported as `exploitable-vulnerability` where it was not. Where no cycle is reached, + the filters and their order do not change. + +### Performance + +- Templates in a cycle of includes are no longer walked again on every path (#284): 20 + levels of doubled includes closed into a cycle took 56 s, and 40 templates including + one another more than a minute; both now take less than a second. + ## 0.24.0 (2026-10-23) ### Compatibility and migration diff --git a/docs/usage.md b/docs/usage.md index 00ab31e..d6870a4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -601,7 +601,10 @@ certain of it: - The filter's value is a context variable, directly or through `{% for %}` and `{% with %}`, the templates the template includes or extends by a constant name, the arguments of the filters before it, or the output of a `{% filter %}` block. A name a - tag binds shadows the context, and `{% include ... only %}` passes none of it. + tag binds shadows the context, and `{% include ... only %}` passes none of it. A cycle + of includes is followed as far as Django may render it: a template reached again on + the way applies its filters to what each include binds at every level, as a template + including itself with other bindings, or a recursive tree, does (#284). - The filter renders: a template extending another renders only its blocks, which replace the blocks of that name in what it extends, unless the block renders `{{ block.super }}`, which keeps the parent's content and the filters it applies. diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index c5ec5e0..1913ea4 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -1050,14 +1050,13 @@ def project_templates(root: Path) -> ProjectTemplates: reason = "not found" if library is None else "not read" if library.uncertain is None else "found in several files" unread.append(f"{relative}:{_position(text, load.offset)[0]} loads {load.library!r}, {reason}") calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications) - # The filters each template reaches, by template and blocks overridden, for this - # analysis only (#279). - reached: dict[tuple[str, frozenset[str]], _Walk] = {} + # What each template reaches rendered on its own, for this analysis only. + settled = _settled(files, read) filters = { name: tuple( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in _reaching(name, files, read, frozenset(), frozenset(), reached).found + for symbol, span, where, value, argument in settled.get((name, frozenset()), ()) ) ) for name in files @@ -1068,72 +1067,199 @@ def project_templates(root: Path) -> ProjectTemplates: # A filter a template's render reaches: the function, where it is applied, the template # applying it, and the names free in the rendered template reaching its value and argument. _Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]] +# A template, and the blocks the templates extending it override: what rendering the +# template applies depends on both. +_Pair = tuple[str, frozenset[str]] -class _Walk(NamedTuple): - """What a walk of a template found; whether a cycle of includes cut it short; and the - templates it went through.""" - - found: list[_Reached] - cut: bool - walked: frozenset[str] - - -_NOTHING = _Walk([], cut=False, walked=frozenset()) - - -def _reaching( - name: str, - files: Mapping[str, list[Path]], - read: Mapping[Path, _Template], - visiting: frozenset[str], - overridden: frozenset[str], - reached: dict[tuple[str, frozenset[str]], _Walk], -) -> _Walk: - """The filters rendering the template ``name`` applies, in it and in what it includes - or extends, each with the template applying it and the names free in ``name`` whose - values reach its value and its argument; the templates extending ``name`` on the way - define the ``overridden`` blocks. A template already ``visiting`` cuts the walk short. - - Each template is walked once for each set of blocks overridden (#279): ``reached`` - keeps what a walk found, its first occurrences in order, which mapped through a - reference give the first occurrences a walk path by path gives. A walk depends on - ``visiting`` only where a cycle cuts it, so one cut short is not kept, and one that - went through a template now on the way is not reused: walked from there, it would be - cut at that template.""" - - paths = files.get(name, []) - if len(paths) != 1 or paths[0] not in read: - return _NOTHING - if name in visiting: - return _Walk([], cut=True, walked=frozenset()) - kept = reached.get((name, overridden)) - if kept is not None and visiting.isdisjoint(kept.walked): - return kept - template = read[paths[0]] - found: dict[_Reached, None] = {} - cut = False - walked = {name} - for application in template.applications: - if template.rendered(application.blocks, overridden): - found.setdefault((application.symbol, application.span, name, application.value, application.argument)) - for reference in template.references: - if reference.name is None: +class _Render(NamedTuple): + """What rendering a pair applies itself, and the pairs its references render, each + with its reference.""" + + applied: tuple[_Reached, ...] + references: tuple[tuple[_Pair, _Reference], ...] + + +def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, _Render]: + """Every pair the render of a template reaches, each template rendered on its own. A + name several files have, or a file the engine could not read, renders nothing.""" + + def pair(name: str, overridden: frozenset[str]) -> _Pair | None: + paths = files.get(name, []) + return (name, overridden) if len(paths) == 1 and paths[0] in read else None + + renders: dict[_Pair, _Render] = {} + pending = [root for name in files if (root := pair(name, frozenset())) is not None] + while pending: + current = pending.pop() + if current in renders: continue - if reference.extends: - below = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced, reached) - elif template.rendered(reference.blocks, overridden): - below = _reaching(reference.name, files, read, visiting | {name}, frozenset(), reached) - else: + name, overridden = current + template = read[files[name][0]] + applied = tuple( + (a.symbol, a.span, name, a.value, a.argument) for a in template.applications if template.rendered(a.blocks, overridden) + ) + references: list[tuple[_Pair, _Reference]] = [] + for reference in template.references: + if reference.name is None: + continue + if reference.extends: + below = pair(reference.name, overridden | template.replaced) + elif template.rendered(reference.blocks, overridden): + below = pair(reference.name, frozenset()) + else: + continue + if below is not None: + references.append((below, reference)) + pending.append(below) + renders[current] = _Render(applied, tuple(references)) + return renders + + +def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]: + """The strongly connected components of the renders, each after every component its + references reach: Tarjan's algorithm, without recursion.""" + + index: dict[_Pair, int] = {} + low: dict[_Pair, int] = {} + stack: list[_Pair] = [] + on_stack: set[_Pair] = set() + components: list[list[_Pair]] = [] + for root in renders: + if root in index: + continue + # Each entry is a pair and the position of the next reference to follow. + work = [(root, 0)] + while work: + current, position = work.pop() + if position == 0: + index[current] = low[current] = len(index) + stack.append(current) + on_stack.add(current) + references = renders[current].references + for following in range(position, len(references)): + below = references[following][0] + if below not in index: + work.append((current, following + 1)) + work.append((below, 0)) + break + if below in on_stack: + low[current] = min(low[current], index[below]) + else: + if low[current] == index[current]: + component: list[_Pair] = [] + while not component or component[-1] != current: + component.append(stack.pop()) + on_stack.discard(component[-1]) + components.append(component) + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[current]) + return components + + +def _mapped(found: Iterable[_Reached], reference: _Reference) -> Iterator[_Reached]: + """The filters a referenced template reaches, their names mapped through the reference.""" + + for symbol, span, where, value, argument in found: + yield symbol, span, where, reference.maps(value), reference.maps(argument) + + +def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str]]: + """The order of the filters a pair in a cycle reaches beyond what it applies.""" + + symbol, span, where, value, argument = found + return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument) + + +# A filter application: the function, where it is applied, and the template applying it. +_Site = tuple[SymbolId, SourceSpan, str] + + +def _joined(found: Iterable[_Reached], into: dict[_Site, tuple[frozenset[str], frozenset[str]]]) -> None: + """Join into ``into`` the names reaching the value and the argument of each filter + application ``found`` holds.""" + + for symbol, span, where, value, argument in found: + values, arguments = into.get((symbol, span, where), (frozenset(), frozenset())) + into[symbol, span, where] = values | value, arguments | argument + + +def _facts(names: Mapping[_Site, tuple[frozenset[str], frozenset[str]]], sites: Iterable[_Site]) -> list[_Reached]: + """The filters at ``sites``, each with the names joined at its application.""" + + return [(symbol, span, where, *names[symbol, span, where]) for symbol, span, where in sites] + + +def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, list[_Reached]]: + """The filters the render of each pair reaches, in order (#284). + + What a pair reaches is the least set holding what it applies, and what each of its + references reaches mapped through the reference's bindings. The sets are computed + component by component, each after those its references reach, which are then + final. No cycle is cut: a template including itself with other bindings reaches what + Django renders at its next levels. + + In a cycle, the names reaching the value and the argument of each filter application + are joined: one fact per application, whatever the paths that bring each name. The + analysis reads those names position by position, joining what each brings, so the + join loses nothing, and it keeps the facts few: bindings permuted around a cycle + would otherwise make one fact of every combination of names. In a component, every + pair starts with no names and is recomputed from its references' facts; names only + join, and the applications and names of a project are finitely many, so the names of + an application grow at most once for each name. A pair is recomputed when a fact it + reads grew, until none grows. + + Once the facts are settled, a pair outside any cycle lists what it applies, then what + each reference reaches, in order, first occurrences only, as a walk of each path + would. A pair in a cycle has no such first occurrence, the cycle having no start: it + lists what it applies, then the rest by template, position, function and names. The + order thus depends only on the facts and the templates, never on the order of the + computation.""" + + renders = _renders(files, read) + reached: dict[_Pair, list[_Reached]] = {} + for component in _components(renders): + single = component[0] + if len(component) == 1 and all(below != single for below, _ in renders[single].references): + # Outside any cycle: what its references reach is final. + found = dict.fromkeys(renders[single].applied) + for below, reference in renders[single].references: + found.update(dict.fromkeys(_mapped(reached[below], reference))) + reached[single] = list(found) continue - cut = cut or below.cut - walked |= below.walked - for symbol, span, where, value, argument in below.found: - found.setdefault((symbol, span, where, reference.maps(value), reference.maps(argument))) - walk = _Walk(list(found), cut, frozenset(walked)) - if not cut: - reached[name, overridden] = walk - return walk + members = set(component) + readers: dict[_Pair, list[_Pair]] = {member: [] for member in component} + for member in component: + for below, _ in renders[member].references: + if below in members: + readers[below].append(member) + names: dict[_Pair, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component} + sizes = dict.fromkeys(component, 0) + pending = deque(component) + queued = set(component) + while pending: + member = pending.popleft() + queued.discard(member) + grown: dict[_Site, tuple[frozenset[str], frozenset[str]]] = {} + _joined(renders[member].applied, grown) + for below, reference in renders[member].references: + below_found = _facts(names[below], names[below]) if below in members else reached[below] + _joined(_mapped(below_found, reference), grown) + # The names only join, so the count of applications and names only grows. + size = sum(1 + len(values) + len(arguments) for values, arguments in grown.values()) + if size > sizes[member]: + names[member], sizes[member] = grown, size + for reader in readers[member]: + if reader not in queued: + pending.append(reader) + queued.add(reader) + for member in component: + applied = _facts(names[member], dict.fromkeys((s, span, w) for s, span, w, _, _ in renders[member].applied)) + sites = {(s, span, w) for s, span, w, _, _ in applied} + rest = _facts(names[member], (site for site in names[member] if site not in sites)) + reached[member] = [*applied, *sorted(rest, key=_canonical)] + return reached def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) -> _Template: diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py new file mode 100644 index 0000000..85eaad2 --- /dev/null +++ b/tests/test_template_filter_cycles.py @@ -0,0 +1,275 @@ +"""Acceptance tests for issue #284: a cycle of includes does not stop the collection of the +filters a template's render reaches. + +A template reached again on the way contributes what it reaches, as Django renders it +when the project's conditions let the cycle turn: the filters of each template, with the +blocks the templates extending it override, are the least fixpoint of what it applies and +what its references reach through their bindings. Each expectation below is what Django +5.1.15 applies when it renders the same templates. +""" + +from __future__ import annotations + +import itertools +import json +import random +import subprocess +import sys +from collections.abc import Callable, Iterable, Iterator +from pathlib import Path +from typing import Any + +import pytest + +from coretrace_python import engine +from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories +from coretrace_python.findings import Severity +from coretrace_python.interprocedural import discover_files +from coretrace_python.semantic.symbols import SymbolId +from coretrace_python.source import SourceId +from coretrace_python.taint import templates +from coretrace_python.taint.templates import project_templates + +TIMEOUT = 60 +FILTERS = "python.django.template.defaultfilters" +STRIPTAGS = f"{FILTERS}.striptags" +ADVISORY = Advisory( + "CVE-2099-8401", + "django", + "<4.2.17", + "striptags is quadratic in its input", + Severity.MEDIUM, + entry_points=(AdvisoryEntryPoint(SymbolId(STRIPTAGS), "a template filter"),), +) +LOCK = """version = 1 + +[[package]] +name = "app" +version = "0.1.0" +source = { virtual = "." } +dependencies = [{ name = "django" }] + +[[package]] +name = "django" +version = "4.2.16" +source = { registry = "https://pypi.org/simple" } +""" +# The child extends the parent and overrides its body; the parent includes the child in +# that body, binding v to other. Django renders b.html as striptags of v, then, through +# a.html, b.html again with the body overridden: striptags of other, then upper of other. +CYCLE = { + "app/templates/a.html": "{% extends 'b.html' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", + "app/templates/b.html": "{{ v|striptags }}\n{% block body %}{% include 'a.html' with v=other %}{% endblock %}\n", +} + + +def write(root: Path, files: dict[str, str]) -> Path: + for relative, text in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def exploitable(root: Path, context: str) -> list[tuple[int, str, str]]: + """The exploitable striptags calls a view rendering b.html with ``context`` makes: the + render line, the template holding the filter, and the line of the filter there.""" + + view = ( + "from django.http import HttpRequest\n" + "from django.shortcuts import render\n\n" + "def page(request: HttpRequest):\n" + f" return render(request, 'b.html', {context})\n" + ) + write(root, {"uv.lock": LOCK, "app/__init__.py": "", "app/views.py": view, **CYCLE}) + (root / "advisories.json").write_text(dump_advisories((ADVISORY,)), encoding="utf-8") + findings = engine.analyze_project(root, [engine.BUNDLED_PLUGINS]).findings + return sorted( + (f.span.start_line, f.metadata["through"], f.metadata["sink_line"]) + for f in findings + if f.rule_id == "exploitable-vulnerability" and f.metadata["advisory"] == ADVISORY.id + ) + + +@pytest.mark.parametrize( + "context", + ["{'v': 'fixed', 'other': request.POST['bio']}", "{'v': request.POST['bio'], 'other': 'fixed'}"], + ids=["through-the-cycle", "control"], +) +def test_input_the_parent_passes_its_child_reaches_the_parent_filter_django_applies(tmp_path: Path, context: str) -> None: + assert exploitable(tmp_path, context) == [(5, "b.html", "1")] + + +def described(root: Path) -> dict[str, list[tuple[str, str, int, int, str, tuple[str, ...], tuple[str, ...]]]]: + """Each template's filters, in order: the function, the file and the line and column it + is applied at, the template applying it, and the names reaching its value and argument.""" + + return { + name: [ + ( + str(f.symbol), + Path(str(f.location.source_id)).name, + f.location.start_line, + f.location.start_column, + f.template, + f.value, + f.argument, + ) + for f in found + ] + for name, found in project_templates(root).filters.items() + } + + +def test_a_template_including_itself_with_other_bindings_reaches_them(tmp_path: Path) -> None: + # Django applies upper to v, then, one level down, to n0; the condition ends there. + # In a cycle, the names reaching a filter application are joined into one call. + root = write(tmp_path, {"app/templates/t.html": "{{ v|upper }}{% if go %}{% include 't.html' with v=n0 go='' %}{% endif %}\n"}) + + assert described(root) == {"t.html": [(f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("n0", "v"), ())]} + + +def test_a_recursive_tree_reaches_the_children_at_every_level(tmp_path: Path) -> None: + # Django applies upper to node.name, then to the name of each node of node.children. + node = "{{ node.name|upper }}{% for child in node.children %}{% include 'node.html' with node=child %}{% endfor %}\n" + root = write(tmp_path, {"app/templates/node.html": node}) + + assert described(root) == {"node.html": [(f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.children", "node.name"), ())]} + + +COUNT_FILTERS = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python.taint.templates import project_templates\n" + "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n" +) + + +def counted(root: Path) -> dict[str, int]: + """Each template's number of filters, counted in a subprocess bounded in time.""" + + try: + completed = subprocess.run( + [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT + ) + except subprocess.TimeoutExpired: + pytest.fail(f"collecting the filters did not end within {TIMEOUT} s") + assert completed.returncode == 0, completed.stderr + counts: dict[str, int] = json.loads(completed.stdout) + return counts + + +def test_a_cycle_through_includes_doubled_at_every_level_is_collected_in_linear_time(tmp_path: Path) -> None: + # Cut at each template on the way, every walk inside the cycle is cut, and each is + # walked again on every path: 20 levels took a minute. + templates = {"app/templates/level0.html": "{{ name|upper }}{% if more %}{% include 'level60.html' %}{% endif %}"} + for level in range(1, 61): + templates[f"app/templates/level{level}.html"] = f"{{% include 'level{level - 1}.html' %}}" * 2 + + assert counted(write(tmp_path, templates)) == {f"level{level}.html": 1 for level in range(61)} + + +def test_templates_each_including_every_other_are_collected_in_polynomial_time(tmp_path: Path) -> None: + # Each template reaches the filter of every other one, through any of their orders. + names = [f"t{index}.html" for index in range(40)] + templates = { + f"app/templates/{name}": "{{ v|upper }}" + "".join(f"{{% if go %}}{{% include '{other}' %}}{{% endif %}}" for other in names if other != name) + for name in names + } + + assert counted(write(tmp_path, templates)) == {name: 40 for name in names} + + +def test_bindings_permuted_around_a_cycle_are_joined_at_each_filter(tmp_path: Path) -> None: + # Each combination of the 24 names would be its own fact; joined at each filter + # application, the chain's 7 filters are 7 calls. + rotated = " ".join(f"v{index}=v{(index + 1) % 24}" for index in range(24)) + template = ( + "{{ v0|default:v1|default:v2|default:v3|default:v4|default:v5|default:v6|upper }}\n" + "{% if go %}\n" + "{% include 't.html' with " + rotated + " go='' %}\n" + "{% include 't.html' with v0=v1 v1=v0 go='' %}\n" + "{% endif %}\n" + ) + + assert counted(write(tmp_path, {"app/templates/t.html": template})) == {"t.html": 7} + + +NAMES = [f"t{index}.html" for index in range(4)] + + +def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: + """Markup applying filters, including other templates, inside blocks of two names and + outside them, each include binding the variable to a name of its own.""" + + pieces: list[Callable[[], str]] = [ + lambda: "{{ v|upper }}", + lambda: "{{ v|striptags }}", + lambda: "{% include '" + rng.choice(NAMES) + "' %}", + lambda: "{% include '" + rng.choice(NAMES) + "' with v=" + next(fresh) + " %}", + ] + if depth < 2: + block = rng.choice(["body", "side"]) + pieces.append(lambda: "{% block " + block + " %}" + generated(rng, fresh, depth + 1) + "{% endblock %}") + return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) + + +def scopes(root: Path) -> tuple[dict[str, list[Path]], dict[Path, Any]]: + """The template files of each name under ``root``, and what each file reads as.""" + + found = {path: tuple(templates._names(path.relative_to(root).parts)) for path in discover_files(root)} + libraries = templates._libraries(found) + paths = sorted(path for path, names in found.items() if names) + files: dict[str, list[Path]] = {} + for path in paths: + for name in found[path]: + files.setdefault(name, []).append(path) + read = {path: templates._read_scopes(text, SourceId(str(path)), libraries) for path in paths if (text := templates._text(path))} + return files, read + + +def joined(found: Iterable[tuple[Any, ...]]) -> dict[tuple[Any, ...], tuple[frozenset[str], frozenset[str]]]: + """The names reaching the value and the argument of each filter application, joined.""" + + names: dict[tuple[Any, ...], tuple[frozenset[str], frozenset[str]]] = {} + for symbol, span, where, value, argument in found: + values, arguments = names.get((symbol, span, where), (frozenset(), frozenset())) + names[symbol, span, where] = values | value, arguments | argument + return names + + +def test_the_filters_are_a_fixpoint_whatever_order_the_templates_are_computed_in(tmp_path: Path) -> None: + """On generated projects of four templates extending and including one another, cycles + among them: what each template, with the blocks overridden, reaches is what it applies + with what its references reach mapped through them, exactly outside cycles and joined + by filter application in them, and the lists are the same, order included, whatever + order the templates are inserted in.""" + + rng = random.Random(284) + for index in range(500): + fresh = (f"n{number}" for number in itertools.count()) + texts = {} + for name in NAMES: + extends = "{% extends '" + rng.choice(NAMES) + "' %}" if rng.random() < 0.5 else "" + texts[f"app/templates/{name}"] = extends + generated(rng, fresh) + "\n" + files, read = scopes(write(tmp_path / str(index), texts)) + renders = templates._renders(files, read) + settled = templates._settled(files, read) + cyclic = { + pair + for component in templates._components(renders) + for pair in component + if len(component) > 1 or any(below == pair for below, _ in renders[pair].references) + } + + for pair, render in renders.items(): + reached = [fact for below, reference in render.references for fact in templates._mapped(settled[below], reference)] + if pair in cyclic: + assert len(settled[pair]) == len(joined(settled[pair])), texts + assert joined(settled[pair]) == joined([*render.applied, *reached]), texts + else: + assert set(settled[pair]) == {*render.applied, *reached}, texts + for shuffle in range(3): + names = list(files) + random.Random(shuffle).shuffle(names) + assert templates._settled({name: files[name] for name in names}, read) == settled, texts diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index e30b498..eaf0f77 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -1,11 +1,13 @@ -"""Acceptance tests for issue #279: the filters a template's render reaches are collected -once for each template and set of blocks the templates extending it override. - -They are the same as before, in the same order: which filters, where they stand, the -template applying each, and the names of the render reaching its value and argument. The +"""Acceptance tests for issues #279 and #284: the filters a template's render reaches are +collected for each template and set of blocks the templates extending it override, in +time linear in the references, and a cycle of includes is not cut (#284). + +Where no cycle is reached, they are the same as before, in the same order: which filters, +where they stand, the template applying each, and the names of the render reaching its +value and argument. Through a cycle, a template reached again contributes what it +reaches, as Django renders it when the project's conditions let the cycle turn. The results are kept for one analysis only, so that projects analysed one after the other do -not share them. A result cut short by a cycle of includes is not reused, nor one that -went through a template already on the way, where the walk would be cut. +not share them. """ from __future__ import annotations @@ -13,6 +15,7 @@ import itertools import json import random +import re import subprocess import sys from collections.abc import Callable, Iterator @@ -161,8 +164,9 @@ def test_includes_doubled_at_every_level_are_walked_in_linear_time(tmp_path: Pat # A child extending a parent that includes it back, inside the block the child overrides. -# The walk of the parent includes the child, whose extends leads back to the parent: a -# cycle, cut there. Whichever of the two is walked first, each gets the filters of main. +# Django renders the parent through the child with that block overridden, which renders +# the child's block instead of the include: the parent's striptags receives other. +# Whichever of the two is walked first, each gets the filters Django applies. def cycle_through_an_overridden_block(child: str, parent: str) -> dict[str, str]: return { child: "{% extends '" + parent + "' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", @@ -186,23 +190,42 @@ def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_wal parent: [ (f"{FILTERS}.striptags", parent, 1, 6, parent, ("v",), ()), (f"{FILTERS}.upper", child, 2, 22, child, ("other",), ()), + (f"{FILTERS}.striptags", parent, 1, 6, parent, ("other",), ()), ], } -def path_by_path( +# The walk path by path lets each cycle turn 2, then 3 times, within this many steps. +REPEATS = (2, 3) +BUDGET = 20_000 +# Projects the walk does not settle within its budget: 10 with this seed. +UNVERIFIED = 15 + + +class Unverified(Exception): + """The walk path by path exceeded its budget: it verifies nothing of the project.""" + + +def unrolled( name: str, files: dict[str, list[Path]], read: dict[Path, Any], - visiting: frozenset[str], + repeats: int, + budget: list[int], + visiting: tuple[tuple[str, frozenset[str]], ...] = (), overridden: frozenset[str] = frozenset(), ) -> Iterator[tuple[Any, ...]]: - """The walk of main, every path through includes and extends followed, a cycle cut at - a template already on the way: the reference the walk of the engine must equal.""" + """Every path through includes and extends, where each template, with the blocks + overridden, may come back ``repeats`` times on the way: what Django renders when the + project's conditions let each cycle turn that many times.""" paths = files.get(name, []) - if len(paths) != 1 or paths[0] not in read or name in visiting: + here = (name, overridden) + if len(paths) != 1 or paths[0] not in read or visiting.count(here) >= repeats: return + budget[0] -= 1 + if budget[0] < 0: + raise Unverified template = read[paths[0]] for application in template.applications: if template.rendered(application.blocks, overridden): @@ -211,17 +234,18 @@ def path_by_path( if reference.name is None: continue if reference.extends: - found = path_by_path(reference.name, files, read, visiting | {name}, overridden | template.replaced) + found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here), overridden | template.replaced) elif template.rendered(reference.blocks, overridden): - found = path_by_path(reference.name, files, read, visiting | {name}) + found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here)) else: continue for symbol, span, where, value, argument in found: yield symbol, span, where, reference.maps(value), reference.maps(argument) -def reference_filters(root: Path) -> dict[str, list[TemplateFilter]]: - """The filters of each template, as project_templates gives them, from the walk of main.""" +def unrolled_filters(root: Path, repeats: int) -> dict[str, list[TemplateFilter]]: + """The filters of each template, as project_templates gives them, from the walk path by + path letting each cycle turn ``repeats`` times.""" found = {path: tuple(templates._names(path.relative_to(root).parts)) for path in discover_files(root)} libraries = templates._libraries(found) @@ -235,13 +259,31 @@ def reference_filters(root: Path) -> dict[str, list[TemplateFilter]]: name: list( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in path_by_path(name, files, read, frozenset()) + for symbol, span, where, value, argument in unrolled(name, files, read, repeats, [BUDGET]) ) ) for name in files } +def reaches_a_cycle(name: str, texts: dict[str, str]) -> bool: + """Whether a template ``name`` includes or extends, or ``name`` itself, reaches itself.""" + + edges = {template: set(re.findall(r"\{% (?:include|extends) '([^']+)'", text)) for template, text in texts.items()} + + def reached(start: str) -> set[str]: + found: set[str] = set() + pending = list(edges.get(start, ())) + while pending: + current = pending.pop() + if current not in found: + found.add(current) + pending.extend(edges.get(current, ())) + return found + + return any(template in reached(template) for template in reached(name) | {name}) + + NAMES = [f"t{index}.html" for index in range(4)] @@ -262,11 +304,28 @@ def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) -def test_the_walk_gives_the_filters_of_main_on_inheritance_overridden_blocks_and_cycles(tmp_path: Path) -> None: +def joined(found: list[TemplateFilter]) -> dict[tuple[Any, ...], tuple[set[str], set[str]]]: + """The names reaching the value and the argument of each filter application, joined + over the paths that bring them: what the analysis reads of a template's filters.""" + + names: dict[tuple[Any, ...], tuple[set[str], set[str]]] = {} + for found_filter in found: + values, arguments = names.setdefault((found_filter.symbol, found_filter.location, found_filter.template), (set(), set())) + values.update(found_filter.value) + arguments.update(found_filter.argument) + return names + + +def test_the_filters_are_those_of_every_path_with_each_cycle_turned_until_it_settles(tmp_path: Path) -> None: """Generated projects of four templates extending and including one another, cycles - among them, each walked by the engine and by the walk of main, path by path.""" + among them, each walked by the engine and path by path, each cycle turned 2, then 3 + times. Where both give the same names at each filter application, the walk has + settled: the engine must give those names, and the same lists, order included, for + the templates no cycle reaches. A project the walk does not settle within its budget + is not verified, and counted.""" rng = random.Random(279) + unverified = 0 for index in range(1_500): fresh = (f"n{number}" for number in itertools.count()) texts = {} @@ -274,6 +333,18 @@ def test_the_walk_gives_the_filters_of_main_on_inheritance_overridden_blocks_and extends = "{% extends '" + rng.choice(NAMES) + "' %}" if rng.random() < 0.5 else "" texts[name] = extends + generated(rng, fresh) + "\n" root = write(tmp_path / str(index), texts) - filters = project_templates(root).filters + filters = {name: list(found) for name, found in project_templates(root).filters.items()} + try: + fewer, more = (unrolled_filters(root, repeats) for repeats in REPEATS) + except Unverified: + unverified += 1 + continue + if {name: joined(found) for name, found in fewer.items()} != {name: joined(found) for name, found in more.items()}: + unverified += 1 + continue - assert {name: list(found) for name, found in filters.items()} == reference_filters(root), texts + assert {name: joined(found) for name, found in filters.items()} == {name: joined(found) for name, found in more.items()}, texts + for name, found in filters.items(): + if not reaches_a_cycle(name, texts): + assert found == more[name], texts + assert unverified <= UNVERIFIED