From 828d713d9f824b55c45d8066397cce74588e9943 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 1/8] fix(templates): collect the filters a render reaches as a least fixpoint over cycles The walk collecting the filters a render reaches cut a cycle of includes at a template already on the way (#284). Django renders such a cycle as far as the project's conditions let it turn, so the walk missed what the next levels apply: a template including itself with other bindings, a recursive tree, or a parent including the child that extends it, where Django applies the parent's filters to what the include binds. A cut walk was also walked again on every path: 20 levels of doubled includes closed into a cycle took 56 s. What each template reaches, with the blocks the templates extending it override, is now the least fixpoint of what it applies and what its references reach mapped through their bindings. The pairs a render reaches are grouped into strongly connected components, each computed after those its references reach. In a component, every set starts empty and a pair is recomputed when a set it reads grows, until none grows: the sets only grow, in a finite domain, so this ends. The lists are built from the settled sets only. A pair outside any cycle keeps the order of a walk of each path, so where no cycle is reached nothing changes; a pair in a cycle lists what it applies, then the rest in a canonical order. The order does not depend on the order of the computation. The two tests that pinned the cut walk now expect what Django applies, and the comparison with the walk path by path lets each cycle turn until it settles. --- src/coretrace_python/taint/templates.py | 226 +++++++++++++++++------- tests/test_template_filter_cycles.py | 88 +++++++++ tests/test_template_filter_walk.py | 104 ++++++++--- 3 files changed, 329 insertions(+), 89 deletions(-) create mode 100644 tests/test_template_filter_cycles.py diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index c5ec5e0..c5c1dfb 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -1050,14 +1050,13 @@ def project_templates(root: Path) -> ProjectTemplates: reason = "not found" if library is None else "not read" if library.uncertain is None else "found in several files" unread.append(f"{relative}:{_position(text, load.offset)[0]} loads {load.library!r}, {reason}") calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications) - # The filters each template reaches, by template and blocks overridden, for this - # analysis only (#279). - reached: dict[tuple[str, frozenset[str]], _Walk] = {} + # What each template reaches rendered on its own, for this analysis only. + settled = _settled(files, read) filters = { name: tuple( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in _reaching(name, files, read, frozenset(), frozenset(), reached).found + for symbol, span, where, value, argument in settled.get((name, frozenset()), ()) ) ) for name in files @@ -1068,72 +1067,167 @@ def project_templates(root: Path) -> ProjectTemplates: # A filter a template's render reaches: the function, where it is applied, the template # applying it, and the names free in the rendered template reaching its value and argument. _Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]] +# A template, and the blocks the templates extending it override: what rendering the +# template applies depends on both. +_Pair = tuple[str, frozenset[str]] -class _Walk(NamedTuple): - """What a walk of a template found; whether a cycle of includes cut it short; and the - templates it went through.""" - - found: list[_Reached] - cut: bool - walked: frozenset[str] - - -_NOTHING = _Walk([], cut=False, walked=frozenset()) - - -def _reaching( - name: str, - files: Mapping[str, list[Path]], - read: Mapping[Path, _Template], - visiting: frozenset[str], - overridden: frozenset[str], - reached: dict[tuple[str, frozenset[str]], _Walk], -) -> _Walk: - """The filters rendering the template ``name`` applies, in it and in what it includes - or extends, each with the template applying it and the names free in ``name`` whose - values reach its value and its argument; the templates extending ``name`` on the way - define the ``overridden`` blocks. A template already ``visiting`` cuts the walk short. - - Each template is walked once for each set of blocks overridden (#279): ``reached`` - keeps what a walk found, its first occurrences in order, which mapped through a - reference give the first occurrences a walk path by path gives. A walk depends on - ``visiting`` only where a cycle cuts it, so one cut short is not kept, and one that - went through a template now on the way is not reused: walked from there, it would be - cut at that template.""" - - paths = files.get(name, []) - if len(paths) != 1 or paths[0] not in read: - return _NOTHING - if name in visiting: - return _Walk([], cut=True, walked=frozenset()) - kept = reached.get((name, overridden)) - if kept is not None and visiting.isdisjoint(kept.walked): - return kept - template = read[paths[0]] - found: dict[_Reached, None] = {} - cut = False - walked = {name} - for application in template.applications: - if template.rendered(application.blocks, overridden): - found.setdefault((application.symbol, application.span, name, application.value, application.argument)) - for reference in template.references: - if reference.name is None: +class _Render(NamedTuple): + """What rendering a pair applies itself, and the pairs its references render, each + with its reference.""" + + applied: tuple[_Reached, ...] + references: tuple[tuple[_Pair, _Reference], ...] + + +def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, _Render]: + """Every pair the render of a template reaches, each template rendered on its own. A + name several files have, or a file the engine could not read, renders nothing.""" + + def pair(name: str, overridden: frozenset[str]) -> _Pair | None: + paths = files.get(name, []) + return (name, overridden) if len(paths) == 1 and paths[0] in read else None + + renders: dict[_Pair, _Render] = {} + pending = [root for name in files if (root := pair(name, frozenset())) is not None] + while pending: + current = pending.pop() + if current in renders: continue - if reference.extends: - below = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced, reached) - elif template.rendered(reference.blocks, overridden): - below = _reaching(reference.name, files, read, visiting | {name}, frozenset(), reached) - else: + name, overridden = current + template = read[files[name][0]] + applied = tuple( + (a.symbol, a.span, name, a.value, a.argument) for a in template.applications if template.rendered(a.blocks, overridden) + ) + references: list[tuple[_Pair, _Reference]] = [] + for reference in template.references: + if reference.name is None: + continue + if reference.extends: + below = pair(reference.name, overridden | template.replaced) + elif template.rendered(reference.blocks, overridden): + below = pair(reference.name, frozenset()) + else: + continue + if below is not None: + references.append((below, reference)) + pending.append(below) + renders[current] = _Render(applied, tuple(references)) + return renders + + +def _components(renders: Mapping[_Pair, _Render]) -> list[list[_Pair]]: + """The strongly connected components of the renders, each after every component its + references reach: Tarjan's algorithm, without recursion.""" + + index: dict[_Pair, int] = {} + low: dict[_Pair, int] = {} + stack: list[_Pair] = [] + on_stack: set[_Pair] = set() + components: list[list[_Pair]] = [] + for root in renders: + if root in index: continue - cut = cut or below.cut - walked |= below.walked - for symbol, span, where, value, argument in below.found: - found.setdefault((symbol, span, where, reference.maps(value), reference.maps(argument))) - walk = _Walk(list(found), cut, frozenset(walked)) - if not cut: - reached[name, overridden] = walk - return walk + # Each entry is a pair and the position of the next reference to follow. + work = [(root, 0)] + while work: + current, position = work.pop() + if position == 0: + index[current] = low[current] = len(index) + stack.append(current) + on_stack.add(current) + references = renders[current].references + for following in range(position, len(references)): + below = references[following][0] + if below not in index: + work.append((current, following + 1)) + work.append((below, 0)) + break + if below in on_stack: + low[current] = min(low[current], index[below]) + else: + if low[current] == index[current]: + component: list[_Pair] = [] + while not component or component[-1] != current: + component.append(stack.pop()) + on_stack.discard(component[-1]) + components.append(component) + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[current]) + return components + + +def _mapped(found: Iterable[_Reached], reference: _Reference) -> Iterator[_Reached]: + """The filters a referenced template reaches, their names mapped through the reference.""" + + for symbol, span, where, value, argument in found: + yield symbol, span, where, reference.maps(value), reference.maps(argument) + + +def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str]]: + """The order of the filters a pair in a cycle reaches beyond what it applies.""" + + symbol, span, where, value, argument = found + return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument) + + +def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, list[_Reached]]: + """The filters the render of each pair reaches, in order (#284). + + What a pair reaches is the least set holding what it applies, and what each of its + references reaches mapped through the reference's bindings. Those sets are finite: + the filters a template applies, each with sets of the names free in the project's + templates. They are computed component by component, each after those its + references reach, which are then final. In a component, every set starts empty and + is recomputed from its references' sets, which only grow, so it only grows too; a + pair is recomputed when a set it reads grew, until none grows. No cycle is cut: a + template including itself with other bindings reaches what Django renders at its + next levels. + + Once the sets are settled, a pair outside any cycle lists what it applies, then what + each reference reaches, in order, first occurrences only, as a walk of each path + would. A pair in a cycle has no such first occurrence, the cycle having no start: it + lists what it applies, then the rest by template, position, function and names. The + order thus depends only on the sets and the templates, never on the order of the + computation.""" + + renders = _renders(files, read) + reached: dict[_Pair, list[_Reached]] = {} + for component in _components(renders): + single = component[0] + if len(component) == 1 and all(below != single for below, _ in renders[single].references): + # Outside any cycle: what its references reach is final. + found = dict.fromkeys(renders[single].applied) + for below, reference in renders[single].references: + found.update(dict.fromkeys(_mapped(reached[below], reference))) + reached[single] = list(found) + continue + members = set(component) + readers: dict[_Pair, list[_Pair]] = {member: [] for member in component} + for member in component: + for below, _ in renders[member].references: + if below in members: + readers[below].append(member) + sets: dict[_Pair, set[_Reached]] = {member: set() for member in component} + pending = deque(component) + queued = set(component) + while pending: + member = pending.popleft() + queued.discard(member) + grown = set(renders[member].applied) + for below, reference in renders[member].references: + grown.update(_mapped(sets[below] if below in members else reached[below], reference)) + if len(grown) > len(sets[member]): + sets[member] = grown + for reader in readers[member]: + if reader not in queued: + pending.append(reader) + queued.add(reader) + for member in component: + applied = renders[member].applied + reached[member] = [*applied, *sorted(sets[member].difference(applied), key=_canonical)] + return reached def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) -> _Template: diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py new file mode 100644 index 0000000..874e905 --- /dev/null +++ b/tests/test_template_filter_cycles.py @@ -0,0 +1,88 @@ +"""Acceptance tests for issue #284: a cycle of includes does not stop the collection of the +filters a template's render reaches. + +A template reached again on the way contributes what it reaches, as Django renders it +when the project's conditions let the cycle turn: the filters of each template, with the +blocks the templates extending it override, are the least fixpoint of what it applies and +what its references reach through their bindings. Each expectation below is what Django +5.1.15 applies when it renders the same templates. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coretrace_python import engine +from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories +from coretrace_python.findings import Severity +from coretrace_python.semantic.symbols import SymbolId + +FILTERS = "python.django.template.defaultfilters" +STRIPTAGS = f"{FILTERS}.striptags" +ADVISORY = Advisory( + "CVE-2099-8401", + "django", + "<4.2.17", + "striptags is quadratic in its input", + Severity.MEDIUM, + entry_points=(AdvisoryEntryPoint(SymbolId(STRIPTAGS), "a template filter"),), +) +LOCK = """version = 1 + +[[package]] +name = "app" +version = "0.1.0" +source = { virtual = "." } +dependencies = [{ name = "django" }] + +[[package]] +name = "django" +version = "4.2.16" +source = { registry = "https://pypi.org/simple" } +""" +# The child extends the parent and overrides its body; the parent includes the child in +# that body, binding v to other. Django renders b.html as striptags of v, then, through +# a.html, b.html again with the body overridden: striptags of other, then upper of other. +CYCLE = { + "app/templates/a.html": "{% extends 'b.html' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", + "app/templates/b.html": "{{ v|striptags }}\n{% block body %}{% include 'a.html' with v=other %}{% endblock %}\n", +} + + +def write(root: Path, files: dict[str, str]) -> Path: + for relative, text in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def exploitable(root: Path, context: str) -> list[tuple[int, str, str]]: + """The exploitable striptags calls a view rendering b.html with ``context`` makes: the + render line, the template holding the filter, and the line of the filter there.""" + + view = ( + "from django.http import HttpRequest\n" + "from django.shortcuts import render\n\n" + "def page(request: HttpRequest):\n" + f" return render(request, 'b.html', {context})\n" + ) + write(root, {"uv.lock": LOCK, "app/__init__.py": "", "app/views.py": view, **CYCLE}) + (root / "advisories.json").write_text(dump_advisories((ADVISORY,)), encoding="utf-8") + findings = engine.analyze_project(root, [engine.BUNDLED_PLUGINS]).findings + return sorted( + (f.span.start_line, f.metadata["through"], f.metadata["sink_line"]) + for f in findings + if f.rule_id == "exploitable-vulnerability" and f.metadata["advisory"] == ADVISORY.id + ) + + +@pytest.mark.parametrize( + "context", + ["{'v': 'fixed', 'other': request.POST['bio']}", "{'v': request.POST['bio'], 'other': 'fixed'}"], + ids=["through-the-cycle", "control"], +) +def test_input_the_parent_passes_its_child_reaches_the_parent_filter_django_applies(tmp_path: Path, context: str) -> None: + assert exploitable(tmp_path, context) == [(5, "b.html", "1")] diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index e30b498..5a31ae3 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -1,11 +1,13 @@ -"""Acceptance tests for issue #279: the filters a template's render reaches are collected -once for each template and set of blocks the templates extending it override. - -They are the same as before, in the same order: which filters, where they stand, the -template applying each, and the names of the render reaching its value and argument. The +"""Acceptance tests for issues #279 and #284: the filters a template's render reaches are +collected for each template and set of blocks the templates extending it override, in +time linear in the references, and a cycle of includes is not cut (#284). + +Where no cycle is reached, they are the same as before, in the same order: which filters, +where they stand, the template applying each, and the names of the render reaching its +value and argument. Through a cycle, a template reached again contributes what it +reaches, as Django renders it when the project's conditions let the cycle turn. The results are kept for one analysis only, so that projects analysed one after the other do -not share them. A result cut short by a cycle of includes is not reused, nor one that -went through a template already on the way, where the walk would be cut. +not share them. """ from __future__ import annotations @@ -13,6 +15,7 @@ import itertools import json import random +import re import subprocess import sys from collections.abc import Callable, Iterator @@ -161,8 +164,9 @@ def test_includes_doubled_at_every_level_are_walked_in_linear_time(tmp_path: Pat # A child extending a parent that includes it back, inside the block the child overrides. -# The walk of the parent includes the child, whose extends leads back to the parent: a -# cycle, cut there. Whichever of the two is walked first, each gets the filters of main. +# Django renders the parent through the child with that block overridden, which renders +# the child's block instead of the include: the parent's striptags receives other. +# Whichever of the two is walked first, each gets the filters Django applies. def cycle_through_an_overridden_block(child: str, parent: str) -> dict[str, str]: return { child: "{% extends '" + parent + "' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", @@ -186,23 +190,42 @@ def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_wal parent: [ (f"{FILTERS}.striptags", parent, 1, 6, parent, ("v",), ()), (f"{FILTERS}.upper", child, 2, 22, child, ("other",), ()), + (f"{FILTERS}.striptags", parent, 1, 6, parent, ("other",), ()), ], } -def path_by_path( +# The walk path by path lets each cycle turn 2, then 3 times, within this many steps. +REPEATS = (2, 3) +BUDGET = 20_000 +# Projects the walk does not settle within its budget: 10 with this seed. +UNVERIFIED = 15 + + +class Unverified(Exception): + """The walk path by path exceeded its budget: it verifies nothing of the project.""" + + +def unrolled( name: str, files: dict[str, list[Path]], read: dict[Path, Any], - visiting: frozenset[str], + repeats: int, + budget: list[int], + visiting: tuple[tuple[str, frozenset[str]], ...] = (), overridden: frozenset[str] = frozenset(), ) -> Iterator[tuple[Any, ...]]: - """The walk of main, every path through includes and extends followed, a cycle cut at - a template already on the way: the reference the walk of the engine must equal.""" + """Every path through includes and extends, where each template, with the blocks + overridden, may come back ``repeats`` times on the way: what Django renders when the + project's conditions let each cycle turn that many times.""" paths = files.get(name, []) - if len(paths) != 1 or paths[0] not in read or name in visiting: + here = (name, overridden) + if len(paths) != 1 or paths[0] not in read or visiting.count(here) >= repeats: return + budget[0] -= 1 + if budget[0] < 0: + raise Unverified template = read[paths[0]] for application in template.applications: if template.rendered(application.blocks, overridden): @@ -211,17 +234,18 @@ def path_by_path( if reference.name is None: continue if reference.extends: - found = path_by_path(reference.name, files, read, visiting | {name}, overridden | template.replaced) + found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here), overridden | template.replaced) elif template.rendered(reference.blocks, overridden): - found = path_by_path(reference.name, files, read, visiting | {name}) + found = unrolled(reference.name, files, read, repeats, budget, (*visiting, here)) else: continue for symbol, span, where, value, argument in found: yield symbol, span, where, reference.maps(value), reference.maps(argument) -def reference_filters(root: Path) -> dict[str, list[TemplateFilter]]: - """The filters of each template, as project_templates gives them, from the walk of main.""" +def unrolled_filters(root: Path, repeats: int) -> dict[str, list[TemplateFilter]]: + """The filters of each template, as project_templates gives them, from the walk path by + path letting each cycle turn ``repeats`` times.""" found = {path: tuple(templates._names(path.relative_to(root).parts)) for path in discover_files(root)} libraries = templates._libraries(found) @@ -235,13 +259,31 @@ def reference_filters(root: Path) -> dict[str, list[TemplateFilter]]: name: list( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in path_by_path(name, files, read, frozenset()) + for symbol, span, where, value, argument in unrolled(name, files, read, repeats, [BUDGET]) ) ) for name in files } +def reaches_a_cycle(name: str, texts: dict[str, str]) -> bool: + """Whether a template ``name`` includes or extends, or ``name`` itself, reaches itself.""" + + edges = {template: set(re.findall(r"\{% (?:include|extends) '([^']+)'", text)) for template, text in texts.items()} + + def reached(start: str) -> set[str]: + found: set[str] = set() + pending = list(edges.get(start, ())) + while pending: + current = pending.pop() + if current not in found: + found.add(current) + pending.extend(edges.get(current, ())) + return found + + return any(template in reached(template) for template in reached(name) | {name}) + + NAMES = [f"t{index}.html" for index in range(4)] @@ -262,11 +304,15 @@ def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) -def test_the_walk_gives_the_filters_of_main_on_inheritance_overridden_blocks_and_cycles(tmp_path: Path) -> None: +def test_the_filters_are_those_of_every_path_with_each_cycle_turned_until_it_settles(tmp_path: Path) -> None: """Generated projects of four templates extending and including one another, cycles - among them, each walked by the engine and by the walk of main, path by path.""" + among them, each walked by the engine and path by path, each cycle turned 2, then 3 + times. Where both give the same sets, the walk has settled: the engine must give those + sets, and the same lists, order included, for the templates no cycle reaches. A project + the walk does not settle within its budget is not verified, and counted.""" rng = random.Random(279) + unverified = 0 for index in range(1_500): fresh = (f"n{number}" for number in itertools.count()) texts = {} @@ -274,6 +320,18 @@ def test_the_walk_gives_the_filters_of_main_on_inheritance_overridden_blocks_and extends = "{% extends '" + rng.choice(NAMES) + "' %}" if rng.random() < 0.5 else "" texts[name] = extends + generated(rng, fresh) + "\n" root = write(tmp_path / str(index), texts) - filters = project_templates(root).filters + filters = {name: list(found) for name, found in project_templates(root).filters.items()} + try: + fewer, more = (unrolled_filters(root, repeats) for repeats in REPEATS) + except Unverified: + unverified += 1 + continue + if {name: set(found) for name, found in fewer.items()} != {name: set(found) for name, found in more.items()}: + unverified += 1 + continue - assert {name: list(found) for name, found in filters.items()} == reference_filters(root), texts + assert {name: set(found) for name, found in filters.items()} == {name: set(found) for name, found in more.items()}, texts + for name, found in filters.items(): + if not reaches_a_cycle(name, texts): + assert found == more[name], texts + assert unverified <= UNVERIFIED From 320fece25fb8940fdd94a999bcfbfbc88c174815 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 2/8] test(templates): cover templates including themselves with other bindings, and trees Django applies the filters of each level of the recursion to what the include binds: v, then n0 for a template including itself with v=n0, and the names of node.children for a tree including itself for each child. --- tests/test_template_filter_cycles.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py index 874e905..ae9dd67 100644 --- a/tests/test_template_filter_cycles.py +++ b/tests/test_template_filter_cycles.py @@ -18,6 +18,7 @@ from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories from coretrace_python.findings import Severity from coretrace_python.semantic.symbols import SymbolId +from coretrace_python.taint.templates import project_templates FILTERS = "python.django.template.defaultfilters" STRIPTAGS = f"{FILTERS}.striptags" @@ -86,3 +87,49 @@ def exploitable(root: Path, context: str) -> list[tuple[int, str, str]]: ) def test_input_the_parent_passes_its_child_reaches_the_parent_filter_django_applies(tmp_path: Path, context: str) -> None: assert exploitable(tmp_path, context) == [(5, "b.html", "1")] + + +def described(root: Path) -> dict[str, list[tuple[str, str, int, int, str, tuple[str, ...], tuple[str, ...]]]]: + """Each template's filters, in order: the function, the file and the line and column it + is applied at, the template applying it, and the names reaching its value and argument.""" + + return { + name: [ + ( + str(f.symbol), + Path(str(f.location.source_id)).name, + f.location.start_line, + f.location.start_column, + f.template, + f.value, + f.argument, + ) + for f in found + ] + for name, found in project_templates(root).filters.items() + } + + +def test_a_template_including_itself_with_other_bindings_reaches_them(tmp_path: Path) -> None: + # Django applies upper to v, then, one level down, to n0; the condition ends there. + root = write(tmp_path, {"app/templates/t.html": "{{ v|upper }}{% if go %}{% include 't.html' with v=n0 go='' %}{% endif %}\n"}) + + assert described(root) == { + "t.html": [ + (f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("v",), ()), + (f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("n0",), ()), + ] + } + + +def test_a_recursive_tree_reaches_the_children_at_every_level(tmp_path: Path) -> None: + # Django applies upper to node.name, then to the name of each node of node.children. + node = "{{ node.name|upper }}{% for child in node.children %}{% include 'node.html' with node=child %}{% endfor %}\n" + root = write(tmp_path, {"app/templates/node.html": node}) + + assert described(root) == { + "node.html": [ + (f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.name",), ()), + (f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.children",), ()), + ] + } From 921b9908a46f56cfe4c38d065b93ac2f61f5fd42 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 3/8] test(templates): bound the time of collecting filters through cycles of includes A cycle through 60 levels of doubled includes, and 40 templates each including every other one, are collected in a subprocess bounded at 60 s. Cut at every template on the way, the former walk took a minute for 20 levels. --- tests/test_template_filter_cycles.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py index ae9dd67..1d25a83 100644 --- a/tests/test_template_filter_cycles.py +++ b/tests/test_template_filter_cycles.py @@ -10,6 +10,9 @@ from __future__ import annotations +import json +import subprocess +import sys from pathlib import Path import pytest @@ -20,6 +23,7 @@ from coretrace_python.semantic.symbols import SymbolId from coretrace_python.taint.templates import project_templates +TIMEOUT = 60 FILTERS = "python.django.template.defaultfilters" STRIPTAGS = f"{FILTERS}.striptags" ADVISORY = Advisory( @@ -133,3 +137,46 @@ def test_a_recursive_tree_reaches_the_children_at_every_level(tmp_path: Path) -> (f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.children",), ()), ] } + + +COUNT_FILTERS = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python.taint.templates import project_templates\n" + "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n" +) + + +def counted(root: Path) -> dict[str, int]: + """Each template's number of filters, counted in a subprocess bounded in time.""" + + try: + completed = subprocess.run( + [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT + ) + except subprocess.TimeoutExpired: + pytest.fail(f"collecting the filters did not end within {TIMEOUT} s") + assert completed.returncode == 0, completed.stderr + counts: dict[str, int] = json.loads(completed.stdout) + return counts + + +def test_a_cycle_through_includes_doubled_at_every_level_is_collected_in_linear_time(tmp_path: Path) -> None: + # Cut at each template on the way, every walk inside the cycle is cut, and each is + # walked again on every path: 20 levels took a minute. + templates = {"app/templates/level0.html": "{{ name|upper }}{% if more %}{% include 'level60.html' %}{% endif %}"} + for level in range(1, 61): + templates[f"app/templates/level{level}.html"] = f"{{% include 'level{level - 1}.html' %}}" * 2 + + assert counted(write(tmp_path, templates)) == {f"level{level}.html": 1 for level in range(61)} + + +def test_templates_each_including_every_other_are_collected_in_polynomial_time(tmp_path: Path) -> None: + # Each template reaches the filter of every other one, through any of their orders. + names = [f"t{index}.html" for index in range(40)] + templates = { + f"app/templates/{name}": "{{ v|upper }}" + "".join(f"{{% if go %}}{{% include '{other}' %}}{{% endif %}}" for other in names if other != name) + for name in names + } + + assert counted(write(tmp_path, templates)) == {name: 40 for name in names} From 8710b520d086c6ab3745504518993c70406c240d Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 4/8] test(templates): check the filters are a fixpoint whatever the order of computation On 500 generated projects of templates extending and including one another, with cycles among them, what each pair reaches is what it applies with what its references reach mapped through them, and the lists are the same, order included, whatever order the templates are inserted in. --- tests/test_template_filter_cycles.py | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py index 1d25a83..3597603 100644 --- a/tests/test_template_filter_cycles.py +++ b/tests/test_template_filter_cycles.py @@ -10,17 +10,24 @@ from __future__ import annotations +import itertools import json +import random import subprocess import sys +from collections.abc import Callable, Iterator from pathlib import Path +from typing import Any import pytest from coretrace_python import engine from coretrace_python.dependency import Advisory, AdvisoryEntryPoint, dump_advisories from coretrace_python.findings import Severity +from coretrace_python.interprocedural import discover_files from coretrace_python.semantic.symbols import SymbolId +from coretrace_python.source import SourceId +from coretrace_python.taint import templates from coretrace_python.taint.templates import project_templates TIMEOUT = 60 @@ -180,3 +187,62 @@ def test_templates_each_including_every_other_are_collected_in_polynomial_time(t } assert counted(write(tmp_path, templates)) == {name: 40 for name in names} + + +NAMES = [f"t{index}.html" for index in range(4)] + + +def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: + """Markup applying filters, including other templates, inside blocks of two names and + outside them, each include binding the variable to a name of its own.""" + + pieces: list[Callable[[], str]] = [ + lambda: "{{ v|upper }}", + lambda: "{{ v|striptags }}", + lambda: "{% include '" + rng.choice(NAMES) + "' %}", + lambda: "{% include '" + rng.choice(NAMES) + "' with v=" + next(fresh) + " %}", + ] + if depth < 2: + block = rng.choice(["body", "side"]) + pieces.append(lambda: "{% block " + block + " %}" + generated(rng, fresh, depth + 1) + "{% endblock %}") + return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) + + +def scopes(root: Path) -> tuple[dict[str, list[Path]], dict[Path, Any]]: + """The template files of each name under ``root``, and what each file reads as.""" + + found = {path: tuple(templates._names(path.relative_to(root).parts)) for path in discover_files(root)} + libraries = templates._libraries(found) + paths = sorted(path for path, names in found.items() if names) + files: dict[str, list[Path]] = {} + for path in paths: + for name in found[path]: + files.setdefault(name, []).append(path) + read = {path: templates._read_scopes(text, SourceId(str(path)), libraries) for path in paths if (text := templates._text(path))} + return files, read + + +def test_the_filters_are_a_fixpoint_whatever_order_the_templates_are_computed_in(tmp_path: Path) -> None: + """On generated projects of four templates extending and including one another, cycles + among them: what each template, with the blocks overridden, reaches is what it applies + with what its references reach mapped through them, and the lists are the same, order + included, whatever order the templates are inserted in.""" + + rng = random.Random(284) + for index in range(500): + fresh = (f"n{number}" for number in itertools.count()) + texts = {} + for name in NAMES: + extends = "{% extends '" + rng.choice(NAMES) + "' %}" if rng.random() < 0.5 else "" + texts[f"app/templates/{name}"] = extends + generated(rng, fresh) + "\n" + files, read = scopes(write(tmp_path / str(index), texts)) + renders = templates._renders(files, read) + settled = templates._settled(files, read) + + for pair, render in renders.items(): + reached = (set(templates._mapped(settled[below], reference)) for below, reference in render.references) + assert set(settled[pair]) == set(render.applied).union(*reached), texts + for shuffle in range(3): + names = list(files) + random.Random(shuffle).shuffle(names) + assert templates._settled({name: files[name] for name in names}, read) == settled, texts From d7f4790e2cd94e496e0e84c472801619d0b155dc Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 5/8] docs: state that a cycle of includes passes its bindings at every level --- docs/usage.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 00ab31e..d6870a4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -601,7 +601,10 @@ certain of it: - The filter's value is a context variable, directly or through `{% for %}` and `{% with %}`, the templates the template includes or extends by a constant name, the arguments of the filters before it, or the output of a `{% filter %}` block. A name a - tag binds shadows the context, and `{% include ... only %}` passes none of it. + tag binds shadows the context, and `{% include ... only %}` passes none of it. A cycle + of includes is followed as far as Django may render it: a template reached again on + the way applies its filters to what each include binds at every level, as a template + including itself with other bindings, or a recursive tree, does (#284). - The filter renders: a template extending another renders only its blocks, which replace the blocks of that name in what it extends, unless the block renders `{{ block.super }}`, which keeps the parent's content and the filters it applies. From ad4dd0453708fc3aff807492c3f5d19f1d2bdd69 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 17:47:48 +0100 Subject: [PATCH 6/8] docs(changelog): announce the filters reached through cycles of includes --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d2d792..892da79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +### Templates + +- A cycle of includes no longer stops the collection of the filters a render reaches + (#284). A template reached again on the way contributes what it reaches, its names + mapped through each include, as Django renders it when the project's conditions let + the cycle turn: a template including itself with other bindings, a recursive tree, or + a parent including the child that extends it. A filter reached this way may be + reported as `exploitable-vulnerability` where it was not. Where no cycle is reached, + the filters and their order do not change. + +### Performance + +- Templates in a cycle of includes are no longer walked again on every path (#284): 20 + levels of doubled includes closed into a cycle took 56 s, and 40 templates including + one another more than a minute; both now take less than a second. + ## 0.24.0 (2026-10-23) ### Compatibility and migration From f6ee609a885825b8dabab563ff9b0809c446dc31 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 23:44:19 +0100 Subject: [PATCH 7/8] fix(templates): join the names reaching each filter application in a cycle Each combination of the names reaching a filter's value and argument was its own fact. Bindings permuted around a cycle of includes made one fact of every combination: a 272-byte template including itself twice, rotating 14 variables and swapping two of them, gave 60,762 filters and took 12 s to collect, where main takes a millisecond. In a cycle, the names reaching the value and the argument of each filter application are now joined: one fact per application. The analysis reads those names position by position, joining what each one brings, so the join loses nothing. The names of an application only grow, at most once per name, so a cycle settles in time polynomial in its applications and names. Outside cycles, nothing changes. The tests of recursion and of the tree expect one call per filter, the fixpoint is checked by application in cycles, and the comparison with the walk path by path compares the names joined at each application. --- src/coretrace_python/taint/templates.py | 68 ++++++++++++++++++------- tests/test_template_filter_cycles.py | 46 ++++++++++------- tests/test_template_filter_walk.py | 23 +++++++-- 3 files changed, 97 insertions(+), 40 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index c5c1dfb..1913ea4 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -1172,24 +1172,49 @@ def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument) +# A filter application: the function, where it is applied, and the template applying it. +_Site = tuple[SymbolId, SourceSpan, str] + + +def _joined(found: Iterable[_Reached], into: dict[_Site, tuple[frozenset[str], frozenset[str]]]) -> None: + """Join into ``into`` the names reaching the value and the argument of each filter + application ``found`` holds.""" + + for symbol, span, where, value, argument in found: + values, arguments = into.get((symbol, span, where), (frozenset(), frozenset())) + into[symbol, span, where] = values | value, arguments | argument + + +def _facts(names: Mapping[_Site, tuple[frozenset[str], frozenset[str]]], sites: Iterable[_Site]) -> list[_Reached]: + """The filters at ``sites``, each with the names joined at its application.""" + + return [(symbol, span, where, *names[symbol, span, where]) for symbol, span, where in sites] + + def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[_Pair, list[_Reached]]: """The filters the render of each pair reaches, in order (#284). What a pair reaches is the least set holding what it applies, and what each of its - references reaches mapped through the reference's bindings. Those sets are finite: - the filters a template applies, each with sets of the names free in the project's - templates. They are computed component by component, each after those its - references reach, which are then final. In a component, every set starts empty and - is recomputed from its references' sets, which only grow, so it only grows too; a - pair is recomputed when a set it reads grew, until none grows. No cycle is cut: a - template including itself with other bindings reaches what Django renders at its - next levels. - - Once the sets are settled, a pair outside any cycle lists what it applies, then what + references reaches mapped through the reference's bindings. The sets are computed + component by component, each after those its references reach, which are then + final. No cycle is cut: a template including itself with other bindings reaches what + Django renders at its next levels. + + In a cycle, the names reaching the value and the argument of each filter application + are joined: one fact per application, whatever the paths that bring each name. The + analysis reads those names position by position, joining what each brings, so the + join loses nothing, and it keeps the facts few: bindings permuted around a cycle + would otherwise make one fact of every combination of names. In a component, every + pair starts with no names and is recomputed from its references' facts; names only + join, and the applications and names of a project are finitely many, so the names of + an application grow at most once for each name. A pair is recomputed when a fact it + reads grew, until none grows. + + Once the facts are settled, a pair outside any cycle lists what it applies, then what each reference reaches, in order, first occurrences only, as a walk of each path would. A pair in a cycle has no such first occurrence, the cycle having no start: it lists what it applies, then the rest by template, position, function and names. The - order thus depends only on the sets and the templates, never on the order of the + order thus depends only on the facts and the templates, never on the order of the computation.""" renders = _renders(files, read) @@ -1209,24 +1234,31 @@ def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> for below, _ in renders[member].references: if below in members: readers[below].append(member) - sets: dict[_Pair, set[_Reached]] = {member: set() for member in component} + names: dict[_Pair, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component} + sizes = dict.fromkeys(component, 0) pending = deque(component) queued = set(component) while pending: member = pending.popleft() queued.discard(member) - grown = set(renders[member].applied) + grown: dict[_Site, tuple[frozenset[str], frozenset[str]]] = {} + _joined(renders[member].applied, grown) for below, reference in renders[member].references: - grown.update(_mapped(sets[below] if below in members else reached[below], reference)) - if len(grown) > len(sets[member]): - sets[member] = grown + below_found = _facts(names[below], names[below]) if below in members else reached[below] + _joined(_mapped(below_found, reference), grown) + # The names only join, so the count of applications and names only grows. + size = sum(1 + len(values) + len(arguments) for values, arguments in grown.values()) + if size > sizes[member]: + names[member], sizes[member] = grown, size for reader in readers[member]: if reader not in queued: pending.append(reader) queued.add(reader) for member in component: - applied = renders[member].applied - reached[member] = [*applied, *sorted(sets[member].difference(applied), key=_canonical)] + applied = _facts(names[member], dict.fromkeys((s, span, w) for s, span, w, _, _ in renders[member].applied)) + sites = {(s, span, w) for s, span, w, _, _ in applied} + rest = _facts(names[member], (site for site in names[member] if site not in sites)) + reached[member] = [*applied, *sorted(rest, key=_canonical)] return reached diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py index 3597603..93f5ff4 100644 --- a/tests/test_template_filter_cycles.py +++ b/tests/test_template_filter_cycles.py @@ -15,7 +15,7 @@ import random import subprocess import sys -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from pathlib import Path from typing import Any @@ -123,14 +123,10 @@ def described(root: Path) -> dict[str, list[tuple[str, str, int, int, str, tuple def test_a_template_including_itself_with_other_bindings_reaches_them(tmp_path: Path) -> None: # Django applies upper to v, then, one level down, to n0; the condition ends there. + # In a cycle, the names reaching a filter application are joined into one call. root = write(tmp_path, {"app/templates/t.html": "{{ v|upper }}{% if go %}{% include 't.html' with v=n0 go='' %}{% endif %}\n"}) - assert described(root) == { - "t.html": [ - (f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("v",), ()), - (f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("n0",), ()), - ] - } + assert described(root) == {"t.html": [(f"{FILTERS}.upper", "t.html", 1, 6, "t.html", ("n0", "v"), ())]} def test_a_recursive_tree_reaches_the_children_at_every_level(tmp_path: Path) -> None: @@ -138,12 +134,7 @@ def test_a_recursive_tree_reaches_the_children_at_every_level(tmp_path: Path) -> node = "{{ node.name|upper }}{% for child in node.children %}{% include 'node.html' with node=child %}{% endfor %}\n" root = write(tmp_path, {"app/templates/node.html": node}) - assert described(root) == { - "node.html": [ - (f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.name",), ()), - (f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.children",), ()), - ] - } + assert described(root) == {"node.html": [(f"{FILTERS}.upper", "node.html", 1, 14, "node.html", ("node.children", "node.name"), ())]} COUNT_FILTERS = ( @@ -222,11 +213,22 @@ def scopes(root: Path) -> tuple[dict[str, list[Path]], dict[Path, Any]]: return files, read +def joined(found: Iterable[tuple[Any, ...]]) -> dict[tuple[Any, ...], tuple[frozenset[str], frozenset[str]]]: + """The names reaching the value and the argument of each filter application, joined.""" + + names: dict[tuple[Any, ...], tuple[frozenset[str], frozenset[str]]] = {} + for symbol, span, where, value, argument in found: + values, arguments = names.get((symbol, span, where), (frozenset(), frozenset())) + names[symbol, span, where] = values | value, arguments | argument + return names + + def test_the_filters_are_a_fixpoint_whatever_order_the_templates_are_computed_in(tmp_path: Path) -> None: """On generated projects of four templates extending and including one another, cycles among them: what each template, with the blocks overridden, reaches is what it applies - with what its references reach mapped through them, and the lists are the same, order - included, whatever order the templates are inserted in.""" + with what its references reach mapped through them, exactly outside cycles and joined + by filter application in them, and the lists are the same, order included, whatever + order the templates are inserted in.""" rng = random.Random(284) for index in range(500): @@ -238,10 +240,20 @@ def test_the_filters_are_a_fixpoint_whatever_order_the_templates_are_computed_in files, read = scopes(write(tmp_path / str(index), texts)) renders = templates._renders(files, read) settled = templates._settled(files, read) + cyclic = { + pair + for component in templates._components(renders) + for pair in component + if len(component) > 1 or any(below == pair for below, _ in renders[pair].references) + } for pair, render in renders.items(): - reached = (set(templates._mapped(settled[below], reference)) for below, reference in render.references) - assert set(settled[pair]) == set(render.applied).union(*reached), texts + reached = [fact for below, reference in render.references for fact in templates._mapped(settled[below], reference)] + if pair in cyclic: + assert len(settled[pair]) == len(joined(settled[pair])), texts + assert joined(settled[pair]) == joined([*render.applied, *reached]), texts + else: + assert set(settled[pair]) == {*render.applied, *reached}, texts for shuffle in range(3): names = list(files) random.Random(shuffle).shuffle(names) diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index 5a31ae3..eaf0f77 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -304,12 +304,25 @@ def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) +def joined(found: list[TemplateFilter]) -> dict[tuple[Any, ...], tuple[set[str], set[str]]]: + """The names reaching the value and the argument of each filter application, joined + over the paths that bring them: what the analysis reads of a template's filters.""" + + names: dict[tuple[Any, ...], tuple[set[str], set[str]]] = {} + for found_filter in found: + values, arguments = names.setdefault((found_filter.symbol, found_filter.location, found_filter.template), (set(), set())) + values.update(found_filter.value) + arguments.update(found_filter.argument) + return names + + def test_the_filters_are_those_of_every_path_with_each_cycle_turned_until_it_settles(tmp_path: Path) -> None: """Generated projects of four templates extending and including one another, cycles among them, each walked by the engine and path by path, each cycle turned 2, then 3 - times. Where both give the same sets, the walk has settled: the engine must give those - sets, and the same lists, order included, for the templates no cycle reaches. A project - the walk does not settle within its budget is not verified, and counted.""" + times. Where both give the same names at each filter application, the walk has + settled: the engine must give those names, and the same lists, order included, for + the templates no cycle reaches. A project the walk does not settle within its budget + is not verified, and counted.""" rng = random.Random(279) unverified = 0 @@ -326,11 +339,11 @@ def test_the_filters_are_those_of_every_path_with_each_cycle_turned_until_it_set except Unverified: unverified += 1 continue - if {name: set(found) for name, found in fewer.items()} != {name: set(found) for name, found in more.items()}: + if {name: joined(found) for name, found in fewer.items()} != {name: joined(found) for name, found in more.items()}: unverified += 1 continue - assert {name: set(found) for name, found in filters.items()} == {name: set(found) for name, found in more.items()}, texts + assert {name: joined(found) for name, found in filters.items()} == {name: joined(found) for name, found in more.items()}, texts for name, found in filters.items(): if not reaches_a_cycle(name, texts): assert found == more[name], texts From 2b987058098ad3240ae2a349f973c46dd11c32cf Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 25 Oct 2026 23:44:19 +0100 Subject: [PATCH 8/8] test(templates): bound the time of bindings permuted around a cycle of includes A filter chain over seven variables, in a template including itself twice, rotating 24 variables and swapping two of them, is collected in a subprocess bounded at 60 s, as 7 filters. --- tests/test_template_filter_cycles.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_template_filter_cycles.py b/tests/test_template_filter_cycles.py index 93f5ff4..85eaad2 100644 --- a/tests/test_template_filter_cycles.py +++ b/tests/test_template_filter_cycles.py @@ -180,6 +180,21 @@ def test_templates_each_including_every_other_are_collected_in_polynomial_time(t assert counted(write(tmp_path, templates)) == {name: 40 for name in names} +def test_bindings_permuted_around_a_cycle_are_joined_at_each_filter(tmp_path: Path) -> None: + # Each combination of the 24 names would be its own fact; joined at each filter + # application, the chain's 7 filters are 7 calls. + rotated = " ".join(f"v{index}=v{(index + 1) % 24}" for index in range(24)) + template = ( + "{{ v0|default:v1|default:v2|default:v3|default:v4|default:v5|default:v6|upper }}\n" + "{% if go %}\n" + "{% include 't.html' with " + rotated + " go='' %}\n" + "{% include 't.html' with v0=v1 v1=v0 go='' %}\n" + "{% endif %}\n" + ) + + assert counted(write(tmp_path, {"app/templates/t.html": template})) == {"t.html": 7} + + NAMES = [f"t{index}.html" for index in range(4)]