From d627ceac0a68d5b4533e9d10c3bdf98706dd07c1 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 20:42:27 +0200 Subject: [PATCH 1/7] test(templates): pin the filters reached through overridden blocks, cycles and successive analyses These tests check, in order, which filters a template's render reaches, where they stand, the template applying each, and the names reaching their value and argument. They cover projects analysed one after the other, a template reached directly and through a child overriding its block, and a template reached inside a cycle of includes and from outside it. The expected results are those of main, which the next commit must keep. --- tests/test_template_filter_walk.py | 121 +++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_template_filter_walk.py diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py new file mode 100644 index 0000000..c9df71e --- /dev/null +++ b/tests/test_template_filter_walk.py @@ -0,0 +1,121 @@ +"""Acceptance tests for issue #279: the filters a template's render reaches are collected +once for each template and set of blocks the templates extending it override. + +They are the same as before, in the same order: which filters, where they stand, the +template applying each, and the names of the render reaching its value and argument. The +results are kept for one analysis only, so that projects analysed one after the other do +not share them, and a result cut short by a cycle of includes is not reused. +""" + +from __future__ import annotations + +from pathlib import Path + +from coretrace_python.taint.templates import project_templates + +FILTERS = "python.django.template.defaultfilters" + + +def write(root: Path, templates: dict[str, str]) -> Path: + for name, text in templates.items(): + path = root / "app" / "templates" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def described(root: Path) -> dict[str, list[tuple[str, str, int, int, str, tuple[str, ...], tuple[str, ...]]]]: + """Each template's filters, in order: the function, the file and the line and column it + is applied at, the template applying it, and the names reaching its value and argument.""" + + return { + name: [ + ( + str(f.symbol), + Path(str(f.location.source_id)).name, + f.location.start_line, + f.location.start_column, + f.template, + f.value, + f.argument, + ) + for f in found + ] + for name, found in project_templates(root).filters.items() + } + + +FIRST = {"page.html": "{{ bio|striptags }}\n{% include 'part.html' with note=bio %}\n", "part.html": "

{{ note|upper }}

\n"} +FIRST_FILTERS = { + "page.html": [ + (f"{FILTERS}.striptags", "page.html", 1, 8, "page.html", ("bio",), ()), + (f"{FILTERS}.upper", "part.html", 1, 12, "part.html", ("bio",), ()), + ], + "part.html": [(f"{FILTERS}.upper", "part.html", 1, 12, "part.html", ("note",), ())], +} +# The same names, other templates. +SECOND = {"page.html": "{% include 'part.html' %}\n{{ title|lower }}\n", "part.html": "{{ other|title }}\n"} +SECOND_FILTERS = { + "page.html": [ + (f"{FILTERS}.lower", "page.html", 2, 10, "page.html", ("title",), ()), + (f"{FILTERS}.title", "part.html", 1, 10, "part.html", ("other",), ()), + ], + "part.html": [(f"{FILTERS}.title", "part.html", 1, 10, "part.html", ("other",), ())], +} + + +def test_projects_analysed_one_after_the_other_each_get_their_own_filters(tmp_path: Path) -> None: + first, second, again = (write(tmp_path / key, files) for key, files in (("first", FIRST), ("second", SECOND), ("again", FIRST))) + + assert described(first) == FIRST_FILTERS + assert described(second) == SECOND_FILTERS + assert described(again) == FIRST_FILTERS + assert described(second) == SECOND_FILTERS + + +def test_a_template_reached_directly_and_through_a_child_overriding_its_block(tmp_path: Path) -> None: + # child.html renders base.html with its own body: base.html's upper is not reached + # through it, though it is through the direct include. + root = write( + tmp_path, + { + "base.html": "{% block body %}{{ a|upper }}{% endblock %}{{ b|lower }}\n", + "child.html": "{% extends 'base.html' %}{% block body %}{{ c|title }}{% endblock %}\n", + "page.html": "{% include 'base.html' %}{% include 'child.html' %}\n", + }, + ) + + assert described(root) == { + "base.html": [ + (f"{FILTERS}.upper", "base.html", 1, 22, "base.html", ("a",), ()), + (f"{FILTERS}.lower", "base.html", 1, 49, "base.html", ("b",), ()), + ], + "child.html": [ + (f"{FILTERS}.title", "child.html", 1, 47, "child.html", ("c",), ()), + (f"{FILTERS}.lower", "base.html", 1, 49, "base.html", ("b",), ()), + ], + "page.html": [ + (f"{FILTERS}.upper", "base.html", 1, 22, "base.html", ("a",), ()), + (f"{FILTERS}.lower", "base.html", 1, 49, "base.html", ("b",), ()), + (f"{FILTERS}.title", "child.html", 1, 47, "child.html", ("c",), ()), + ], + } + + +def test_a_template_reached_inside_a_cycle_and_from_outside_it(tmp_path: Path) -> None: + # a.html is walked first: inside it, b.html's walk stops at a.html and misses its + # upper. c.html reaches b.html from outside the cycle, and gets a.html's upper. + root = write( + tmp_path, + { + "a.html": "{{ x|upper }}{% include 'b.html' %}\n", + "b.html": "{{ y|lower }}{% include 'a.html' %}\n", + "c.html": "{% include 'b.html' %}\n", + }, + ) + + assert described(root) == { + "a.html": [(f"{FILTERS}.upper", "a.html", 1, 6, "a.html", ("x",), ()), (f"{FILTERS}.lower", "b.html", 1, 6, "b.html", ("y",), ())], + "b.html": [(f"{FILTERS}.lower", "b.html", 1, 6, "b.html", ("y",), ()), (f"{FILTERS}.upper", "a.html", 1, 6, "a.html", ("x",), ())], + "c.html": [(f"{FILTERS}.lower", "b.html", 1, 6, "b.html", ("y",), ()), (f"{FILTERS}.upper", "a.html", 1, 6, "a.html", ("x",), ())], + } From d2c848e5f32b2aaaf4019b753060e0e4e27383f3 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 20:42:27 +0200 Subject: [PATCH 2/7] fix(templates): collect each template's filters once per set of overridden blocks project_templates followed every path through includes and extends to collect the filters a render reaches. Two includes of one template at each of n levels made it follow 2^n paths: 20 levels took 31 s (#279). Each template is now walked once for each set of blocks the templates extending it override. Its first occurrences are kept in order, then mapped through each reference, which gives the same filters in the same order. The results last for one analysis, and a walk cut short by a cycle of includes is not kept. 30 levels now take a few milliseconds. --- src/coretrace_python/taint/templates.py | 48 +++++++++++++++++++------ tests/test_template_filter_walk.py | 31 ++++++++++++++++ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index e18312b..45e3265 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -992,11 +992,14 @@ def project_templates(root: Path) -> ProjectTemplates: reason = "not found" if library is None else "not read" if library.uncertain is None else "found in several files" unread.append(f"{relative}:{_position(text, load.offset)[0]} loads {load.library!r}, {reason}") calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications) + # The filters each template reaches, by template and blocks overridden, for this + # analysis only (#279). + reached: dict[tuple[str, frozenset[str]], list[_Reached]] = {} filters = { name: tuple( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in _reaching(name, files, read, frozenset()) + for symbol, span, where, value, argument in _reaching(name, files, read, frozenset(), frozenset(), reached)[0] ) ) for name in files @@ -1004,36 +1007,59 @@ def project_templates(root: Path) -> ProjectTemplates: return ProjectTemplates(frozenset(files), calls, tuple(unread), filters) +# A filter a template's render reaches: the function, where it is applied, the template +# applying it, and the names free in the rendered template reaching its value and argument. +_Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]] + + def _reaching( name: str, files: Mapping[str, list[Path]], read: Mapping[Path, _Template], visiting: frozenset[str], - overridden: frozenset[str] = frozenset(), -) -> Iterator[tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]]]: + overridden: frozenset[str], + reached: dict[tuple[str, frozenset[str]], list[_Reached]], +) -> tuple[list[_Reached], bool]: """The filters rendering the template ``name`` applies, in it and in what it includes or extends, each with the template applying it and the names free in ``name`` whose values reach its value and its argument; the templates extending ``name`` on the way - define the ``overridden`` blocks.""" + define the ``overridden`` blocks. Also whether a cycle of includes, through a template + ``visiting``, cut the walk short. + + Each template is walked once for each set of blocks overridden: ``reached`` keeps what + a walk found, its first occurrences in order, which mapped through a reference give + the first occurrences a walk path by path gives. A walk cut short by a cycle depends + on where the cycle was entered, and is not kept (#279).""" paths = files.get(name, []) - if len(paths) != 1 or paths[0] not in read or name in visiting: - return + if len(paths) != 1 or paths[0] not in read: + return [], False + if name in visiting: + return [], True + if (name, overridden) in reached: + return reached[name, overridden], False template = read[paths[0]] + found: dict[_Reached, None] = {} + cut = False for application in template.applications: if template.rendered(application.blocks, overridden): - yield application.symbol, application.span, name, application.value, application.argument + found.setdefault((application.symbol, application.span, name, application.value, application.argument)) for reference in template.references: if reference.name is None: continue if reference.extends: - found = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced) + below, below_cut = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced, reached) elif template.rendered(reference.blocks, overridden): - found = _reaching(reference.name, files, read, visiting | {name}) + below, below_cut = _reaching(reference.name, files, read, visiting | {name}, frozenset(), reached) else: continue - for symbol, span, where, value, argument in found: - yield symbol, span, where, reference.maps(value), reference.maps(argument) + cut = cut or below_cut + for symbol, span, where, value, argument in below: + found.setdefault((symbol, span, where, reference.maps(value), reference.maps(argument))) + result = list(found) + if not cut: + reached[name, overridden] = result + return result, cut def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) -> _Template: diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index c9df71e..9784156 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -9,10 +9,16 @@ from __future__ import annotations +import json +import subprocess +import sys from pathlib import Path +import pytest + from coretrace_python.taint.templates import project_templates +TIMEOUT = 60 FILTERS = "python.django.template.defaultfilters" @@ -119,3 +125,28 @@ def test_a_template_reached_inside_a_cycle_and_from_outside_it(tmp_path: Path) - "b.html": [(f"{FILTERS}.lower", "b.html", 1, 6, "b.html", ("y",), ()), (f"{FILTERS}.upper", "a.html", 1, 6, "a.html", ("x",), ())], "c.html": [(f"{FILTERS}.lower", "b.html", 1, 6, "b.html", ("y",), ()), (f"{FILTERS}.upper", "a.html", 1, 6, "a.html", ("x",), ())], } + + +COUNT_FILTERS = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python.taint.templates import project_templates\n" + "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n" +) + + +def test_includes_doubled_at_every_level_are_walked_in_linear_time(tmp_path: Path) -> None: + # Followed path by path, level30.html leads to level0.html 2 ** 30 times. + templates = {"level0.html": "{{ name|upper }}"} + for level in range(1, 31): + templates[f"level{level}.html"] = f"{{% include 'level{level - 1}.html' %}}" * 2 + root = write(tmp_path, templates) + try: + completed = subprocess.run( + [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT + ) + except subprocess.TimeoutExpired: + pytest.fail(f"the walk did not end within {TIMEOUT} s") + + assert completed.returncode == 0, completed.stderr + assert json.loads(completed.stdout) == {name: 1 for name in templates} From fac9e4611c6fb38912b8ef327d45b33300ec4479 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 20:42:27 +0200 Subject: [PATCH 3/7] docs(changelog): announce the linear walk of template filters --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc8376c..08e9701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,12 @@ `Finding.related` lists `RelatedLocation`s (a span and a message). `Report.rules` carries the descriptions to the reporters. +### Performance + +- Collecting the filters a template's render reaches walks each template once for each set + of blocks the templates extending it override (#279). It followed every path through + nested includes: two includes of one template at each of 20 levels took 31 s. + ## 0.23.0 (2026-10-16) ### Compatibility and migration From 78236b90442d0574db441ba90f5d2b57bd37cf2b Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 22:00:00 +0200 Subject: [PATCH 4/7] refactor(templates): return a template's walk of filters as a named tuple _reaching returned the filters it found and whether a cycle cut its walk short as a bare pair. It now returns a _Walk naming both, which the cache keeps whole. No behavior changes. --- src/coretrace_python/taint/templates.py | 38 ++++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 45e3265..b47d659 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -994,12 +994,12 @@ def project_templates(root: Path) -> ProjectTemplates: calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications) # The filters each template reaches, by template and blocks overridden, for this # analysis only (#279). - reached: dict[tuple[str, frozenset[str]], list[_Reached]] = {} + reached: dict[tuple[str, frozenset[str]], _Walk] = {} filters = { name: tuple( dict.fromkeys( TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) - for symbol, span, where, value, argument in _reaching(name, files, read, frozenset(), frozenset(), reached)[0] + for symbol, span, where, value, argument in _reaching(name, files, read, frozenset(), frozenset(), reached).found ) ) for name in files @@ -1012,14 +1012,24 @@ def project_templates(root: Path) -> ProjectTemplates: _Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]] +class _Walk(NamedTuple): + """What a walk of a template found, and whether a cycle of includes cut it short.""" + + found: list[_Reached] + cut: bool + + +_NOTHING = _Walk([], cut=False) + + def _reaching( name: str, files: Mapping[str, list[Path]], read: Mapping[Path, _Template], visiting: frozenset[str], overridden: frozenset[str], - reached: dict[tuple[str, frozenset[str]], list[_Reached]], -) -> tuple[list[_Reached], bool]: + reached: dict[tuple[str, frozenset[str]], _Walk], +) -> _Walk: """The filters rendering the template ``name`` applies, in it and in what it includes or extends, each with the template applying it and the names free in ``name`` whose values reach its value and its argument; the templates extending ``name`` on the way @@ -1033,11 +1043,11 @@ def _reaching( paths = files.get(name, []) if len(paths) != 1 or paths[0] not in read: - return [], False + return _NOTHING if name in visiting: - return [], True + return _Walk([], cut=True) if (name, overridden) in reached: - return reached[name, overridden], False + return reached[name, overridden] template = read[paths[0]] found: dict[_Reached, None] = {} cut = False @@ -1048,18 +1058,18 @@ def _reaching( if reference.name is None: continue if reference.extends: - below, below_cut = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced, reached) + below = _reaching(reference.name, files, read, visiting | {name}, overridden | template.replaced, reached) elif template.rendered(reference.blocks, overridden): - below, below_cut = _reaching(reference.name, files, read, visiting | {name}, frozenset(), reached) + below = _reaching(reference.name, files, read, visiting | {name}, frozenset(), reached) else: continue - cut = cut or below_cut - for symbol, span, where, value, argument in below: + cut = cut or below.cut + for symbol, span, where, value, argument in below.found: found.setdefault((symbol, span, where, reference.maps(value), reference.maps(argument))) - result = list(found) + walk = _Walk(list(found), cut) if not cut: - reached[name, overridden] = result - return result, cut + reached[name, overridden] = walk + return walk def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) -> _Template: From cf2dbb6e12ad4f09597d836ab770abd129ba75fd Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 22:00:00 +0200 Subject: [PATCH 5/7] fix(templates): make the filters of a template independent of the order of the walk A walk of a template is cut where it reaches a template already on the way. The cache kept a walk that no cycle cut, and returned it to any later walk of that template with the same blocks overridden, even where a template it had gone through was then on the way. The filters then depended on the template walked first: a child extending a parent that includes it back, in the block the child overrides, got its own filter reached through the parent's include, which a walk path by path cuts. Each walk now keeps the templates it went through, and is reused only where none of them is on the way. Elsewhere the walk would be cut, and is walked again. --- src/coretrace_python/taint/templates.py | 30 +++++++++++++--------- tests/test_template_filter_walk.py | 33 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index b47d659..91caa32 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -1013,13 +1013,15 @@ def project_templates(root: Path) -> ProjectTemplates: class _Walk(NamedTuple): - """What a walk of a template found, and whether a cycle of includes cut it short.""" + """What a walk of a template found; whether a cycle of includes cut it short; and the + templates it went through.""" found: list[_Reached] cut: bool + walked: frozenset[str] -_NOTHING = _Walk([], cut=False) +_NOTHING = _Walk([], cut=False, walked=frozenset()) def _reaching( @@ -1033,24 +1035,27 @@ def _reaching( """The filters rendering the template ``name`` applies, in it and in what it includes or extends, each with the template applying it and the names free in ``name`` whose values reach its value and its argument; the templates extending ``name`` on the way - define the ``overridden`` blocks. Also whether a cycle of includes, through a template - ``visiting``, cut the walk short. + define the ``overridden`` blocks. A template already ``visiting`` cuts the walk short. - Each template is walked once for each set of blocks overridden: ``reached`` keeps what - a walk found, its first occurrences in order, which mapped through a reference give - the first occurrences a walk path by path gives. A walk cut short by a cycle depends - on where the cycle was entered, and is not kept (#279).""" + Each template is walked once for each set of blocks overridden (#279): ``reached`` + keeps what a walk found, its first occurrences in order, which mapped through a + reference give the first occurrences a walk path by path gives. A walk depends on + ``visiting`` only where a cycle cuts it, so one cut short is not kept, and one that + went through a template now on the way is not reused: walked from there, it would be + cut at that template.""" paths = files.get(name, []) if len(paths) != 1 or paths[0] not in read: return _NOTHING if name in visiting: - return _Walk([], cut=True) - if (name, overridden) in reached: - return reached[name, overridden] + return _Walk([], cut=True, walked=frozenset()) + kept = reached.get((name, overridden)) + if kept is not None and visiting.isdisjoint(kept.walked): + return kept template = read[paths[0]] found: dict[_Reached, None] = {} cut = False + walked = {name} for application in template.applications: if template.rendered(application.blocks, overridden): found.setdefault((application.symbol, application.span, name, application.value, application.argument)) @@ -1064,9 +1069,10 @@ def _reaching( else: continue cut = cut or below.cut + walked |= below.walked for symbol, span, where, value, argument in below.found: found.setdefault((symbol, span, where, reference.maps(value), reference.maps(argument))) - walk = _Walk(list(found), cut) + walk = _Walk(list(found), cut, frozenset(walked)) if not cut: reached[name, overridden] = walk return walk diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index 9784156..6a3a1c5 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -4,7 +4,8 @@ They are the same as before, in the same order: which filters, where they stand, the template applying each, and the names of the render reaching its value and argument. The results are kept for one analysis only, so that projects analysed one after the other do -not share them, and a result cut short by a cycle of includes is not reused. +not share them. A result cut short by a cycle of includes is not reused, nor one that +went through a template already on the way, where the walk would be cut. """ from __future__ import annotations @@ -150,3 +151,33 @@ def test_includes_doubled_at_every_level_are_walked_in_linear_time(tmp_path: Pat assert completed.returncode == 0, completed.stderr assert json.loads(completed.stdout) == {name: 1 for name in templates} + + +# A child extending a parent that includes it back, inside the block the child overrides. +# The walk of the parent includes the child, whose extends leads back to the parent: a +# cycle, cut there. Whichever of the two is walked first, each gets the filters of main. +def cycle_through_an_overridden_block(child: str, parent: str) -> dict[str, str]: + return { + child: "{% extends '" + parent + "' %}\n{% block body %}{{ v|upper }}{% endblock %}\n", + parent: "{{ v|striptags }}\n{% block body %}{% include '" + child + "' with v=other %}{% endblock %}\n", + } + + +@pytest.mark.parametrize( + ("child", "parent"), [("a.html", "b.html"), ("b.html", "a.html")], ids=["child-walked-first", "parent-walked-first"] +) +def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_walked_first( + tmp_path: Path, child: str, parent: str +) -> None: + root = write(tmp_path, cycle_through_an_overridden_block(child, parent)) + + assert described(root) == { + child: [ + (f"{FILTERS}.upper", child, 2, 22, child, ("v",), ()), + (f"{FILTERS}.striptags", parent, 1, 6, parent, ("v",), ()), + ], + parent: [ + (f"{FILTERS}.striptags", parent, 1, 6, parent, ("v",), ()), + (f"{FILTERS}.upper", child, 2, 22, child, ("other",), ()), + ], + } From 24320af082b1ba86b345a796bfb90b936dea4921 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 22:00:00 +0200 Subject: [PATCH 6/7] test(templates): compare the walk of template filters with a walk path by path 1,500 generated projects of four templates extend and include one another, inside and outside overridden blocks, with cycles among them; each include binds the variable to a name of its own. The filters of every template must be those of the walk of main, written in the test, which follows every path: which filters, where, the template applying each, the names reaching them, and their order. --- tests/test_template_filter_walk.py | 96 ++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/test_template_filter_walk.py b/tests/test_template_filter_walk.py index 6a3a1c5..e30b498 100644 --- a/tests/test_template_filter_walk.py +++ b/tests/test_template_filter_walk.py @@ -10,13 +10,20 @@ from __future__ import annotations +import itertools import json +import random import subprocess import sys +from collections.abc import Callable, Iterator from pathlib import Path +from typing import Any import pytest +from coretrace_python.interprocedural import TemplateFilter, discover_files +from coretrace_python.source import SourceId +from coretrace_python.taint import templates from coretrace_python.taint.templates import project_templates TIMEOUT = 60 @@ -181,3 +188,92 @@ def test_a_child_its_parent_includes_back_gets_the_same_filters_whichever_is_wal (f"{FILTERS}.upper", child, 2, 22, child, ("other",), ()), ], } + + +def path_by_path( + name: str, + files: dict[str, list[Path]], + read: dict[Path, Any], + visiting: frozenset[str], + overridden: frozenset[str] = frozenset(), +) -> Iterator[tuple[Any, ...]]: + """The walk of main, every path through includes and extends followed, a cycle cut at + a template already on the way: the reference the walk of the engine must equal.""" + + paths = files.get(name, []) + if len(paths) != 1 or paths[0] not in read or name in visiting: + return + template = read[paths[0]] + for application in template.applications: + if template.rendered(application.blocks, overridden): + yield application.symbol, application.span, name, application.value, application.argument + for reference in template.references: + if reference.name is None: + continue + if reference.extends: + found = path_by_path(reference.name, files, read, visiting | {name}, overridden | template.replaced) + elif template.rendered(reference.blocks, overridden): + found = path_by_path(reference.name, files, read, visiting | {name}) + else: + continue + for symbol, span, where, value, argument in found: + yield symbol, span, where, reference.maps(value), reference.maps(argument) + + +def reference_filters(root: Path) -> dict[str, list[TemplateFilter]]: + """The filters of each template, as project_templates gives them, from the walk of main.""" + + found = {path: tuple(templates._names(path.relative_to(root).parts)) for path in discover_files(root)} + libraries = templates._libraries(found) + paths = sorted(path for path, names in found.items() if names) + files: dict[str, list[Path]] = {} + for path in paths: + for name in found[path]: + files.setdefault(name, []).append(path) + read = {path: templates._read_scopes(text, SourceId(str(path)), libraries) for path in paths if (text := templates._text(path))} + return { + name: list( + dict.fromkeys( + TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument))) + for symbol, span, where, value, argument in path_by_path(name, files, read, frozenset()) + ) + ) + for name in files + } + + +NAMES = [f"t{index}.html" for index in range(4)] + + +def generated(rng: random.Random, fresh: Iterator[str], depth: int = 0) -> str: + """Markup applying filters, including other templates, inside blocks of two names and + outside them. An include binds the variable to a name of its own, so that what reaches + a filter tells the paths apart.""" + + pieces: list[Callable[[], str]] = [ + lambda: "{{ v|upper }}", + lambda: "{{ v|striptags }}", + lambda: "{% include '" + rng.choice(NAMES) + "' %}", + lambda: "{% include '" + rng.choice(NAMES) + "' with v=" + next(fresh) + " %}", + ] + if depth < 2: + block = rng.choice(["body", "side"]) + pieces.append(lambda: "{% block " + block + " %}" + generated(rng, fresh, depth + 1) + "{% endblock %}") + return "".join(rng.choice(pieces)() for _ in range(rng.randint(1, 3))) + + +def test_the_walk_gives_the_filters_of_main_on_inheritance_overridden_blocks_and_cycles(tmp_path: Path) -> None: + """Generated projects of four templates extending and including one another, cycles + among them, each walked by the engine and by the walk of main, path by path.""" + + rng = random.Random(279) + for index in range(1_500): + fresh = (f"n{number}" for number in itertools.count()) + texts = {} + for name in NAMES: + extends = "{% extends '" + rng.choice(NAMES) + "' %}" if rng.random() < 0.5 else "" + texts[name] = extends + generated(rng, fresh) + "\n" + root = write(tmp_path / str(index), texts) + filters = project_templates(root).filters + + assert {name: list(found) for name, found in filters.items()} == reference_filters(root), texts From 0759552013ba88b54521b96012d4887b9ab9f188 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 23 Oct 2026 22:00:00 +0200 Subject: [PATCH 7/7] docs(changelog): state that templates in a cycle of includes are walked again A walk cut short by a cycle of includes is not kept, so templates in such a cycle are walked again wherever the cycle cuts their walk: with two includes of one template at each level, the cost there stays exponential, as it was. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08e9701..aa16f6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ - Collecting the filters a template's render reaches walks each template once for each set of blocks the templates extending it override (#279). It followed every path through nested includes: two includes of one template at each of 20 levels took 31 s. + A template in a cycle of includes is still walked again wherever the cycle cuts its + walk short. ## 0.23.0 (2026-10-16)