From 5f00505f564e0e1a5be5089f1764fbc28337cf2d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:01:27 +1000 Subject: [PATCH 001/110] drop from uv.lock the five zensical 0.0.57 wheels pypi no longer serves, for 32-bit arm, 32-bit x86 and 32-bit windows, which made the uv-lock hook fail in ci --- uv.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/uv.lock b/uv.lock index 67dfec337f..01a7226f83 100644 --- a/uv.lock +++ b/uv.lock @@ -1875,13 +1875,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/b9/49c37dc65105d1ca4a8b600a02c84ece00218d2293b2630611c620185ca3/zensical-0.0.57-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:98867d1a6ea2c57f1ebcf4902f61601f427350f2df0c04e30cfac8ba6163cd29", size = 12888507, upload-time = "2026-08-21T20:43:20.365Z" }, { url = "https://files.pythonhosted.org/packages/05/f7/54539984418de11387bbace39a744195555d32c98c95bf4d112b432548f5/zensical-0.0.57-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:0d7935d77d73a279545052e05d89d31960f30c1f33f53933f4c101fa271aee74", size = 12778169, upload-time = "2026-08-21T20:43:22.879Z" }, { url = "https://files.pythonhosted.org/packages/40/16/74aa60aa4cfecd5bd31ce60cb6a092cb56f1bc1aaadcc173463861ea4eb5/zensical-0.0.57-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7046d433511d97aa603915f0f6792d15b7f839793abc2b66ab7b7ff753ecff5", size = 13230823, upload-time = "2026-08-21T20:43:25.141Z" }, - { url = "https://files.pythonhosted.org/packages/8a/d1/742d2487dd65dd18277daebcd37db56d5bd4a2408df02bde703ef8fb7b64/zensical-0.0.57-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab85c5066b95e3a877cf8971e4ce30abb1ca1459fbfcc631f0a5a2bab56351a4", size = 13170523, upload-time = "2026-08-21T20:43:27.456Z" }, - { url = "https://files.pythonhosted.org/packages/56/6f/12b570775d344f1a3d77e26d4ae0160bcac9e41ca38f7135352ccdf9b2c8/zensical-0.0.57-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f13d1b57ad3c8b8634933a93ea870ebac11245fe0c968d27fd2a059ee1c6311", size = 13549941, upload-time = "2026-08-21T20:43:29.964Z" }, { url = "https://files.pythonhosted.org/packages/7b/4e/436e6fc76674244c084ef7f6f17dc5ff85c76b15aef77c48b703fd0a2dda/zensical-0.0.57-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:021dd8fb70d1816cd012684fcf45d32b8f88a0cd28b7cbe71e5f8564f6d5764d", size = 13210086, upload-time = "2026-08-21T20:43:32.098Z" }, { url = "https://files.pythonhosted.org/packages/ef/52/20f3aeda9af1090f24241670a5cc20fff7494545fea9f5fa094c82f3dbdf/zensical-0.0.57-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7e10f3c27fdc3eac3a9ae6ddcd87f3f00edc9f332050923313c95537961bfadd", size = 13408253, upload-time = "2026-08-21T20:43:34.258Z" }, - { url = "https://files.pythonhosted.org/packages/e1/f2/2b18ba2f19674dbfcf745f3b66e005cc8efa66a1bcaba5e1b4f79467868a/zensical-0.0.57-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:78c85fee55c5aac3bdf8157e980c56397dca835167a5577c5429b5eb24ed990c", size = 13446689, upload-time = "2026-08-21T20:43:36.527Z" }, - { url = "https://files.pythonhosted.org/packages/05/ba/68cdba447a9097e5f97742eef046020c6fa42d82972849b3a46a0718e890/zensical-0.0.57-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:478d252e1924f3876e72cf7806967cb62e50d86eddb3da04bf43e882b532fa1b", size = 13598580, upload-time = "2026-08-21T20:43:38.646Z" }, { url = "https://files.pythonhosted.org/packages/ec/89/6358a4df272328bed5bea90b04d43e73758bc45ff058c5cb2665e1147314/zensical-0.0.57-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66a9ca6b5f625b2a2b215eec2f3c72843a92d5d512042045ac6351d5dee9b339", size = 13557609, upload-time = "2026-08-21T20:43:40.866Z" }, - { url = "https://files.pythonhosted.org/packages/77/e1/8831301a24f736743e3788f09ea048918b0bdcea4aaa90f7770a433d6eec/zensical-0.0.57-cp310-abi3-win32.whl", hash = "sha256:f0fe3dc27ca7dc4e168eddd0fe5b0f4d44e311fd4e0019241e289819e445203c", size = 12446805, upload-time = "2026-08-21T20:43:43.097Z" }, { url = "https://files.pythonhosted.org/packages/d7/3f/5d0ecd77d9ce962fdfde22dec036f4257a43ef6dbd55fb5c05fd294985ad/zensical-0.0.57-cp310-abi3-win_amd64.whl", hash = "sha256:a756834025c1c54e806e943be6d8df1048d0f8bcf6086e958568407a070a2572", size = 12716781, upload-time = "2026-08-21T20:43:45.273Z" }, ] From 9e252ac418370044ff39101319aa4b6b299cedb7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:23:11 +1000 Subject: [PATCH 002/110] widen a kind of literal that keeps growing from one round of a cycle to the next, so a string built in a loop converges on str when the loop sits inside a larger cycle, as it does when an unannotated parameter's bound is read off the body that checks a call with it, instead of gaining one literal a round until salsa panics with too many cycle iterations  Conflicts:  crates/ty_python_semantic/src/types.rs --- .../resources/mdtest/cycle/basic.md | 22 ++++++ crates/ty_python_semantic/src/types.rs | 77 +++++++++++++++++++ .../src/types/set_theoretic/builder.rs | 2 +- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md index 1c3c5301a7..5aafe59ecd 100644 --- a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md +++ b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md @@ -756,6 +756,28 @@ def holder(): reveal_type(holder()) # revealed: Box[Literal[1]] ``` +## a string built in a loop inside a larger cycle widens rather than diverging + +A loop that prepends to a string is a different literal on every trip. On its own the loop is a +cycle, and its literals widen to `str`. Here it is part of a larger one: the bound of `many`'s +unannotated parameter `x` is read off `many`'s body, and checking the call `one(x)` in that body +consults the bound. Each round of that cycle used to add one more literal — `" "`, then `" "` — +until salsa gave up on the iteration and the checker panicked. A kind of literal that keeps growing +from round to round is widened, as it would be in the loop alone. + +```py +def one(x): + if x: + return x + return "" + +def many(x): + result = "" + while x: + result = one(x) + " " + result + reveal_type(result) # revealed: str | Unknown +``` + ## the recursive `defaultdict` factory converges The same shape reaches ty through the standard library, where it is a common idiom for a tree of diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 35c9033f87..046ecf20ae 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -2850,6 +2850,7 @@ impl<'db> Type<'db> { // would normalize it, and the widened and marked forms would alternate without // either being reached .without_growing_tuple_lengths(db, env, previous) + .without_growing_literals(db, env, previous) .without_growing_self_nesting(db, env, previous, cycle) .recursive_type_normalized_impl_with_cycle(db, env, cycle) } @@ -3065,6 +3066,82 @@ impl<'db> Type<'db> { UnionType::from_elements_cycle_recovery(db, env, rest) } + /// basedpython: `self`, with a family of literals the fixed-point iteration is growing + /// widened to the class they are instances of. + /// + /// A string built up in a loop is a different literal every time round: + /// + /// ```python + /// result = "" + /// while x: + /// result = " " + result + /// ``` + /// + /// The loop is ordinarily a cycle of its own, and the union builder widens it: its values + /// carry the cycle's `Divergent` marker, which marks the literals built from them as + /// recursively defined, and a union holding [`MAX_RECURSIVE_UNION_LITERALS`] of those is + /// widened to `str`. But a loop can also sit inside a larger cycle — a body whose inference + /// depends on itself, as one does when an unannotated parameter's bound is read off that very + /// body — and then the loop is no cycle of its own: the marker belongs to the outer cycle, is + /// gone after its first round, and each round adds one more literal, `" "`, then `" "`, until + /// salsa gives up on the iteration. + /// + /// So the growth is recognized where it shows, in the rounds themselves: a kind of literal + /// that already had as many members as the union builder lets a recursively defined union + /// hold, and gains more past the rounds [`crate::TAINTED_CYCLES`] leaves alone, is the + /// iteration enumerating values rather than the program naming them. That kind is widened, + /// which is what the same loop infers on its own. A kind that stopped growing is left as it + /// is, and so is one that arrives whole: `Literal["a", "b", "c", "d", "e", "f"]` written in an + /// annotation can arrive in a single round, after rounds that held fewer of its members. + /// + /// [`MAX_RECURSIVE_UNION_LITERALS`]: crate::types::set_theoretic::builder::MAX_RECURSIVE_UNION_LITERALS + fn without_growing_literals( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + previous: Self, + ) -> Self { + let literal_counts = |ty: Self| { + let mut counts: FxHashMap, usize> = FxHashMap::default(); + for element in ty.union_elements(db) { + if let Type::LiteralValue(literal) = element { + *counts + .entry(literal.fallback_instance(db, env)) + .or_default() += 1; + } + } + counts + }; + let after = literal_counts(self); + if after.is_empty() { + return self; + } + let before = literal_counts(previous); + let is_growing = |class: &Type<'db>| { + let before = before.get(class).copied().unwrap_or(0); + before >= set_theoretic::builder::MAX_RECURSIVE_UNION_LITERALS && after[class] > before + }; + if !after.keys().any(is_growing) { + return self; + } + // each growing kind takes the place of its first literal, so the union keeps its order + let mut widened: FxHashSet> = FxHashSet::default(); + let elements = self.union_elements(db).filter_map(|element| match element { + Type::LiteralValue(literal) => { + let class = literal.fallback_instance(db, env); + if !is_growing(&class) { + Some(element) + } else if widened.insert(class) { + Some(class) + } else { + None + } + } + _ => Some(element), + }); + UnionType::from_elements_cycle_recovery(db, env, elements.collect::>()) + } + /// collapse a union that is accumulating one tuple per *length* into a single /// variable-length tuple /// diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index 1116fe6475..8761d36d82 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -618,7 +618,7 @@ enum ReduceResult<'db> { /// If the value ​​is defined recursively, widening is performed from fewer literal elements, /// resulting in faster convergence of the fixed-point iteration. -const MAX_RECURSIVE_UNION_LITERALS: usize = 5; +pub(crate) const MAX_RECURSIVE_UNION_LITERALS: usize = 5; /// If the value ​​is defined non-recursively, the fixed-point iteration will converge in one go, /// so in principle we can have as many literal elements as we want. /// We set a large limit for union and enum literals. From 6b3dc35fe49ab815cc67e3d1d5453233e84c42df Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:29:50 +1000 Subject: [PATCH 003/110] find a decorated override as the def it is, not as the function its decorator returns, so go to implementation lands on Circle.area rather than on logged's wrapper, and a decorated method nothing overrides counts no implementations --- crates/ty_ide/src/goto_implementation.rs | 41 +++++++++++++++++++ crates/ty_ide/src/usage_counts.rs | 30 ++++++++++++++ .../src/types/ide_support.rs | 17 +++++++- docs/basedpython/features/editor.md | 4 +- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/crates/ty_ide/src/goto_implementation.rs b/crates/ty_ide/src/goto_implementation.rs index ccd0244e37..ae63e274a7 100644 --- a/crates/ty_ide/src/goto_implementation.rs +++ b/crates/ty_ide/src/goto_implementation.rs @@ -350,6 +350,47 @@ mod tests { "); } + /// An override with a decorator is its own `def`, not the function the decorator returns, + /// which is defined somewhere else entirely. + #[test] + fn implementation_decorated_override_is_its_def() { + let test = cursor_test( + r#" + def logged(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + + class Shape: + def area(self) -> float: + return 0.0 + + class Circle(Shape): + @logged + def area(self) -> float: + return 1.0 + "#, + ); + + assert_snapshot!(test.goto_implementation(), @" + info[goto-implementation]: Go to implementation + --> main.py:8:9 + | + 8 | def area(self) -> float: + | ^^^^ Clicking here + info: Found 2 implementations + --> main.py:8:9 + | + 8 | def area(self) -> float: + | ---- + | + ::: main.py:13:9 + | + 13 | def area(self) -> float: + | ---- + "); + } + #[test] fn implementation_transitive_subclass_overrides() { let test = cursor_test( diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index dd16964bcd..79651afeec 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -338,6 +338,36 @@ print(Shape().area(), Square().area()) "); } + /// A decorated method is counted by its own `def`: what the decorator hands back is defined + /// elsewhere, and is neither an override of the method nor overridden by it. + #[test] + fn a_decorated_method_is_counted_as_its_def() { + let test = cursor_test( + " +def logged(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + +class Shape: + def area(self) -> float: + return 0.0 + +class Circle(Shape): + @logged + def area(self) -> float: + return 1.0 +", + ); + assert_snapshot!(test.usage_counts(), @" + function logged 2 refs=1 + class Shape 7 refs=1 impls=1 + method area 8 refs=0 impls=1 + class Circle 11 refs=0 impls=0 + method area 13 refs=0 impls=0 + "); + } + #[test] fn dunders_nested_functions_and_extensions_are_not_counted() { let test = CursorTest::builder() diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 1f9ed66463..8aa8fe141b 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -1160,7 +1160,22 @@ fn member_implementation_definition<'db>( // Use the inferred function type to collapse overload declarations to their concrete // implementation. If inference cannot produce a function literal, keep the original `def` as a // conservative fallback. - let Some(function) = binding_type(db, definition).as_function_literal() else { + // + // This is the function the `def` defines, before any decorator. The name's binding type is + // what the decorators return, which for + // + // ```py + // def logged(func): + // def wrapper(*args, **kwargs): ... + // return wrapper + // + // class Circle(Shape): + // @logged + // def area(self) -> float: ... + // ``` + // + // is `wrapper`, a function defined somewhere else entirely. + let Some(function) = infer_definition_types(db, definition).function_type(definition) else { return Some(ResolvedDefinition::Definition(definition)); }; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index d004a4cfa9..08d63840a1 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -506,7 +506,9 @@ language services are off ## go to super `textDocument/implementation` on a method goes down, to the methods that -override it. `by/superMembers` goes up: given a position on the name a class +override it. a decorated override is its own `def`, not the function its +decorator returns, which a decorator like `logged`, returning a `wrapper` of its +own, defines somewhere else. `by/superMembers` goes up: given a position on the name a class member is declared with — a `def` or a nested class, a name the class body assigns, annotates or captures, or an import's alias — it answers the superclass members that member overrides, each with its class, its file and the From 8cee3baebad3a292559e91028b3698049f44d1dd Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:32:51 +1000 Subject: [PATCH 004/110] give a type that mentions Unknown no annotation, so add return annotation refuses a partly unknown return type rather than writing -> Unknown | ... and importing ty_extensions._internal, which fails when the program runs, and an inlay hint's insertion leaves such a type unwritten --- crates/ty_ide/src/inlay_hints.rs | 7 +++++++ .../ty_ide/src/refactor/return_annotation.rs | 19 +++++++++++++++++++ docs/basedpython/features/editor.md | 6 ++++++ 3 files changed, 32 insertions(+) diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 79d01de34e..b57397be3e 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -1409,6 +1409,13 @@ fn spell_type<'db>( for (target, detail) in details.targets.iter().zip(&details.details) { match detail { TypeDetail::Type(ty) => { + // `Unknown` is what ty says where it knows nothing, not a type a program can + // name: `ty_extensions` exists only for the checker, so an annotation naming it + // fails to import when the program runs, and one that held would claim a + // gradual type the author never chose. a type mentioning it has no annotation + if ty.is_unknown() { + editable = false; + } let start = target.start().to_usize(); let end = target.end().to_usize(); // If we skipped over some bytes, push them with no target diff --git a/crates/ty_ide/src/refactor/return_annotation.rs b/crates/ty_ide/src/refactor/return_annotation.rs index be9940d237..39c169ea6b 100644 --- a/crates/ty_ide/src/refactor/return_annotation.rs +++ b/crates/ty_ide/src/refactor/return_annotation.rs @@ -198,6 +198,25 @@ mod tests { "); } + /// `x + y` of two unannotated parameters is a type ty does not know, and `Unknown` is no type + /// an annotation can name: `ty_extensions` exists only for the checker. writing the rest of + /// the union would claim more than the body says + #[test] + fn a_partly_unknown_return_type_is_not_written() { + assert_snapshot!( + RefactorTest::basedpython( + " + def combine(x, y): + if x and not y: + return x + y + return \"a\" + \"b\" + ", + ) + .apply(RefactorKind::AddReturnAnnotation), + @"refused: Add return annotation (the inferred return type has no spelling this file can write)" + ); + } + #[test] fn annotated_function_is_not_offered() { assert_snapshot!(annotate( diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 08d63840a1..36a66c6a18 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -363,6 +363,12 @@ while queue.pending(): # extracting `queue.pending()` is refused: it runs every queue.pop() ``` +adding a return annotation writes the return type the checker infers, importing +what it names. it is refused when that type is partly `Unknown` — `x + y` of two +unannotated parameters — since `Unknown` is no type a program can name, and the +rest of the union would claim more than the body says. an inlay hint's +double-click edit follows the same rule + extracting statements from a method makes a method of the same class, called through the receiver. what the statements read from the function becomes a parameter, and what they assign that is read afterwards is returned From c392cdd085db20c24b1f7320823caac71121e090 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:38:26 +1000 Subject: [PATCH 005/110] complete a callable that takes no arguments as its whole call, with the caret after the parentheses and no signature help asked for, instead of leaving the caret inside an empty pair, and test that completions.completeFunctionParentheses in the initialization options turns the parentheses off --- crates/ty_ide/src/completion.rs | 60 ++++++++++++- .../src/types/ide_support.rs | 21 +++++ crates/ty_server/tests/e2e/completions.rs | 90 +++++++++++++++++-- docs/basedpython/features/editor.md | 10 ++- 4 files changed, 170 insertions(+), 11 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 537cccdf2c..54d419a546 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -30,6 +30,7 @@ use ty_python_semantic::importer::{ImportRequest, Importer, MembersInScope}; use ty_python_semantic::types::format::{SpecLanguage, spec_language}; use ty_python_semantic::types::ide_support::{ context_sensitive_members, extension_members, is_awaitable, overridable_members, + takes_no_arguments, }; use ty_python_semantic::types::{SpecialFormType, UnionType}; use ty_python_semantic::{ @@ -734,7 +735,12 @@ impl<'db> CompletionBuilder<'db> { && collection_context.should_complete_callable_parentheses(kind) { let label = self.insert.unwrap_or_else(|| self.name.clone()); - if collection_context.capabilities.snippets { + // a call that takes nothing has nothing to type between its parentheses, so the + // caret goes after them, and there is no signature to show + let takes_no_arguments = self.ty.is_some_and(|ty| { + takes_no_arguments(db, &ProgramEnvironment::from_file(program_file), ty) + }); + if collection_context.capabilities.snippets && !takes_no_arguments { let insert = compact_str::format_compact!("{label}($0)"); ( Some(label), @@ -14685,7 +14691,7 @@ def foo(): fn complete_function_parentheses() { let builder = completion_test_builder( "\ -def callable(): +def callable(x): pass call @@ -14703,6 +14709,56 @@ call ); } + /// a call that takes nothing has nothing to type between its parentheses, so the caret goes + /// after them. one whose parameters are all optional still leaves the caret inside: what goes + /// there is up to the caller + #[test] + fn complete_function_parentheses_without_arguments() { + let snapshot = |source: &str| { + completion_test_builder(source) + .skip_auto_import() + .skip_builtins() + .complete_function_parentheses() + .snippets_support(true) + .build() + .snapshot() + }; + assert_snapshot!( + snapshot( + "\ +def callable(): + pass + +call +" + ), + @"callable()", + ); + assert_snapshot!( + snapshot( + "\ +class Cart: + def total(self) -> int: + return 0 + +Cart().tot +" + ), + @"total()", + ); + assert_snapshot!( + snapshot( + "\ +def callable(x=1): + pass + +call +" + ), + @"callable($0)", + ); + } + #[test] fn complete_function_parentheses_existing_parenthesis() { let builder = completion_test_builder( diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 8aa8fe141b..3987d12aeb 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -1627,6 +1627,27 @@ fn displayed_parameters_for_signature<'db>( } } +/// whether `ty` is called with no arguments at all: it is callable, and none of the ways it can be +/// called has a parameter left once a bound method's receiver is bound +/// +/// `Cart().total` with `def total(self) -> int` is, so a completion that writes the call can put +/// the caret after `()` rather than inside it. a function with only optional parameters is not: +/// what goes between the parentheses is still up to the caller +pub fn takes_no_arguments<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> bool { + ty.try_upcast_to_callable(db, env).is_some_and(|callables| { + callables.into_iter().all(|callable| { + callable + .signatures(db) + .iter() + .all(|signature| signature.parameters().len() == 0) + }) + }) +} + /// Extract signature details from a function call expression. /// This function analyzes the callable being invoked and returns zero or more /// `CallSignatureDetails` objects, each representing one possible signature diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 7f0df92fea..b9a0462eae 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -122,7 +122,7 @@ fn complete_function_parentheses_enabled_by_default() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); let foo_content = "\ -def complete_parentheses() -> None: ... +def complete_parentheses(x: int) -> None: ... complete_parenth "; @@ -143,7 +143,7 @@ complete_parenth { "label": "complete_parentheses", "kind": 3, - "detail": "def complete_parentheses()", + "detail": "def complete_parentheses(x: int)", "sortText": "0", "insertText": "complete_parentheses($0)", "insertTextFormat": 2 @@ -159,7 +159,7 @@ fn complete_function_parentheses() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); let foo_content = "\ -def complete_parentheses() -> None: ... +def complete_parentheses(x: int) -> None: ... complete_parenth "; @@ -183,7 +183,7 @@ complete_parenth { "label": "complete_parentheses", "kind": 3, - "detail": "def complete_parentheses()", + "detail": "def complete_parentheses(x: int)", "sortText": "0", "insertText": "complete_parentheses($0)", "insertTextFormat": 2, @@ -201,13 +201,91 @@ complete_parenth /// Tests that the signature-help command is omitted when the client has not /// advertised support for it (for example, editors that would surface an /// "unsupported command" error). +/// a function that takes no arguments is written whole, `()` included, with the caret after it: +/// there is nothing to type between the parentheses and no signature to show #[test] -fn complete_function_parentheses_without_command_support() -> Result<()> { +fn complete_function_parentheses_without_arguments() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); let foo_content = "\ def complete_parentheses() -> None: ... +complete_parenth +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .enable_completion_snippets(true) + .with_trigger_parameter_hints_command() + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let completions = server.completion_request(&server.file_uri(foo), Position::new(2, 16)); + insta::assert_json_snapshot!(completions, @r#" + [ + { + "label": "complete_parentheses", + "kind": 3, + "detail": "def complete_parentheses()", + "sortText": "0", + "insertText": "complete_parentheses()" + } + ] + "#); + + Ok(()) +} + +/// `completions.completeFunctionParentheses: false` in the initialization options turns the +/// parentheses off, for a client whose user does not want them +#[test] +fn complete_function_parentheses_disabled() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +def complete_parentheses(x: int) -> None: ... + +complete_parenth +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options( + &ClientOptions::default().with_complete_function_parentheses(false), + ) + .enable_completion_snippets(true) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let completions = server.completion_request(&server.file_uri(foo), Position::new(2, 16)); + insta::assert_json_snapshot!(completions, @r#" + [ + { + "label": "complete_parentheses", + "kind": 3, + "detail": "def complete_parentheses(x: int)", + "sortText": "0" + } + ] + "#); + + Ok(()) +} + +#[test] +fn complete_function_parentheses_without_command_support() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +def complete_parentheses(x: int) -> None: ... + complete_parenth "; @@ -229,7 +307,7 @@ complete_parenth { "label": "complete_parentheses", "kind": 3, - "detail": "def complete_parentheses()", + "detail": "def complete_parentheses(x: int)", "sortText": "0", "insertText": "complete_parentheses($0)", "insertTextFormat": 2 diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 36a66c6a18..f5c9e9290a 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -191,9 +191,13 @@ block in scope declares on the receiver, alongside the type's own ### callables accepting a completion for anything callable writes its parentheses and leaves -the caret between them, ready for the first argument. set -`ty.completions.completeFunctionParentheses` to `false` to insert the bare name -instead +the caret between them, ready for the first argument — or after them, when the +callable takes no arguments at all, as `cart.total()` does for +`def total(self) -> int`. one whose parameters are all optional still leaves the +caret inside. set `ty.completions.completeFunctionParentheses` to `false` (or +`completions.completeFunctionParentheses` in the initialization options, for a +client that does not answer `workspace/configuration` with it) to insert the +bare name instead ## inlay hints From 245887ad2aeb7d571e06d62f98e8b6e2d29738ab Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:42:02 +1000 Subject: [PATCH 006/110] fold a docstring once, as the documentation range, not a second time as the multi-line string it also is --- crates/ty_ide/src/folding_range.rs | 92 +++++------------------------- 1 file changed, 15 insertions(+), 77 deletions(-) diff --git a/crates/ty_ide/src/folding_range.rs b/crates/ty_ide/src/folding_range.rs index dc62bde168..c2cbf4f454 100644 --- a/crates/ty_ide/src/folding_range.rs +++ b/crates/ty_ide/src/folding_range.rs @@ -11,6 +11,7 @@ use ruff_python_ast::{ use ruff_python_trivia::{CommentLinePosition, is_python_whitespace}; use ruff_source_file::{LineRanges, UniversalNewlines}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashSet; use crate::Db; @@ -66,6 +67,7 @@ pub fn folding_ranges( let mut visitor = FoldingRangeVisitor { source: source.as_str(), ranges: vec![], + docstrings: FxHashSet::default(), active_block_fold_start_lines: vec![], tokens: parsed.tokens(), range_filter, @@ -102,6 +104,9 @@ pub fn folding_ranges( struct FoldingRangeVisitor<'a> { source: &'a str, ranges: Vec, + /// the docstrings given a range of their own, which the string literal each of them is + /// does not get a second time + docstrings: FxHashSet, active_block_fold_start_lines: Vec>, tokens: &'a Tokens, range_filter: Option, @@ -416,7 +421,11 @@ impl<'a> FoldingRangeVisitor<'a> { if !is_string_like { return; } - self.add_range(FoldingRange::from(first_stmt.range()).with_kind(FoldingRangeKind::Comment)); + if self + .add_range(FoldingRange::from(first_stmt.range()).with_kind(FoldingRangeKind::Comment)) + { + self.docstrings.insert(expr_stmt.value.range()); + } } /// Attempts to find the start of the given token in the given range. @@ -672,20 +681,10 @@ impl<'a> SourceOrderVisitor<'a> for FoldingRangeVisitor<'a> { // Compound statements that create folding regions AnyNodeRef::StmtFunctionDef(func) => { self.add_documented_block_ranges(node, &func.body); - // Note that this may be duplicative with folding - // ranges added for string literals. But I don't think - // the LSP protocol specifies that this is a problem. - // If we do need to de-dupe, then we'll want to keep - // this one since it attaches a "comment" folding range - // kind to the range. So we'll need to skip over the - // corresponding range for the literal. self.add_docstring_range(&func.body); } AnyNodeRef::StmtClassDef(class) => { self.add_documented_block_ranges(node, &class.body); - // See comment above for class docstrings about this - // being duplicative with adding folding ranges for - // string literals. self.add_docstring_range(&class.body); } AnyNodeRef::StmtIf(if_stmt) => { @@ -800,12 +799,15 @@ impl<'a> SourceOrderVisitor<'a> for FoldingRangeVisitor<'a> { self.add_call_expression_ranges(call, node); } - // String and bytes literals + // string and bytes literals. a docstring already has its range, and an editor + // given the same range twice draws two folds on one line AnyNodeRef::ExprStringLiteral(_) | AnyNodeRef::ExprBytesLiteral(_) | AnyNodeRef::ExprFString(_) | AnyNodeRef::ExprTString(_) => { - self.add_expression_range(node.range()); + if !self.docstrings.contains(&node.range()) { + self.add_expression_range(node.range()); + } } _ => {} @@ -2310,15 +2312,6 @@ def my_function(): 3 | / """ 4 | | This is a multiline 5 | | docstring. - 6 | | """ - | |_______^ - - info[folding-range]: Folding Range - --> main.py:3:5 - | - 3 | / """ - 4 | | This is a multiline - 5 | | docstring. 6 | | """ | |_______^ "#); @@ -2380,15 +2373,6 @@ def with_rawstring_doc(): 6 | | """ | |_______^ - info[folding-range]: Folding Range - --> main.py:3:5 - | - 3 | / f""" - 4 | | This is an f-string - 5 | | used as a docstring. - 6 | | """ - | |_______^ - info[folding-range]: Folding Range --> main.py:10:24 | @@ -2410,15 +2394,6 @@ def with_rawstring_doc(): 14 | | """ | |_______^ - info[folding-range]: Folding Range - --> main.py:11:5 - | - 11 | / t""" - 12 | | This is a t-string - 13 | | used as a docstring. - 14 | | """ - | |_______^ - info[folding-range]: Folding Range --> main.py:18:26 | @@ -2437,15 +2412,6 @@ def with_rawstring_doc(): 19 | / r""" 20 | | This is a raw string 21 | | used as a docstring. - 22 | | """ - | |_______^ - - info[folding-range]: Folding Range - --> main.py:19:5 - | - 19 | / r""" - 20 | | This is a raw string - 21 | | used as a docstring. 22 | | """ | |_______^ "#); @@ -2587,16 +2553,6 @@ with open("file.txt") as f: 7 | | """ | |_______________________^ - info[folding-range]: Folding Range - --> main.py:3:21 - | - 3 | / """Split a date range into chunks respecting the maximum days limit. - 4 | | - 5 | | The API has a 1-month limit, so this function splits larger ranges - 6 | | into smaller chunks that can be requested individually. - 7 | | """ - | |_______________________^ - info[folding-range]: Folding Range --> main.py:11:42 | @@ -2652,15 +2608,6 @@ with open("file.txt") as f: 2 | / """Docs. 3 | | 4 | | More. - 5 | | """ - | |_______^ - - info[folding-range]: Folding Range - --> main.py:2:5 - | - 2 | / """Docs. - 3 | | - 4 | | More. 5 | | """ | |_______^ "#); @@ -2692,15 +2639,6 @@ with open("file.txt") as f: 4 | / """Docs. 5 | | 6 | | More. - 7 | | """ - | |_______^ - - info[folding-range]: Folding Range - --> main.py:4:5 - | - 4 | / """Docs. - 5 | | - 6 | | More. 7 | | """ | |_______^ "#); From 8a8e86771f6f5cf9b5e47819aafc8e9d37d048f6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:52:57 +1000 Subject: [PATCH 007/110] =?UTF-8?q?give=20folding=20ranges=20basedpython's?= =?UTF-8?q?=20own=20kinds=20=E2=80=94=20docstring,=20function,=20class,=20?= =?UTF-8?q?string,=20collection=20and=20annotation,=20each=20sent=20to=20a?= =?UTF-8?q?=20client=20that=20lists=20it=20in=20its=20valueSet=20=E2=80=94?= =?UTF-8?q?=20and=20the=20text=20to=20fold=20to:=20import=20...,=20a=20reg?= =?UTF-8?q?ion's=20name,=20a=20docstring's=20first=20line=20between=20its?= =?UTF-8?q?=20quotes=20and=20a=20long=20string's=20quotes,=20and=20fold=20?= =?UTF-8?q?every=20annotation,=20a=20single-line=20one=20for=20a=20client?= =?UTF-8?q?=20that=20folds=20within=20a=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/ty_ide/src/folding_range.rs | 476 +++++++++++++----- crates/ty_server/src/capabilities.rs | 83 ++- .../src/server/api/requests/folding_range.rs | 61 ++- crates/ty_server/tests/e2e/folding_range.rs | 100 ++++ crates/ty_server/tests/e2e/main.rs | 32 ++ ...folding_range_kinds_a_client_asks_for.snap | 81 +++ ...range_kinds_a_client_does_not_ask_for.snap | 71 +++ docs/basedpython/features/editor.md | 28 ++ 8 files changed, 800 insertions(+), 132 deletions(-) create mode 100644 crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_asks_for.snap create mode 100644 crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_does_not_ask_for.snap diff --git a/crates/ty_ide/src/folding_range.rs b/crates/ty_ide/src/folding_range.rs index c2cbf4f454..7d86fd7fb0 100644 --- a/crates/ty_ide/src/folding_range.rs +++ b/crates/ty_ide/src/folding_range.rs @@ -6,11 +6,12 @@ use ruff_python_ast::visitor::source_order::{ SourceOrderVisitor, TraversalSignal, walk_body, walk_node, }; use ruff_python_ast::{ - AnyNodeRef, ExprAttribute, ExprCall, ExprGenerator, ExprRef, ExprSubscript, Stmt, + AnyNodeRef, Expr, ExprAttribute, ExprCall, ExprGenerator, ExprRef, ExprSubscript, Stmt, + StringFlags, StringLike, }; use ruff_python_trivia::{CommentLinePosition, is_python_whitespace}; use ruff_source_file::{LineRanges, UniversalNewlines}; -use ruff_text_size::{Ranged, TextRange, TextSize}; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use rustc_hash::FxHashSet; use crate::Db; @@ -19,15 +20,60 @@ const PARENTHESES: (TokenKind, TokenKind) = (TokenKind::Lpar, TokenKind::Rpar); const BRACKETS: (TokenKind, TokenKind) = (TokenKind::Lsqb, TokenKind::Rsqb); const BRACES: (TokenKind, TokenKind) = (TokenKind::Lbrace, TokenKind::Rbrace); -/// The kind of a folding range. +/// the kind of a folding range: what is folded, for an editor to decide which ranges to fold +/// from the start +/// +/// the first three are the kinds LSP itself defines. the others are basedpython's own, which a +/// client has to ask for by name; see [`FoldingRangeKind::as_lsp_custom`] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FoldingRangeKind { - /// A comment block. + /// a block of comments on lines of their own Comment, - /// An import block. + /// a block of imports Imports, - /// A region (e.g., `# region` / `# endregion`). + /// a `# region` comment, through its `# endregion` Region, + /// a docstring: a module's, a class's or a function's + Docstring, + /// the body of a function or a method, and its signature when that spans lines + Function, + /// the body of a class + Class, + /// a string literal spanning lines that is not a docstring + String, + /// the inside of a list, tuple, set or dict display, or of a comprehension + Collection, + /// a type annotation: a parameter's, a variable's, or what a function returns or raises. + /// unlike every other range, one on a single line is folded too + Annotation, +} + +impl FoldingRangeKind { + /// every one of basedpython's own kinds + pub const CUSTOM: [FoldingRangeKind; 6] = [ + FoldingRangeKind::Docstring, + FoldingRangeKind::Function, + FoldingRangeKind::Class, + FoldingRangeKind::String, + FoldingRangeKind::Collection, + FoldingRangeKind::Annotation, + ]; + + /// the name LSP knows this kind by when it is one of basedpython's own, and `None` for the + /// kinds LSP defines + pub const fn as_lsp_custom(self) -> Option<&'static str> { + match self { + FoldingRangeKind::Comment | FoldingRangeKind::Imports | FoldingRangeKind::Region => { + None + } + FoldingRangeKind::Docstring => Some("docstring"), + FoldingRangeKind::Function => Some("function"), + FoldingRangeKind::Class => Some("class"), + FoldingRangeKind::String => Some("string"), + FoldingRangeKind::Collection => Some("collection"), + FoldingRangeKind::Annotation => Some("annotation"), + } + } } /// A folding range in the source code. @@ -37,6 +83,9 @@ pub struct FoldingRange { pub range: TextRange, /// The kind of folding range. pub kind: Option, + /// what an editor shows in place of the range when it is folded, when it is something more + /// telling than an ellipsis: `import ...`, a region's name, a docstring's first line + pub collapsed_text: Option, } impl FoldingRange { @@ -46,11 +95,22 @@ impl FoldingRange { ..self } } + + fn with_collapsed_text(self, collapsed_text: impl Into) -> Self { + Self { + collapsed_text: Some(collapsed_text.into()), + ..self + } + } } impl From for FoldingRange { fn from(range: TextRange) -> FoldingRange { - FoldingRange { range, kind: None } + FoldingRange { + range, + kind: None, + collapsed_text: None, + } } } @@ -144,31 +204,42 @@ impl<'a> FoldingRangeVisitor<'a> { /// Adds a single folding range for a block. fn add_block_range(&mut self, parent: AnyNodeRef<'a>, range: TextRange) { + let line_start = self.source.line_start(range.start()); + let range = match parent { + AnyNodeRef::StmtFunctionDef(_) => { + FoldingRange::from(range).with_kind(FoldingRangeKind::Function) + } + AnyNodeRef::StmtClassDef(_) => { + FoldingRange::from(range).with_kind(FoldingRangeKind::Class) + } + _ => FoldingRange::from(range), + }; if self.add_range(range) { self.active_block_fold_start_lines - .push(ActiveBlockFoldStartLine { - parent, - line_start: self.source.line_start(range.start()), - }); + .push(ActiveBlockFoldStartLine { parent, line_start }); } } /// Adds a folding range for an expression. - fn add_expression_range(&mut self, range: TextRange) -> bool { - // Block folds preserve statement headers, so they take precedence over expression folds - // when both would render a folding trigger on the same line. - let line_start = self.source.line_start(range.start()); - if self - .active_block_fold_start_lines - .iter() - .any(|block_fold| block_fold.line_start == line_start) - { + fn add_expression_range(&mut self, range: impl Into) -> bool { + let range = range.into(); + if self.starts_on_block_fold_line(range.range) { return false; } self.add_range(range) } + /// whether `range` starts on the line a block fold starts on. block folds preserve statement + /// headers, so they take precedence over expression folds when both would render a folding + /// trigger on the same line + fn starts_on_block_fold_line(&self, range: TextRange) -> bool { + let line_start = self.source.line_start(range.start()); + self.active_block_fold_start_lines + .iter() + .any(|block_fold| block_fold.line_start == line_start) + } + /// Add a folding range for an expression that excludes enclosing delimiters like parentheses /// or square braces. fn add_delimited_expression_range( @@ -187,6 +258,19 @@ impl<'a> FoldingRangeVisitor<'a> { self.add_expression_range(range) } + /// adds a folding range for the inside of a collection display or a comprehension + fn add_collection_range(&mut self, range: TextRange, delimiters: (TokenKind, TokenKind)) { + if !self.is_multiline(range) { + return; + } + let Some(range) = self.delimited_range(range, delimiters) else { + return; + }; + self.add_expression_range( + FoldingRange::from(range).with_kind(FoldingRangeKind::Collection), + ); + } + /// Adds a folding range for an expression if it is enclosed in parentheses. fn add_parenthesized_expression_range( &mut self, @@ -249,9 +333,11 @@ impl<'a> FoldingRangeVisitor<'a> { node: AnyNodeRef<'a>, ) { if generator.parenthesized { - self.add_delimited_expression_range(node.range(), PARENTHESES); + self.add_collection_range(node.range(), PARENTHESES); } else { - self.add_expression_range(node.range()); + self.add_expression_range( + FoldingRange::from(node.range()).with_kind(FoldingRangeKind::Collection), + ); } } @@ -274,7 +360,7 @@ impl<'a> FoldingRangeVisitor<'a> { for stmt in stmts { if !self.intersects_range_filter(stmt.range()) { if let Some(range) = import_range { - self.add_range(FoldingRange::from(range).with_kind(FoldingRangeKind::Imports)); + self.add_range(Self::imports(range)); } import_range = None; prev_import_end = None; @@ -289,9 +375,7 @@ impl<'a> FoldingRangeVisitor<'a> { if has_blank_line { // Finalize the current import block and start a new one. if let Some(range) = import_range { - self.add_range( - FoldingRange::from(range).with_kind(FoldingRangeKind::Imports), - ); + self.add_range(Self::imports(range)); } import_range = Some(stmt.range()); } else if let Some(ref mut range) = import_range { @@ -302,17 +386,23 @@ impl<'a> FoldingRangeVisitor<'a> { prev_import_end = Some(stmt.end()); } else { if let Some(range) = import_range { - self.add_range(FoldingRange::from(range).with_kind(FoldingRangeKind::Imports)); + self.add_range(Self::imports(range)); } import_range = None; prev_import_end = None; } } if let Some(range) = import_range { - self.add_range(FoldingRange::from(range).with_kind(FoldingRangeKind::Imports)); + self.add_range(Self::imports(range)); } } + fn imports(range: TextRange) -> FoldingRange { + FoldingRange::from(range) + .with_kind(FoldingRangeKind::Imports) + .with_collapsed_text("import ...") + } + /// Check if there's a blank line appearing anywhere between two positions. fn has_blank_line_between(&self, start: TextSize, end: TextSize) -> bool { let mut count = 0usize; @@ -327,7 +417,7 @@ impl<'a> FoldingRangeVisitor<'a> { /// Compute folding ranges for `# region` / `# endregion` comments. fn add_custom_region_ranges(&mut self, own_line_comment_ranges: &[TextRange]) { - let mut region_starts: Vec = Vec::new(); + let mut region_starts: Vec<(TextSize, &str)> = Vec::new(); for &comment_range in own_line_comment_ranges { if !self.intersects_range_filter(comment_range) { @@ -335,14 +425,18 @@ impl<'a> FoldingRangeVisitor<'a> { } let text = &self.source[comment_range]; let trimmed = text.strip_prefix('#').unwrap_or(text).trim_start(); - if trimmed.starts_with("region") { - region_starts.push(comment_range.start()); + if let Some(name) = trimmed.strip_prefix("region") { + region_starts.push((comment_range.start(), name.trim())); } else if trimmed.starts_with("endregion") { - if let Some(start) = region_starts.pop() { - self.add_range( - FoldingRange::from(TextRange::new(start, comment_range.end())) - .with_kind(FoldingRangeKind::Region), - ); + if let Some((start, name)) = region_starts.pop() { + let region = FoldingRange::from(TextRange::new(start, comment_range.end())) + .with_kind(FoldingRangeKind::Region); + // `# region` with no name shows what any other fold shows + self.add_range(if name.is_empty() { + region + } else { + region.with_collapsed_text(name) + }); } } } @@ -421,13 +515,44 @@ impl<'a> FoldingRangeVisitor<'a> { if !is_string_like { return; } - if self - .add_range(FoldingRange::from(first_stmt.range()).with_kind(FoldingRangeKind::Comment)) - { + let mut docstring = + FoldingRange::from(first_stmt.range()).with_kind(FoldingRangeKind::Docstring); + if let Some(collapsed_text) = self.docstring_collapsed_text(&expr_stmt.value) { + docstring = docstring.with_collapsed_text(collapsed_text); + } + if self.add_range(docstring) { self.docstrings.insert(expr_stmt.value.range()); } } + /// the opening and closing quotes of a string literal, prefix included: `"""` and `"""`, + /// or `f'` and `'`. an implicitly concatenated literal opens as its first part does and closes + /// as its last part does + fn quotes(&self, string: StringLike<'a>) -> Option<(&'a str, &'a str)> { + let mut parts = string.parts(); + let first = parts.next()?; + let last = parts.next_back().unwrap_or(first); + let opener = TextRange::at(first.start(), first.flags().opener_len()); + let closer = TextRange::new(last.end() - last.flags().closer_len(), last.end()); + Some((&self.source[opener], &self.source[closer])) + } + + /// a docstring folded to its first line between its quotes: `"""Process the items...."""` + /// for a docstring that goes on after its summary + fn docstring_collapsed_text(&self, docstring: &'a Expr) -> Option { + let (opener, closer) = self.quotes(StringLike::try_from(docstring).ok()?)?; + let inside = TextRange::new( + docstring.start() + opener.text_len(), + docstring.end() - closer.text_len(), + ); + let first_line = self.source[inside] + .lines() + .map(str::trim) + .find(|line| !line.is_empty()) + .unwrap_or_default(); + Some(format!("{opener}{first_line}...{closer}")) + } + /// Attempts to find the start of the given token in the given range. fn find_token_start(&self, target: TokenKind, search_range: TextRange) -> Option { self.tokens @@ -771,18 +896,21 @@ impl<'a> SourceOrderVisitor<'a> for FoldingRangeVisitor<'a> { } // Multiline expressions - AnyNodeRef::ExprList(_) | AnyNodeRef::ExprListComp(_) | AnyNodeRef::TypeParams(_) => { + AnyNodeRef::ExprList(_) | AnyNodeRef::ExprListComp(_) => { + self.add_collection_range(node.range(), BRACKETS); + } + AnyNodeRef::TypeParams(_) => { self.add_delimited_expression_range(node.range(), BRACKETS); } // Only fold parenthesized tuples. AnyNodeRef::ExprTuple(tuple) if tuple.parenthesized => { - self.add_delimited_expression_range(node.range(), PARENTHESES); + self.add_collection_range(node.range(), PARENTHESES); } AnyNodeRef::ExprDict(_) | AnyNodeRef::ExprSet(_) | AnyNodeRef::ExprSetComp(_) | AnyNodeRef::ExprDictComp(_) => { - self.add_delimited_expression_range(node.range(), BRACES); + self.add_collection_range(node.range(), BRACES); } AnyNodeRef::ExprGenerator(generator) => { self.add_generator_expression_range(generator, node); @@ -806,7 +934,16 @@ impl<'a> SourceOrderVisitor<'a> for FoldingRangeVisitor<'a> { | AnyNodeRef::ExprFString(_) | AnyNodeRef::ExprTString(_) => { if !self.docstrings.contains(&node.range()) { - self.add_expression_range(node.range()); + let mut range = + FoldingRange::from(node.range()).with_kind(FoldingRangeKind::String); + // folded to its quotes, `"""..."""`, which say what is folded away + if let Some((opener, closer)) = StringLike::try_from(node) + .ok() + .and_then(|string| self.quotes(string)) + { + range = range.with_collapsed_text(format!("{opener}...{closer}")); + } + self.add_expression_range(range); } } @@ -831,6 +968,21 @@ impl<'a> SourceOrderVisitor<'a> for FoldingRangeVisitor<'a> { self.add_import_ranges(body); walk_body(self, body); } + + /// an annotation folds whole, on one line or several, and nothing inside it folds on its + /// own: folding a type is hiding it, not hiding part of it + fn visit_annotation(&mut self, annotation: &'a Expr) { + let range = annotation.range(); + // an annotation the parser wrote rather than the author (`init(...)`'s `-> None`) spans + // no text, and there is nothing to fold + if !range.is_empty() + && self.contains_range_filter(range) + && !self.starts_on_block_fold_line(range) + { + self.ranges + .push(FoldingRange::from(range).with_kind(FoldingRangeKind::Annotation)); + } + } } #[cfg(test)] @@ -859,7 +1011,7 @@ class A[T]: .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.by:2:12 | 2 | class A[T]: @@ -867,6 +1019,18 @@ class A[T]: 3 | | t: T 4 | | init(self, t: T) | |____________________^ + + info[folding-range]: Folding Range (annotation) + --> main.by:3:8 + | + 3 | t: T + | ^ + + info[folding-range]: Folding Range (annotation) + --> main.by:4:19 + | + 4 | init(self, t: T) + | ^ "); } @@ -888,7 +1052,7 @@ class MyClass: .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:2:15 | 2 | class MyClass: @@ -900,7 +1064,7 @@ class MyClass: 7 | | return self.value | |_________________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:3:24 | 3 | def __init__(self): @@ -908,7 +1072,7 @@ class MyClass: 4 | | self.value = 1 | |______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:6:22 | 6 | def method(self): @@ -937,7 +1101,7 @@ class MyClass: .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:2:15 | 2 | class MyClass: @@ -950,7 +1114,7 @@ class MyClass: 8 | | """ | |___________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:3:24 | 3 | def __init__(self): @@ -962,7 +1126,7 @@ class MyClass: 8 | | """ | |___________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `"""..."""` --> main.py:5:9 | 5 | / """ @@ -990,7 +1154,7 @@ def main(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:2:1 | 2 | / import os @@ -998,7 +1162,7 @@ def main(): 4 | | from typing import List, Dict | |_____________________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:6:12 | 6 | def main(): @@ -1027,14 +1191,14 @@ import requests .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:2:1 | 2 | / import os 3 | | import sys | |__________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:5:1 | 5 | / import numpy @@ -1069,14 +1233,14 @@ from fastapi import FastAPI .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:2:1 | 2 | / import os 3 | | from math import prod | |_____________________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:12:1 | 12 | / import requests @@ -1092,7 +1256,7 @@ from fastapi import FastAPI 7 | | import bar | |______________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:6:5 | 6 | / import foo @@ -1135,7 +1299,7 @@ class MyClass: .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:19 | 2 | def my_function(): @@ -1149,21 +1313,21 @@ class MyClass: 9 | | do_something() | |__________________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:3:5 | 3 | / import os 4 | | import sys | |______________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:6:5 | 6 | / import numpy 7 | | import pandas | |_________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:12:15 | 12 | class MyClass: @@ -1172,7 +1336,7 @@ class MyClass: 14 | | import collections | |______________________^ - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:13:5 | 13 | / import typing @@ -1318,7 +1482,7 @@ match value: .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:12 | 2 | def choose( @@ -1338,7 +1502,7 @@ match value: 15 | | return fallback | |_______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:5:3 | 5 | ): @@ -1375,7 +1539,7 @@ match value: 11 | | return value | |____________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:7:10 | 7 | [ @@ -1402,7 +1566,7 @@ match value: 15 | | return fallback | |_______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:17:18 | 17 | class Repository[ @@ -1416,7 +1580,7 @@ match value: 24 | | pass | |________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:23:3 | 23 | ): @@ -1502,7 +1666,7 @@ def foo(x=[ .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:12 | 2 | def foo(x=[ @@ -1514,7 +1678,7 @@ def foo(x=[ 7 | | return qux | |______________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:5:4 | 5 | ]): @@ -1525,6 +1689,40 @@ def foo(x=[ "); } + /// an annotation on the line a block fold starts on would put a second fold on the header + /// line, which the block's fold keeps, as it does against any other expression there + #[test] + fn test_folding_range_suppresses_annotation_on_block_start_line() { + let test = CursorTest::builder() + .source( + "main.py", + r#" +def area(radius: float) -> float: + scale: float = 2.0 + return radius * scale + +"#, + ) + .build(); + + assert_snapshot!(test.folding_ranges(), @" + info[folding-range]: Folding Range (function) + --> main.py:2:34 + | + 2 | def area(radius: float) -> float: + | __________________________________^ + 3 | | scale: float = 2.0 + 4 | | return radius * scale + | |_________________________^ + + info[folding-range]: Folding Range (annotation) + --> main.py:3:12 + | + 3 | scale: float = 2.0 + | ^^^^^ + "); + } + #[test] fn test_folding_range_preserves_inline_block_header_comment() { let test = CursorTest::builder() @@ -1775,7 +1973,7 @@ my_list_with_trailing_own_line_comment = [ .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:2:12 | 2 | my_list = [ @@ -1785,7 +1983,7 @@ my_list_with_trailing_own_line_comment = [ 5 | | 3, | |_______^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:8:12 | 8 | my_dict = { @@ -1794,7 +1992,7 @@ my_list_with_trailing_own_line_comment = [ 10 | | "b": 2, | |____________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:13:42 | 13 | my_list_with_trailing_element_comment = [ @@ -1804,7 +2002,7 @@ my_list_with_trailing_own_line_comment = [ 16 | | 3, # reason | |_________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:19:43 | 19 | my_list_with_trailing_own_line_comment = [ @@ -1877,7 +2075,7 @@ type Alias[ 4 | | second, | |____________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:7:11 | 7 | my_set = { @@ -1886,7 +2084,7 @@ type Alias[ 9 | | "b", | |_________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:12:13 | 12 | my_tuple = ( @@ -1895,7 +2093,7 @@ type Alias[ 14 | | second, | |____________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:17:17 | 17 | my_generator = ( @@ -1904,7 +2102,7 @@ type Alias[ 19 | | for item in items | |______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:22:17 | 22 | my_list_comp = [ @@ -1913,7 +2111,7 @@ type Alias[ 24 | | for item in items | |______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:27:16 | 27 | my_set_comp = { @@ -1922,7 +2120,7 @@ type Alias[ 29 | | for item in items | |______________________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (collection) --> main.py:32:17 | 32 | my_dict_comp = { @@ -2090,7 +2288,7 @@ multiline t-string .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `"""..."""` --> main.py:2:20 | 2 | multiline_string = """ @@ -2100,7 +2298,7 @@ multiline t-string 5 | | """ | |___^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `b"""..."""` --> main.py:7:19 | 7 | multiline_bytes = b""" @@ -2110,7 +2308,7 @@ multiline t-string 10 | | """ | |___^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `f"""..."""` --> main.py:12:21 | 12 | multiline_fstring = f""" @@ -2120,7 +2318,7 @@ multiline t-string 15 | | """ | |___^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `t"""..."""` --> main.py:17:21 | 17 | multiline_tstring = t""" @@ -2211,14 +2409,14 @@ def main(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range (imports) + info[folding-range]: Folding Range (imports), folded to `import ...` --> main.py:3:1 | 3 | / import os 4 | | import sys | |__________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:8:12 | 8 | def main(): @@ -2226,7 +2424,7 @@ def main(): 9 | | pass | |________^ - info[folding-range]: Folding Range (region) + info[folding-range]: Folding Range (region), folded to `Imports` --> main.py:2:1 | 2 | / # region Imports @@ -2235,7 +2433,7 @@ def main(): 5 | | # endregion | |___________^ - info[folding-range]: Folding Range (region) + info[folding-range]: Folding Range (region), folded to `Main` --> main.py:7:1 | 7 | / # region Main @@ -2263,7 +2461,7 @@ message = f""" .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (string), folded to `f"""..."""` --> main.py:2:11 | 2 | message = f""" @@ -2294,7 +2492,7 @@ def my_function(): .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:19 | 2 | def my_function(): @@ -2306,7 +2504,7 @@ def my_function(): 7 | | pass | |________^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `"""This is a multiline..."""` --> main.py:3:5 | 3 | / """ @@ -2352,7 +2550,7 @@ def with_rawstring_doc(): .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:24 | 2 | def with_fstring_doc(): @@ -2364,7 +2562,7 @@ def with_rawstring_doc(): 7 | | pass | |________^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `f"""This is an f-string..."""` --> main.py:3:5 | 3 | / f""" @@ -2373,7 +2571,7 @@ def with_rawstring_doc(): 6 | | """ | |_______^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:10:24 | 10 | def with_tstring_doc(): @@ -2385,7 +2583,7 @@ def with_rawstring_doc(): 15 | | pass | |________^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `t"""This is a t-string..."""` --> main.py:11:5 | 11 | / t""" @@ -2394,7 +2592,7 @@ def with_rawstring_doc(): 14 | | """ | |_______^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:18:26 | 18 | def with_rawstring_doc(): @@ -2406,7 +2604,7 @@ def with_rawstring_doc(): 23 | | pass | |________^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `r"""This is a raw string..."""` --> main.py:19:5 | 19 | / r""" @@ -2440,7 +2638,7 @@ def foo(): assert_snapshot!( test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:6:11 | 6 | def foo(): @@ -2521,7 +2719,7 @@ with open("file.txt") as f: .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:40 | 2 | def chunk_date_range(): @@ -2543,7 +2741,7 @@ with open("file.txt") as f: 17 | | c = 30 | |______________________________^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `"""Split a date range into chunks respecting the maximum days limit...."""` --> main.py:3:21 | 3 | / """Split a date range into chunks respecting the maximum days limit. @@ -2602,7 +2800,7 @@ with open("file.txt") as f: ) .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `"""Docs...."""` --> main.py:2:5 | 2 | / """Docs. @@ -2624,7 +2822,7 @@ with open("file.txt") as f: ) .build(); assert_snapshot!(test.folding_ranges(), @r#" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:1:7 | 1 | def f( @@ -2633,7 +2831,7 @@ with open("file.txt") as f: 3 | | ): | |__^ - info[folding-range]: Folding Range (comment) + info[folding-range]: Folding Range (docstring), folded to `"""Docs...."""` --> main.py:4:5 | 4 | / """Docs. @@ -2641,6 +2839,12 @@ with open("file.txt") as f: 6 | | More. 7 | | """ | |_______^ + + info[folding-range]: Folding Range (annotation) + --> main.py:2:8 + | + 2 | a: int + | ^^^ "#); } @@ -2651,7 +2855,7 @@ with open("file.txt") as f: .source("main.py", "class MyClass:\n pass\n") .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:1:15 | 1 | class MyClass: @@ -2665,7 +2869,7 @@ with open("file.txt") as f: .source("main.py", "class MyClass:\r\n pass\r\n") .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:1:15 | 1 | class MyClass: @@ -2679,7 +2883,7 @@ with open("file.txt") as f: .source("main.py", "class MyClass:\r pass\r") .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:1:15 | 1 | class MyClass: @@ -2725,7 +2929,7 @@ def my_function(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:3:19 | 3 | def my_function(): @@ -2752,7 +2956,7 @@ def my_function(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:5:19 | 5 | def my_function(): @@ -2779,7 +2983,7 @@ class MyClass: // Single decorator is one line, so no decorator folding range is emitted. assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:3:15 | 3 | class MyClass: @@ -2787,6 +2991,18 @@ class MyClass: 4 | | value: int 5 | | name: str | |_____________^ + + info[folding-range]: Folding Range (annotation) + --> main.py:4:12 + | + 4 | value: int + | ^^^ + + info[folding-range]: Folding Range (annotation) + --> main.py:5:11 + | + 5 | name: str + | ^^^ "); } @@ -2806,13 +3022,19 @@ class MyClass: .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:4:15 | 4 | class MyClass: | _______________^ 5 | | value: int | |______________^ + + info[folding-range]: Folding Range (annotation) + --> main.py:5:12 + | + 5 | value: int + | ^^^ "); } @@ -2831,7 +3053,7 @@ async def my_async_function(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:3:31 | 3 | async def my_async_function(): @@ -2857,7 +3079,7 @@ def outer_function(): .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:2:22 | 2 | def outer_function(): @@ -2867,7 +3089,7 @@ def outer_function(): 5 | | pass | |____________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:4:26 | 4 | def inner_function(): @@ -2893,7 +3115,7 @@ class MyClass: .build(); assert_snapshot!(test.folding_ranges(), @" - info[folding-range]: Folding Range + info[folding-range]: Folding Range (class) --> main.py:2:15 | 2 | class MyClass: @@ -2903,7 +3125,7 @@ class MyClass: 5 | | pass | |____________^ - info[folding-range]: Folding Range + info[folding-range]: Folding Range (function) --> main.py:4:37 | 4 | async def my_async_method(self): @@ -2929,17 +3151,29 @@ class MyClass: impl crate::tests::IntoDiagnostic for FoldingRangeDiagnostic { fn into_diagnostic(self) -> Diagnostic { - let message = match self.folding_range.kind { - Some(FoldingRangeKind::Comment) => "Folding Range (comment)", - Some(FoldingRangeKind::Imports) => "Folding Range (imports)", - Some(FoldingRangeKind::Region) => "Folding Range (region)", - None => "Folding Range", - }; + let kind = self.folding_range.kind.map(|kind| match kind { + FoldingRangeKind::Comment => "comment", + FoldingRangeKind::Imports => "imports", + FoldingRangeKind::Region => "region", + FoldingRangeKind::Docstring => "docstring", + FoldingRangeKind::Function => "function", + FoldingRangeKind::Class => "class", + FoldingRangeKind::String => "string", + FoldingRangeKind::Collection => "collection", + FoldingRangeKind::Annotation => "annotation", + }); + let mut message = "Folding Range".to_string(); + if let Some(kind) = kind { + message = format!("{message} ({kind})"); + } + if let Some(collapsed_text) = &self.folding_range.collapsed_text { + message = format!("{message}, folded to `{collapsed_text}`"); + } let mut diagnostic = Diagnostic::new( DiagnosticId::Lint(LintName::of("folding-range")), Severity::Info, - message.to_string(), + message, ); diagnostic.annotate(Annotation::primary( diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index b227cc4e8c..9d8711be70 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -15,7 +15,7 @@ bitflags::bitflags! { /// /// This tracks various capabilities that the client supports. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] - pub(crate) struct ResolvedClientCapabilities: u32 { + pub(crate) struct ResolvedClientCapabilities: u64 { const WORKSPACE_DIAGNOSTIC_REFRESH = 1 << 0; const INLAY_HINT_REFRESH = 1 << 1; const PULL_DIAGNOSTICS = 1 << 2; @@ -42,6 +42,14 @@ bitflags::bitflags! { const LANGUAGE_INJECTION = 1 << 23; const SEMANTIC_TOKENS_REFRESH = 1 << 24; const CODE_ACTION_EDIT_RESOLVE = 1 << 25; + const FOLDING_RANGE_LINE_FOLDING_ONLY = 1 << 26; + const FOLDING_RANGE_COLLAPSED_TEXT = 1 << 27; + const FOLDING_RANGE_KIND_DOCSTRING = 1 << 28; + const FOLDING_RANGE_KIND_FUNCTION = 1 << 29; + const FOLDING_RANGE_KIND_CLASS = 1 << 30; + const FOLDING_RANGE_KIND_STRING = 1 << 31; + const FOLDING_RANGE_KIND_COLLECTION = 1 << 32; + const FOLDING_RANGE_KIND_ANNOTATION = 1 << 33; } } @@ -246,6 +254,44 @@ impl ResolvedClientCapabilities { self.contains(Self::LANGUAGE_INJECTION) } + /// returns `true` if the client folds whole lines only, as + /// `textDocument.foldingRange.lineFoldingOnly` says + pub(crate) const fn folds_whole_lines_only(self) -> bool { + self.contains(Self::FOLDING_RANGE_LINE_FOLDING_ONLY) + } + + /// returns `true` if the client shows a folding range's `collapsedText` + pub(crate) const fn supports_folding_range_collapsed_text(self) -> bool { + self.contains(Self::FOLDING_RANGE_COLLAPSED_TEXT) + } + + /// returns `true` if the client listed `kind` among the folding range kinds it supports + /// + /// only basedpython's own kinds are asked about: the kinds LSP defines are sent to every + /// client, which the specification requires to handle a kind it does not know + pub(crate) const fn supports_folding_range_kind(self, kind: ty_ide::FoldingRangeKind) -> bool { + match Self::folding_range_kind_flag(kind) { + Some(flag) => self.contains(flag), + None => true, + } + } + + /// the flag a client that lists basedpython's own folding range `kind` is given. `None` for + /// a kind LSP defines, which no client lists + const fn folding_range_kind_flag(kind: ty_ide::FoldingRangeKind) -> Option { + match kind { + ty_ide::FoldingRangeKind::Comment + | ty_ide::FoldingRangeKind::Imports + | ty_ide::FoldingRangeKind::Region => None, + ty_ide::FoldingRangeKind::Docstring => Some(Self::FOLDING_RANGE_KIND_DOCSTRING), + ty_ide::FoldingRangeKind::Function => Some(Self::FOLDING_RANGE_KIND_FUNCTION), + ty_ide::FoldingRangeKind::Class => Some(Self::FOLDING_RANGE_KIND_CLASS), + ty_ide::FoldingRangeKind::String => Some(Self::FOLDING_RANGE_KIND_STRING), + ty_ide::FoldingRangeKind::Collection => Some(Self::FOLDING_RANGE_KIND_COLLECTION), + ty_ide::FoldingRangeKind::Annotation => Some(Self::FOLDING_RANGE_KIND_ANNOTATION), + } + } + pub(super) fn new(client_capabilities: &ClientCapabilities) -> Self { let mut flags = Self::empty(); @@ -483,6 +529,41 @@ impl ResolvedClientCapabilities { flags |= Self::COMPLETION_ITEM_SNIPPET_SUPPORT; } + let folding_range = + text_document.and_then(|text_document| text_document.folding_range.as_ref()); + if folding_range + .and_then(|folding_range| folding_range.line_folding_only) + .unwrap_or_default() + { + flags |= Self::FOLDING_RANGE_LINE_FOLDING_ONLY; + } + if folding_range + .and_then(|folding_range| folding_range.folding_range.as_ref()?.collapsed_text) + .unwrap_or_default() + { + flags |= Self::FOLDING_RANGE_COLLAPSED_TEXT; + } + let folding_range_kinds = folding_range + .and_then(|folding_range| { + folding_range + .folding_range_kind + .as_ref()? + .value_set + .as_ref() + }) + .map(Vec::as_slice) + .unwrap_or_default(); + for kind in ty_ide::FoldingRangeKind::CUSTOM { + if let Some(name) = kind.as_lsp_custom() + && let Some(flag) = Self::folding_range_kind_flag(kind) + && folding_range_kinds.iter().any( + |listed| matches!(listed, types::FoldingRangeKind::Custom(listed) if listed == name), + ) + { + flags |= flag; + } + } + flags } } diff --git a/crates/ty_server/src/server/api/requests/folding_range.rs b/crates/ty_server/src/server/api/requests/folding_range.rs index f9735bbc17..165950b0bc 100644 --- a/crates/ty_server/src/server/api/requests/folding_range.rs +++ b/crates/ty_server/src/server/api/requests/folding_range.rs @@ -5,6 +5,7 @@ use lsp_types::{FoldingRange, FoldingRangeKind, FoldingRangeParams, Uri}; use ty_ide::{django_template_folding_ranges, folding_ranges}; use ty_project::{ProjectDatabase, SemanticDb as _}; +use crate::capabilities::ResolvedClientCapabilities; use crate::document::{ToRangeExt, cell_range}; use crate::server::api::traits::{ BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, @@ -59,26 +60,41 @@ impl BackgroundDocumentRequestHandler for FoldingRangeRequestHandler { folding_ranges(db, db.program_file(file).python_file(db), cell_range) }; + let capabilities = snapshot.resolved_client_capabilities(); let results: Vec<_> = ranges .into_iter() .filter_map(|folding_range| { let lsp_range = folding_range .range .to_lsp_range(db, file, snapshot.encoding())?; + let range = lsp_range.local_range(); - let kind = folding_range.kind.map(|k| match k { - ty_ide::FoldingRangeKind::Comment => FoldingRangeKind::Comment, - ty_ide::FoldingRangeKind::Imports => FoldingRangeKind::Imports, - ty_ide::FoldingRangeKind::Region => FoldingRangeKind::Region, - }); + // a range on one line folds part of that line, which a client that folds whole + // lines cannot do, and which a client that does not know what the range is (an + // annotation is the only such range) has no reason to want + if range.start.line == range.end.line + && (capabilities.folds_whole_lines_only() + || !folding_range + .kind + .is_some_and(|kind| capabilities.supports_folding_range_kind(kind))) + { + return None; + } + + let kind = folding_range + .kind + .and_then(|kind| lsp_kind(kind, capabilities)); + let collapsed_text = folding_range + .collapsed_text + .filter(|_| capabilities.supports_folding_range_collapsed_text()); Some(FoldingRange { - start_line: lsp_range.local_range().start.line, - start_character: Some(lsp_range.local_range().start.character), - end_line: lsp_range.local_range().end.line, - end_character: Some(lsp_range.local_range().end.character), + start_line: range.start.line, + start_character: Some(range.start.character), + end_line: range.end.line, + end_character: Some(range.end.character), kind, - collapsed_text: None, + collapsed_text, }) }) .collect(); @@ -91,4 +107,29 @@ impl BackgroundDocumentRequestHandler for FoldingRangeRequestHandler { } } +/// the kind a client is told a range is +/// +/// the kinds LSP defines go to every client. basedpython's own go to a client that listed them +/// in `textDocument.foldingRange.foldingRangeKind.valueSet`; any other client is told a docstring +/// is a comment, as LSP has no kind closer to it, and is told nothing of the others +fn lsp_kind( + kind: ty_ide::FoldingRangeKind, + capabilities: ResolvedClientCapabilities, +) -> Option { + match kind { + ty_ide::FoldingRangeKind::Comment => Some(FoldingRangeKind::Comment), + ty_ide::FoldingRangeKind::Imports => Some(FoldingRangeKind::Imports), + ty_ide::FoldingRangeKind::Region => Some(FoldingRangeKind::Region), + custom if capabilities.supports_folding_range_kind(custom) => custom + .as_lsp_custom() + .map(|name| FoldingRangeKind::Custom(name.into())), + ty_ide::FoldingRangeKind::Docstring => Some(FoldingRangeKind::Comment), + ty_ide::FoldingRangeKind::Function + | ty_ide::FoldingRangeKind::Class + | ty_ide::FoldingRangeKind::String + | ty_ide::FoldingRangeKind::Collection + | ty_ide::FoldingRangeKind::Annotation => None, + } +} + impl RetriableRequestHandler for FoldingRangeRequestHandler {} diff --git a/crates/ty_server/tests/e2e/folding_range.rs b/crates/ty_server/tests/e2e/folding_range.rs index fa68204e0f..7a41e95968 100644 --- a/crates/ty_server/tests/e2e/folding_range.rs +++ b/crates/ty_server/tests/e2e/folding_range.rs @@ -31,6 +31,106 @@ fn folding_range_basic_functionality() -> Result<()> { Ok(()) } +/// a file with a range of every kind +const KINDS_SOURCE: &str = r#""""Module docs. + +More module docs. +""" +import os +import sys + +# region Helpers +# a block of +# comments + + +def process(items: list[int]) -> int: + """Process the items. + + Returns the total. + """ + return sum(items) + + +class Shape: + def area(self) -> float: + return 0.0 + + +# endregion + +TEXT = """ +long +""" + +NUMBERS = [ + 1, + 2, +] +"#; + +/// a client that lists basedpython's own kinds, shows `collapsedText` and folds within a line +/// is told every range's kind, what it folds to, and gets a range for every annotation, on one +/// line or not +#[test] +fn folding_range_kinds_a_client_asks_for() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + + let mut server = TestServerBuilder::new()? + .with_folding_range_capabilities( + false, + true, + &[ + "comment", + "imports", + "region", + "docstring", + "function", + "class", + "string", + "collection", + "annotation", + ], + ) + .with_workspace(workspace_root, None)? + .with_file(foo, KINDS_SOURCE)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, KINDS_SOURCE, 1); + + let ranges = server.folding_range_request(&server.file_uri(foo)); + + insta::assert_json_snapshot!(ranges); + + Ok(()) +} + +/// a client that folds whole lines, lists only the kinds LSP defines and does not show +/// `collapsedText` gets the ranges it can draw: a docstring as a comment, the others of +/// basedpython's own kinds with no kind, no range on a single line, and no `collapsedText` +#[test] +fn folding_range_kinds_a_client_does_not_ask_for() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + + let mut server = TestServerBuilder::new()? + .with_folding_range_capabilities(true, false, &["comment", "imports", "region"]) + .with_workspace(workspace_root, None)? + .with_file(foo, KINDS_SOURCE)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, KINDS_SOURCE, 1); + + let ranges = server.folding_range_request(&server.file_uri(foo)); + + insta::assert_json_snapshot!(ranges); + + Ok(()) +} + #[test] fn folding_range_multiline_block_headers() -> Result<()> { let workspace_root = SystemPath::new("src"); diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 983cfd576c..76ca637e18 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -1647,6 +1647,38 @@ impl TestServerBuilder { self } + /// set what the client says it does with folding ranges: whether it folds whole lines only, + /// whether it shows `collapsedText`, and the folding range kinds it lists as supported + pub(crate) fn with_folding_range_capabilities( + mut self, + line_folding_only: bool, + collapsed_text: bool, + kinds: &[&'static str], + ) -> Self { + self.client_capabilities + .text_document + .get_or_insert_default() + .folding_range = Some(lsp_types::FoldingRangeClientCapabilities { + line_folding_only: Some(line_folding_only), + folding_range: Some(lsp_types::ClientFoldingRangeOptions::new(Some( + collapsed_text, + ))), + folding_range_kind: Some(lsp_types::ClientFoldingRangeKindOptions::new(Some( + kinds + .iter() + .map(|kind| match *kind { + "comment" => lsp_types::FoldingRangeKind::Comment, + "imports" => lsp_types::FoldingRangeKind::Imports, + "region" => lsp_types::FoldingRangeKind::Region, + custom => lsp_types::FoldingRangeKind::Custom(custom.into()), + }) + .collect(), + ))), + ..Default::default() + }); + self + } + /// Set the hover content format preference for the client pub(crate) fn with_hover_content_format(mut self, formats: Vec) -> Self { self.client_capabilities diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_asks_for.snap b/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_asks_for.snap new file mode 100644 index 0000000000..3141affa8e --- /dev/null +++ b/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_asks_for.snap @@ -0,0 +1,81 @@ +--- +source: crates/ty_server/tests/e2e/folding_range.rs +expression: ranges +--- +[ + { + "startLine": 0, + "startCharacter": 0, + "endLine": 3, + "endCharacter": 3, + "kind": "docstring", + "collapsedText": "\"\"\"Module docs....\"\"\"" + }, + { + "startLine": 4, + "startCharacter": 0, + "endLine": 5, + "endCharacter": 10, + "kind": "imports", + "collapsedText": "import ..." + }, + { + "startLine": 12, + "startCharacter": 37, + "endLine": 17, + "endCharacter": 21, + "kind": "function" + }, + { + "startLine": 13, + "startCharacter": 4, + "endLine": 16, + "endCharacter": 7, + "kind": "docstring", + "collapsedText": "\"\"\"Process the items....\"\"\"" + }, + { + "startLine": 20, + "startCharacter": 12, + "endLine": 22, + "endCharacter": 18, + "kind": "class" + }, + { + "startLine": 21, + "startCharacter": 28, + "endLine": 22, + "endCharacter": 18, + "kind": "function" + }, + { + "startLine": 27, + "startCharacter": 7, + "endLine": 29, + "endCharacter": 3, + "kind": "string", + "collapsedText": "\"\"\"...\"\"\"" + }, + { + "startLine": 31, + "startCharacter": 11, + "endLine": 34, + "endCharacter": 0, + "kind": "collection" + }, + { + "startLine": 8, + "startCharacter": 0, + "endLine": 9, + "endCharacter": 10, + "kind": "comment" + }, + { + "startLine": 7, + "startCharacter": 0, + "endLine": 25, + "endCharacter": 11, + "kind": "region", + "collapsedText": "Helpers" + } +] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_does_not_ask_for.snap b/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_does_not_ask_for.snap new file mode 100644 index 0000000000..617018a753 --- /dev/null +++ b/crates/ty_server/tests/e2e/snapshots/e2e__folding_range__folding_range_kinds_a_client_does_not_ask_for.snap @@ -0,0 +1,71 @@ +--- +source: crates/ty_server/tests/e2e/folding_range.rs +expression: ranges +--- +[ + { + "startLine": 0, + "startCharacter": 0, + "endLine": 3, + "endCharacter": 3, + "kind": "comment" + }, + { + "startLine": 4, + "startCharacter": 0, + "endLine": 5, + "endCharacter": 10, + "kind": "imports" + }, + { + "startLine": 12, + "startCharacter": 37, + "endLine": 17, + "endCharacter": 21 + }, + { + "startLine": 13, + "startCharacter": 4, + "endLine": 16, + "endCharacter": 7, + "kind": "comment" + }, + { + "startLine": 20, + "startCharacter": 12, + "endLine": 22, + "endCharacter": 18 + }, + { + "startLine": 21, + "startCharacter": 28, + "endLine": 22, + "endCharacter": 18 + }, + { + "startLine": 27, + "startCharacter": 7, + "endLine": 29, + "endCharacter": 3 + }, + { + "startLine": 31, + "startCharacter": 11, + "endLine": 34, + "endCharacter": 0 + }, + { + "startLine": 8, + "startCharacter": 0, + "endLine": 9, + "endCharacter": 10, + "kind": "comment" + }, + { + "startLine": 7, + "startCharacter": 0, + "endLine": 25, + "endCharacter": 11, + "kind": "region" + } +] diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index f5c9e9290a..5dde859638 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -332,6 +332,34 @@ a [property](properties.md) is one member in the source, so it is one entry in the outline — not the getter, backing field and setter it lowers into. enum variants and an extension's methods appear under their declarations +## folding + +`textDocument/foldingRange` gives every range a kind saying what it folds, so +that an editor can fold some kinds from the start, and the text to show in its +place when that says more than an ellipsis. the kinds LSP defines go to every +client; the others are basedpython's own, and go to a client that lists them in +`textDocument.foldingRange.foldingRangeKind.valueSet` + +| kind | folds | folded to | +| ------------ | ------------------------------------------------------------------------- | ---------------------------- | +| `imports` | a run of imports | `import ...` | +| `region` | `# region ` through its `# endregion` | the name | +| `comment` | a run of comments on lines of their own | | +| `docstring` | a module's, class's or function's docstring | `"""Its first line..."""` | +| `function` | a function's body, and its signature when that spans lines | | +| `class` | a class's body | | +| `string` | any other string literal spanning lines | `"""..."""`, with its prefix | +| `collection` | the inside of a list, tuple, set or dict display, or of a comprehension | | +| `annotation` | a parameter's or variable's annotation, or what a `def` returns or raises | | + +a client that does not list a kind of basedpython's own is told a docstring is a +`comment`, and is told no kind for the others. an annotation is the one range +that can lie on a single line: it is sent only to a client that lists the +`annotation` kind and does not set `lineFoldingOnly`, and not when it begins on the +line a block fold starts on, where it would put a second fold on that line. the text to fold to is sent +to a client that sets `foldingRange.collapsedText`. other blocks — an `if`, a +loop, a `with`, a `try` — and the arguments of a call fold with no kind + ## refactorings the server offers refactorings as code actions. each is worked out from what the From a15d15579ddeea59692f5d3d8e3c20a3dd44c493 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:59:21 +1000 Subject: [PATCH 008/110] mark builtins defaultLibrary, every token of a type expression annotation and a dunder name dunder, and give the name a keyword argument is passed by a parameter token marked keywordArgument, and document the whole semantic token legend --- crates/ty_ide/src/semantic_tokens.rs | 200 +++++++++++++++++- .../e2e__initialize__initialization.snap | 5 +- ...ialize__initialization_with_workspace.snap | 5 +- .../e2e__notebook__semantic_tokens.snap | 18 +- ...lient_not_supporting_multiline_tokens.snap | 6 +- ...en_client_supporting_multiline_tokens.snap | 6 +- docs/basedpython/features/editor.md | 57 ++++- 7 files changed, 264 insertions(+), 33 deletions(-) diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 5efe88d4f5..761a46a176 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -51,6 +51,7 @@ use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use std::ops::Deref; use toml_parser::decoder::ScalarKind; use toml_parser::parser::{Event, EventKind}; +use ty_module_resolver::{KnownModule, file_to_module}; use ty_project::script_tag; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; @@ -160,9 +161,10 @@ bitflags! { const READONLY = 1 << 1; const ASYNC = 1 << 2; const DOCUMENTATION = 1 << 3; - /// The name is the language's own rather than the project's. Only - /// produced for django templates, where it separates a builtin tag or - /// filter from one the project registered. + /// the name is the language's own rather than the project's: in a + /// python or basedpython file, one the `builtins` module defines + /// (`print`, `len`, `str`); in a django template, a builtin tag or + /// filter rather than one the project registered const DEFAULT_LIBRARY = 1 << 4; /// The name is bound in the scope of a function, a lambda or a /// comprehension: a parameter or a local, as against a module global or @@ -170,6 +172,17 @@ bitflags! { /// highlighting — which only reads well for names that live and die with /// one call. const FUNCTION_SCOPE = 1 << 5; + /// the token is part of a type expression: an annotation — a + /// parameter's, a variable's, or what a function returns or raises — + /// a type alias's value, a type parameter's bound or default, or an + /// argument the checker reads as a type (`cast(list[int], x)`) + const ANNOTATION = 1 << 6; + /// the name is a dunder, `__name__` or `__init__`: one the language + /// gives a meaning of its own + const DUNDER = 1 << 7; + /// the name of a keyword argument in a call: `scale` in + /// `cart.total(scale=2.0)`. its token type is `parameter` + const KEYWORD_ARGUMENT = 1 << 8; } } @@ -191,6 +204,9 @@ impl SemanticTokenModifier { "documentation", "defaultLibrary", "functionScope", + "annotation", + "dunder", + "keywordArgument", ] } } @@ -776,7 +792,7 @@ impl<'db> SemanticTokenVisitor<'db> { &mut self, ranged: impl Ranged, token_type: SemanticTokenType, - modifiers: SemanticTokenModifier, + mut modifiers: SemanticTokenModifier, ) { let range = ranged.range(); @@ -784,6 +800,22 @@ impl<'db> SemanticTokenVisitor<'db> { return; } + if self.in_type_form { + modifiers |= SemanticTokenModifier::ANNOTATION; + } + if matches!( + token_type, + SemanticTokenType::Variable + | SemanticTokenType::Property + | SemanticTokenType::Function + | SemanticTokenType::Method + | SemanticTokenType::Class + | SemanticTokenType::Parameter + ) && self.source.get(range.to_std_range()).is_some_and(is_dunder) + { + modifiers |= SemanticTokenModifier::DUNDER; + } + // Only emit tokens that intersect with the range filter, if one is specified if let Some(range_filter) = self.range_filter { // Only include ranges that have a non-empty overlap. Adjacent ranges @@ -842,9 +874,20 @@ impl<'db> SemanticTokenVisitor<'db> { if self.binds_in_function_scope(name, definition) { modifiers |= SemanticTokenModifier::FUNCTION_SCOPE; } + if definition.is_some_and(|definition| self.is_builtin(definition)) { + modifiers |= SemanticTokenModifier::DEFAULT_LIBRARY; + } Some((token_type, modifiers)) } + /// whether `definition` is one of the `builtins` module's, which a name reaches without an + /// import + fn is_builtin(&self, definition: Definition<'db>) -> bool { + let db = self.model.db(); + file_to_module(db, definition.program_file(db).resolver_file(db)) + .is_some_and(|module| module.is_known(db, KnownModule::Builtins)) + } + /// Whether `name` is bound in a function's, a lambda's or a comprehension's /// scope, given the definition it resolves to. See /// [`SemanticTokenModifier::FUNCTION_SCOPE`]. @@ -1662,6 +1705,16 @@ impl<'db> SemanticTokenVisitor<'db> { } } +/// whether `name` is a dunder: `__name__`, `__init__` +fn is_dunder(name: &str) -> bool { + name.len() > 4 + && name.starts_with("__") + && name.ends_with("__") + && name[2..name.len() - 2] + .chars() + .all(|c| c.is_alphanumeric() || c == '_') +} + /// The body statements that stand for source text of their own. /// /// basedpython's `init(...)` shorthand prepends a synthesized @@ -2409,6 +2462,16 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { // based on the position. let argument_forms = call_argument_forms(self.model, call); for (argument, form) in call.arguments.iter_source_order().zip(argument_forms) { + if let ArgOrKeyword::Keyword(ast::Keyword { + arg: Some(name), .. + }) = argument + { + self.add_token( + name.range(), + SemanticTokenType::Parameter, + SemanticTokenModifier::KEYWORD_ARGUMENT, + ); + } match form { CallArgumentForm::Type => self.visit_annotation(argument.value()), CallArgumentForm::Unknown | CallArgumentForm::Value => match argument { @@ -4786,24 +4849,32 @@ x4 = assert_type(value="", type=PathLike[str]) "cast" @ 67..71: Function "x1" @ 73..75: Variable [definition] "cast" @ 78..82: Function + "typ" @ 83..86: Parameter "os" @ 87..89: Namespace "PathLike" @ 90..98: Class "str" @ 99..102: Class + "val" @ 105..108: Parameter "\"\"" @ 109..111: String "x2" @ 113..115: Variable [definition] "cast" @ 118..122: Function + "val" @ 123..126: Parameter "\"\"" @ 127..129: String + "typ" @ 131..134: Parameter "PathLike" @ 135..143: Class "str" @ 144..147: Class "x3" @ 150..152: Variable [definition] "assert_type" @ 155..166: Function + "type" @ 167..171: Parameter "os" @ 172..174: Namespace "PathLike" @ 175..183: Class "str" @ 184..187: Class + "value" @ 190..195: Parameter "\"\"" @ 196..198: String "x4" @ 200..202: Variable [definition] "assert_type" @ 205..216: Function + "value" @ 217..222: Parameter "\"\"" @ 223..225: String + "type" @ 227..231: Parameter "PathLike" @ 232..240: Class "str" @ 241..244: Class "#); @@ -6392,6 +6463,112 @@ def outer(): ); } + /// a name the `builtins` module defines is the language's own; one a module the program + /// imports is not, even one from the standard library + #[test] + fn default_library_marks_builtins() { + let test = SemanticTokenTest::new( + r#" +import os + +def count(items: list[str]) -> int: + print(os.sep) + return len(items) +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.names_with(&tokens, SemanticTokenModifier::DEFAULT_LIBRARY), + ["list@29", "str@34", "int@43", "print@52", "len@77"] + ); + } + + /// every token of a type expression is marked: an annotation, a type alias's value, and an + /// argument the checker reads as a type. a default value is not one + #[test] + fn annotation_marks_type_expressions() { + let test = SemanticTokenTest::new( + r#" +from typing import cast + +def scale(value: float = 1.0) -> list[float]: + total: float = cast(float, value) + return [total] +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.names_with(&tokens, SemanticTokenModifier::ANNOTATION), + ["float@43", "list@59", "float@64", "float@83", "float@96"] + ); + } + + /// a dunder is marked where it is defined and where it is read; a name that only starts with + /// underscores is not one + #[test] + fn dunder_marks_dunder_names() { + let test = SemanticTokenTest::new( + r#" +class Box: + def __init__(self) -> None: + self.__size = 0 + +print(__name__) +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.names_with(&tokens, SemanticTokenModifier::DUNDER), + ["__init__@20", "__name__@75"] + ); + } + + /// the name a keyword argument is passed by is a `parameter` token marked + /// `keywordArgument`, whichever argument it is among, including one the checker reads as a + /// type + #[test] + fn keyword_argument_names() { + let test = SemanticTokenTest::new( + r#" +from typing import TypeVar + +def f(*args, scale=1, **kwargs): ... + +f(1, scale=2, extra=3) +T = TypeVar("T", bound=int) +"#, + ); + + let tokens = test.highlight_file(); + let source = ruff_db::source::source_text(&test.db, test.file); + + assert_eq!( + tokens + .iter() + .filter(|token| token + .modifiers + .contains(SemanticTokenModifier::KEYWORD_ARGUMENT)) + .map(|token| format!( + "{}@{} {:?}", + &source[token.range()], + u32::from(token.start()), + token.token_type + )) + .collect::>(), + [ + "scale@72 Parameter", + "extra@81 Parameter", + "bound@107 Parameter" + ] + ); + } + #[test] fn augmented_assignment() { let test = SemanticTokenTest::new( @@ -8291,14 +8468,19 @@ x = cast(int, "") /// The tokens marked `functionScope`, each as its text and start offset. fn function_scoped_names(&self, tokens: &SemanticTokens) -> Vec { + self.names_with(tokens, SemanticTokenModifier::FUNCTION_SCOPE) + } + + /// the tokens marked with `modifier`, each as its text and start offset + fn names_with( + &self, + tokens: &SemanticTokens, + modifier: SemanticTokenModifier, + ) -> Vec { let source = ruff_db::source::source_text(&self.db, self.file); tokens .iter() - .filter(|token| { - token - .modifiers - .contains(SemanticTokenModifier::FUNCTION_SCOPE) - }) + .filter(|token| token.modifiers.contains(modifier)) .map(|token| format!("{}@{}", &source[token.range()], u32::from(token.start()))) .collect() } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 72e6f8cd14..6e273961df 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -116,7 +116,10 @@ expression: initialization_result "async", "documentation", "defaultLibrary", - "functionScope" + "functionScope", + "annotation", + "dunder", + "keywordArgument" ] }, "range": true, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 72e6f8cd14..6e273961df 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -116,7 +116,10 @@ expression: initialization_result "async", "documentation", "defaultLibrary", - "functionScope" + "functionScope", + "annotation", + "dunder", + "keywordArgument" ] }, "range": true, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap index 93ed2f4c9b..991a6d02ef 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__semantic_tokens.snap @@ -24,22 +24,22 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 8, 7, 5, - 0, + 64, 0, 8, 8, 10, - 0, + 64, 0, 10, 6, 10, - 0, + 64, 0, 8, 11, 10, - 0 + 64 ] }, { @@ -58,7 +58,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 6, 3, 1, - 0, + 80, 0, 5, 4, @@ -73,12 +73,12 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 7, 5, 1, - 0, + 64, 0, 10, 3, 1, - 0, + 80, 1, 7, 5, @@ -223,7 +223,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 6, 3, 7, - 0, + 16, 0, 4, 4, @@ -237,7 +237,7 @@ expression: "[cell1_tokens, cell2_tokens, cell3_tokens]" 0, 5, 7, - 0, + 16, 0, 6, 10, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap index 596706f8c3..766201f838 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_not_supporting_multiline_tokens.snap @@ -18,7 +18,7 @@ expression: tokens 8, 3, 1, - 0, + 80, 0, 5, 6, @@ -28,12 +28,12 @@ expression: tokens 8, 3, 1, - 0, + 80, 0, 8, 4, 1, - 0, + 80, 1, 4, 51, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap index c59148eb59..31384e5b45 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__semantic_tokens__multiline_token_client_supporting_multiline_tokens.snap @@ -18,7 +18,7 @@ expression: tokens 8, 3, 1, - 0, + 80, 0, 5, 6, @@ -28,12 +28,12 @@ expression: tokens 8, 3, 1, - 0, + 80, 0, 8, 4, 1, - 0, + 80, 1, 4, 220, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 5dde859638..b8a707601f 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -452,13 +452,56 @@ are, because their indentation is part of the string ## semantic tokens -besides the standard modifiers, a token can carry `functionScope`: the name is -bound in the scope of a function, a lambda or a comprehension — a parameter or a -local, as against a module global or a class attribute. it is the binding that -decides, not where the name is written: a global read inside a function is not -marked, a `nonlocal` name is, and an assignment expression in a comprehension at -the top of a module binds a global. an editor that colours each local by its name -("rainbow" highlighting) reads it off this modifier +the legend `initialize` answers with, in its order — a token's type and +modifiers are sent as positions in these lists. the names LSP defines are used +where one fits; the others are basedpython's own + +| token type | what it is | +| ----------------- | --------------------------------------------------------------------------------------------------------------- | +| `namespace` | a module | +| `class` | a class, or a type alias | +| `parameter` | a parameter, and the name a keyword argument is passed by | +| `selfParameter` | a method's `self` | +| `clsParameter` | a class method's `cls` | +| `variable` | a variable or an attribute | +| `property` | a property | +| `function` | a function | +| `method` | a method | +| `keyword` | a keyword spelled like a name (`var`, `let`, `typeof`), and a directive of a format spec or a `strftime` format | +| `string` | a string literal | +| `number` | a number literal | +| `decorator` | a decorator's name | +| `builtinConstant` | `None`, `True`, `False` | +| `typeParameter` | a type parameter | +| `comment` | a comment in a django template | +| `operator` | punctuation in a django template or in a script's `# /// script` metadata | +| `regexp` | a date or time in a script's `# /// script` metadata | + +| modifier | the token is | +| ----------------- | ---------------------------------------------------------------------------------- | +| `definition` | where the name is defined, rather than used | +| `readonly` | a name written as a constant, `MAX_SIZE`, or a property with no setter | +| `async` | the name of an `async def` where it is defined | +| `documentation` | a docstring (on a `string`) | +| `defaultLibrary` | a name the `builtins` module defines — `print`, `len`, `str` — or a django builtin | +| `functionScope` | a name bound in the scope of a function, a lambda or a comprehension | +| `annotation` | part of a type expression | +| `dunder` | a dunder name, `__init__` or `__name__` | +| `keywordArgument` | the name a keyword argument is passed by, `scale` in `total(scale=2.0)` | + +`functionScope` marks a parameter or a local, as against a module global or a +class attribute. it is the binding that decides, not where the name is written: +a global read inside a function is not marked, a `nonlocal` name is, and an +assignment expression in a comprehension at the top of a module binds a global. +an editor that colours each local by its name ("rainbow" highlighting) reads it +off this modifier + +`annotation` marks every token of a type expression: a parameter's or a +variable's annotation, what a function returns or raises, a type alias's value, +a type parameter's bound or default, and an argument the checker reads as a type, +as `cast(list[int], x)` does its first. a default value is not one. `dunder` +marks a dunder wherever it is defined or read; a name that only starts with +underscores is not one ## keyword highlighting From 227a8fc10e2def98fcabb75866e6989d76300733 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:12:58 +1000 Subject: [PATCH 009/110] put a pytest plugin in the tree by run stages, which moves the functions of each test module and conftest pytest compiles to the lines of the .by they were transpiled from, so a failing test or fixture is reported at its .by file and line, quoting the .by, rather than at a line of the generated python past the end of the .by --- crates/ty/src/by_commands.rs | 168 ++++++++++++++++++--- crates/ty/tests/by_e2e.rs | 82 ++++++++++ docs/basedpython/development/sourcemaps.md | 13 ++ 3 files changed, 246 insertions(+), 17 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index ea71d97a56..18ef9b59c8 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1836,9 +1836,60 @@ fn write_traceback_runtime( Path::new(BY_COVERAGE_PLUGIN_FILENAME), None, BY_COVERAGE_PLUGIN_SRC, - ) + )?; + staging.write( + Path::new(BY_PYTEST_PLUGIN_FILENAME), + None, + BY_PYTEST_PLUGIN_SRC, + )?; + for (name, text) in BY_PYTEST_DISTRIBUTION { + staging.write( + &Path::new(BY_PYTEST_DISTRIBUTION_DIR).join(name), + None, + text, + )?; + } + Ok(()) } +/// the pytest plugin that moves a test module's functions to the `.by` it was +/// transpiled from, so that a failure is reported at the `.by` line that failed +const BY_PYTEST_PLUGIN_FILENAME: &str = "_by_pytest.py"; + +const BY_PYTEST_PLUGIN_SRC: &str = r#"# generated by `by run` — reports a failing test at the `.by` line that failed, for pytest +import types + +import _by_runner + + +def pytest_pycollect_makeitem(collector): + # called for each name of a test module or class pytest collects, once the module is + # imported and before any of its tests runs + _by_runner.reline_module(collector.module) + + +def pytest_plugin_registered(plugin): + # a `conftest.py` is a plugin, and its fixtures run in the tests + if isinstance(plugin, types.ModuleType): + _by_runner.reline_module(plugin) +"#; + +/// the distribution that names `_by_pytest` as a pytest plugin. pytest loads the +/// plugins every distribution on `sys.path` declares, and the tree is at the head +/// of `sys.path` in the process `by run` starts, and in any python started in it: +/// the plugin is loaded wherever pytest can import it, and nowhere it cannot. +/// `PYTEST_DISABLE_PLUGIN_AUTOLOAD` turns it off with every other plugin, and +/// `-p _by_pytest` turns it back on +const BY_PYTEST_DISTRIBUTION_DIR: &str = "_by_pytest-0.dist-info"; + +const BY_PYTEST_DISTRIBUTION: [(&str, &str); 2] = [ + ( + "METADATA", + "Metadata-Version: 2.1\nName: by-pytest\nVersion: 0\n", + ), + ("entry_points.txt", "[pytest11]\nby = _by_pytest\n"), +]; + /// the coverage.py plugin `by run --coverage` measures with, which the runner /// names by its module name const BY_COVERAGE_PLUGIN_FILENAME: &str = "_by_coverage_plugin.py"; @@ -2203,6 +2254,94 @@ def _relined(code, filename, by_line, width): ) +def _by_lines(origin): + """The `.by` the staged `.py` at `origin` was transpiled from, as `_relined` takes + it: its path, the `.by` line of each line of the `.py` (0 for one the transpiler + wrote on its own), and the width of each `.by` line. + + `None` when there is no `.by` to answer with, or it has changed since it was + transpiled.""" + entry = _BY_MAP.get(os.path.realpath(origin)) if origin else None + if entry is None: + return None + key, (by_path, lines) = entry + if not _is_current(key, by_path): + return None + with open(by_path, "rb") as handle: + widths = [len(line.rstrip(b"\r\n")) for line in handle] + + def by_line(line): + mapped = lines[line - 1] if line is not None and 0 < line <= len(lines) else None + return 0 if mapped is None else mapped + 1 + + def width(line): + return widths[line - 1] if 0 < line <= len(widths) else 0 + + return by_path, by_line, width + + +def reline_module(module): + """Every function a staged module compiled by another loader defines, moved to the + `.by` the module was transpiled from, as `_by_code` compiles the modules no other + loader stands behind. + + The other loader is a test runner's, which compiles a test module itself to rewrite + its assertions. A failure in a test is then reported from frames of the staged `.py`, + at lines of the generated python and quoting it — `tests/test_shapes.py:129`, in a + file of 24 lines. Once its functions are relined, the report names the `.by`, at the + line that failed, and quotes the `.by`. + + Only the code a test runs is reached: the module's functions, the methods of the + classes it defines, and what a decorator wraps, each with the functions nested in it. + The module's own top-level code ran as it was imported, before this, so an error + raised while importing it still names the staged `.py`. + """ + if id(module) in _RELINED_MODULES: + return + _RELINED_MODULES.add(id(module)) + mappings = {} + seen = set() + + def mapping(origin): + if origin not in mappings: + mappings[origin] = _by_lines(origin) + return mappings[origin] + + def visit(value): + if id(value) in seen: + return + seen.add(id(value)) + if isinstance(value, types.FunctionType): + lines = mapping(value.__code__.co_filename) + if lines is not None: + value.__code__ = _relined(value.__code__, *lines) + elif isinstance(value, type): + # a class another module defines is that module's to reline + if value.__module__ == module.__name__: + for member in vars(value).values(): + visit(member) + return + elif isinstance(value, (staticmethod, classmethod)): + visit(value.__func__) + elif isinstance(value, property): + for accessor in (value.fget, value.fset, value.fdel): + if accessor is not None: + visit(accessor) + # what `functools.wraps` wraps — a pytest fixture's function among them. read from + # the object's own `__dict__`, so no `__getattr__` of the program's is run + wrapped = getattr(value, "__dict__", {}).get("__wrapped__") + if wrapped is not None: + visit(wrapped) + + for value in list(vars(module).values()): + visit(value) + + +# the modules `reline_module` has seen, by `id`: a module stays in `sys.modules`, so +# its `id` names no other module while the run lasts +_RELINED_MODULES = set() + + def _by_code(origin): """The staged `.py` at `origin` compiled as the `.by` it was transpiled from. @@ -2226,23 +2365,12 @@ def _by_code(origin): `None` when there is no `.by` to answer with, or it has changed since it was transpiled, which leaves the module to python's own loader. """ - entry = _BY_MAP.get(os.path.realpath(origin)) if origin else None - if entry is None: - return None - key, (by_path, lines) = entry - if not _is_current(key, by_path): + lines = _by_lines(origin) + if lines is None: return None + by_path, by_line, width = lines with open(origin, "rb") as handle: source = handle.read() - with open(by_path, "rb") as handle: - widths = [len(line.rstrip(b"\r\n")) for line in handle] - - def by_line(line): - mapped = lines[line - 1] if line is not None and 0 < line <= len(lines) else None - return 0 if mapped is None else mapped + 1 - - def width(line): - return widths[line - 1] if 0 < line <= len(widths) else 0 with warnings.catch_warnings(record=True) as caught: try: @@ -2282,7 +2410,9 @@ class _ByLoader: This is the loader of a module some other loader stands behind — a test runner that rewrites assertions as it compiles. It is left to that loader, and only its `__file__` moves; its frames are rewritten to `.by` lines when they reach the - excepthook. A module python's own loader would compile gets `_BySourceLoader`. + excepthook, and pytest's plugin, `_by_pytest`, moves its functions to the `.by` + with `reline_module` before a test runs. A module python's own loader would + compile gets `_BySourceLoader`. `__file__` is set as the module is created, and `exec_module` is not wrapped: the inner loader's own is the one python calls. A wrapper's frame between @@ -2549,6 +2679,9 @@ def _write_coverage(measurement, report): def main(): + # the plugins `by run` puts in the tree reach this module by name; it is running + # as `__main__`, which the program's own module takes over below + sys.modules["_by_runner"] = sys.modules[__name__] sys.excepthook = _excepthook sys.meta_path.insert(0, _ByFinder()) # `by run --coverage` names where to write what ran in front of the module, @@ -2570,7 +2703,8 @@ def main(): sys.exit(1) -main() +if __name__ == "__main__": + main() "#; // ── helpers ────────────────────────────────────────────────────────────────── diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 16bc9d3f32..926c201869 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1534,6 +1534,88 @@ fn run_coverage_maps_a_module_a_test_runner_compiled() { ); } +/// pytest compiles a test module itself, to rewrite its assertions, so a failure +/// in a test was reported from the staged `.py`, at a line of the generated python +/// and quoting it. the plugin `by run` puts in the tree moves the module's functions +/// to the `.by`, so a failure — in a test, or in a fixture of a `conftest.by` — is +/// reported at the `.by` line that failed, quoting the `.by` +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_pytest_reports_a_failure_at_the_by_line() { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let pytest = Command::new(&python) + .args(["-c", "import pytest"]) + .status() + .is_ok_and(|status| status.success()); + if !pytest { + eprintln!("skipping: {python} cannot import pytest"); + return; + } + // the conftest imports pytest, so the check has to see the environment the tests + // run in, which `--python` does not configure + let executable = Command::new(&python) + .args(["-c", "import sys; print(sys.executable)"]) + .output() + .expect("python runs"); + let executable = String::from_utf8_lossy(&executable.stdout) + .trim() + .to_owned(); + let dir = tempfile::tempdir().expect("tempdir"); + project_with_partly_covered_modules(dir.path()); + fs::write( + dir.path().join("pyproject.toml"), + format!("[tool.basedpython.environment]\npython = {executable:?}\n"), + ) + .unwrap(); + fs::create_dir_all(dir.path().join("tests")).unwrap(); + // `let` is basedpython's own, so the quoted source is the `.by`'s and not python + fs::write( + dir.path().join("tests/test_calc.by"), + "from pkg.calc import add\n\n\ndef test_sum():\n let total = add(1, 2)\n assert total == 4\n\n\nclass TestGroup:\n def test_fixture(self, broken: int):\n assert broken\n", + ) + .unwrap(); + fs::write( + dir.path().join("tests/conftest.by"), + "import pytest\n\n\n@pytest.fixture\ndef broken() -> int:\n let value = 0\n assert value == 1\n return value\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("run") + .args(["--python", &python, "--in-build"]) + .args(["pytest", "-p", "no:cacheprovider"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&format!( + "{}:6: AssertionError", + Path::new("tests").join("test_calc.by").display() + )) && stdout.contains("> assert total == 4"), + "the test's failure is not reported at its `.by` line:\n{stdout}" + ); + assert!( + stdout.contains(&format!( + "{}:7: AssertionError", + Path::new("tests").join("conftest.by").display() + )) && stdout.contains("> assert value == 1"), + "the fixture's failure is not reported at its `.by` line:\n{stdout}" + ); + // a node id still names the staged `.py`, as `tests/test_calc.py::test_sum`; a + // location names a line, as `tests/test_calc.py:129` + assert!( + !stdout + .match_indices(".py:") + .any(|(at, found)| stdout[at + found.len()..].starts_with(|c: char| c.is_ascii_digit())), + "a failure is reported at a line of the generated python:\n{stdout}" + ); +} + /// `multiprocessing`'s spawn start method — the default on macos and windows — /// reads `__main__.__spec__.name` to tell the child what to re-import, and falls /// back to running `__file__` as a *path* when there is none. `__file__` is a diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index 708375058e..aa79ddf3fc 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -118,6 +118,19 @@ that imported and the line that failed. a module some other loader stands behind — a test runner that rewrites assertions — keeps that loader, which python calls directly; only its `__file__` is set, as the module is created +pytest is that test runner, and it reports a failure from the frames it caught, +not through the excepthook, so its report used to name the staged `.py` at a line +of the generated python and quote the generated python. `by run` puts a pytest +plugin in the tree, `_by_pytest`, declared by a `pytest11` entry point in a +`_by_pytest-0.dist-info` beside it, so pytest loads it wherever the tree is on +`sys.path`. once pytest has imported a test module or a `conftest`, and before any +of its tests runs, the plugin moves the code of every function the module defines +— the methods of its classes, what a decorator wraps, a fixture's function, and +the functions nested in each — to the `.by` lines, as the shim's own loader does. +a failure in a test or a fixture is then reported at the `.by` line, quoting the +`.by`. the module's top level ran as it was imported, before the plugin saw it, so +an error raised while importing a test module still names the staged `.py` + both tables are keyed by the generated path exactly as the map spells it. the shim resolves symlinks before matching a frame's filename against it — a temporary tree under `/tmp` on macOS is reached by a different path than the one From 8403fbc6455598b2379bbd915fc40cc246b90d04 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:28:09 +1000 Subject: [PATCH 010/110] go to the constructor a class's call runs from the class's name, as on Circle in Circle(3) to the __init__ Circle writes or inherits, and to the class when it writes none, while a rename or a reference search from the name stays about the class --- crates/ty_ide/src/call_hierarchy.rs | 4 +- .../src/call_hierarchy/incoming_calls.rs | 8 +- .../src/call_hierarchy/outgoing_calls.rs | 6 +- crates/ty_ide/src/goto.rs | 75 +++++++++++--- crates/ty_ide/src/goto_declaration.rs | 10 +- crates/ty_ide/src/goto_definition.rs | 97 ++++++++++++++++++- crates/ty_ide/src/references.rs | 8 +- crates/ty_ide/src/rename.rs | 6 +- docs/basedpython/features/editor.md | 11 +++ 9 files changed, 192 insertions(+), 33 deletions(-) diff --git a/crates/ty_ide/src/call_hierarchy.rs b/crates/ty_ide/src/call_hierarchy.rs index 7dd33cad8a..50a6b0a359 100644 --- a/crates/ty_ide/src/call_hierarchy.rs +++ b/crates/ty_ide/src/call_hierarchy.rs @@ -11,7 +11,7 @@ pub(crate) mod incoming_calls; pub(crate) mod outgoing_calls; -use crate::goto::{GotoTarget, find_goto_target}; +use crate::goto::{ClassCallName, GotoTarget, find_goto_target}; use crate::{Db, SymbolKind}; use ruff_db::files::File; use ruff_db::parsed::parsed_module; @@ -42,7 +42,7 @@ pub fn prepare_call_hierarchy( let goto_target = find_goto_target(&model, &module, offset)?; let definitions = goto_target .definitions(&model, ImportAliasResolution::ResolveAliases)? - .goto_declaration(&model, &goto_target)?; + .goto_declaration(&model, &goto_target, ClassCallName::Class)?; let mut items = Vec::new(); for resolved in &definitions { diff --git a/crates/ty_ide/src/call_hierarchy/incoming_calls.rs b/crates/ty_ide/src/call_hierarchy/incoming_calls.rs index d0ae708e0c..42edd03601 100644 --- a/crates/ty_ide/src/call_hierarchy/incoming_calls.rs +++ b/crates/ty_ide/src/call_hierarchy/incoming_calls.rs @@ -1,5 +1,5 @@ use crate::call_hierarchy::{CalleeLeaf, module_detail}; -use crate::goto::{Definitions, GotoTarget, find_goto_target}; +use crate::goto::{ClassCallName, Definitions, GotoTarget, find_goto_target}; use crate::references::has_any_external_visible_definitions; use crate::{CallHierarchyItem, Db, SymbolKind}; use rayon::prelude::*; @@ -43,7 +43,9 @@ pub fn incoming_calls(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> V return Vec::new(); }; let is_externally_visible = has_any_external_visible_definitions(db, &target_definitions); - let Some(target_definitions) = target_definitions.goto_declaration(&model, &goto_target) else { + let Some(target_definitions) = + target_definitions.goto_declaration(&model, &goto_target, ClassCallName::Class) + else { return Vec::new(); }; let Some(target_text) = goto_target.to_string() else { @@ -332,7 +334,7 @@ impl<'a> CallSitesFinder<'a, '_> { let Some(current_definitions) = goto_target .definitions(self.model, ImportAliasResolution::ResolveAliases) - .and_then(|d| d.goto_declaration(self.model, &goto_target)) + .and_then(|d| d.goto_declaration(self.model, &goto_target, ClassCallName::Class)) else { return; }; diff --git a/crates/ty_ide/src/call_hierarchy/outgoing_calls.rs b/crates/ty_ide/src/call_hierarchy/outgoing_calls.rs index cd80dfa3f5..13baca4da8 100644 --- a/crates/ty_ide/src/call_hierarchy/outgoing_calls.rs +++ b/crates/ty_ide/src/call_hierarchy/outgoing_calls.rs @@ -1,7 +1,7 @@ use std::collections::hash_map::Entry; use crate::call_hierarchy::CalleeLeaf; -use crate::goto::find_goto_target; +use crate::goto::{ClassCallName, find_goto_target}; use crate::{CallHierarchyItem, Db}; use ruff_db::files::File; use ruff_db::parsed::parsed_module; @@ -38,7 +38,7 @@ pub fn outgoing_calls(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> V }; let Some(definitions) = goto_target .definitions(&model, ImportAliasResolution::ResolveAliases) - .and_then(|d| d.goto_declaration(&model, &goto_target)) + .and_then(|d| d.goto_declaration(&model, &goto_target, ClassCallName::Class)) else { return Vec::new(); }; @@ -135,7 +135,7 @@ impl<'a> OutgoingCallsFinder<'a, '_> { let Some(definitions) = goto_target .definitions(self.model, ImportAliasResolution::ResolveAliases) - .and_then(|d| d.goto_declaration(self.model, &goto_target)) + .and_then(|d| d.goto_declaration(self.model, &goto_target, ClassCallName::Class)) else { return; }; diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index 79ca7f213f..1c1ffd70be 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -8,7 +8,7 @@ use std::borrow::Cow; use crate::NavigationTarget; use crate::stub_mapping::StubMapper; -use ruff_db::parsed::ParsedModuleRef; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_python_ast::find_node::{CoveringNode, covering_node}; use ruff_python_ast::token::{Token, TokenAt, TokenKind, Tokens}; use ruff_python_ast::{self as ast, AnyNodeRef, ExprRef}; @@ -16,12 +16,12 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind}; -use ty_python_semantic::types::Type; use ty_python_semantic::types::ide_support::{ call_signature_details, call_type_simplified_by_overloads, constructor_signature, definitions_and_overloads_for_function, definitions_for_keyword_argument, typed_dict_key_definition, }; +use ty_python_semantic::types::{KnownClass, Type, TypeDefinition}; use ty_python_semantic::{Db as SemanticDb, ResolvedDefinition}; use ty_python_semantic::{ HasDefinition, HasType, ImportAliasResolution, ProgramEnvironment, SemanticModel, @@ -244,6 +244,41 @@ pub(crate) enum GotoTarget<'a> { }, } +/// where a goto on the name of a called class lands, `Circle` in `Circle(3)` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ClassCallName { + /// the class. what a rename or a search for references is about: the name + /// is the class's, whatever the call goes on to run + Class, + /// the constructor the call runs, the `__init__` or `__new__` the class or + /// one of its bases writes, as an editor's *go to declaration* takes a + /// class's call to be a call of its constructor. the class when it writes + /// none, since `object`'s says nothing about it + Constructor, +} + +/// whether `definition` is a member of `object`, the class every class inherits from: the class +/// whose body declares it is the one `object` resolves to +fn is_declared_by_object(model: &SemanticModel<'_>, definition: Definition<'_>) -> bool { + let db = model.db(); + let Some(class) = definition.scope(db).node(db).as_class() else { + return false; + }; + let env = model.program_environment(); + let Some(TypeDefinition::StaticClass(object)) = KnownClass::Object + .to_instance(db, &env) + .definition(db, &env) + else { + return false; + }; + let file = definition.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + class + .node(&module) + .definition(&SemanticModel::new(db, file)) + == object +} + /// The resolved definitions for a `GotoTarget` #[derive(Debug)] pub(crate) struct Definitions<'db>(Vec>); @@ -284,37 +319,49 @@ impl<'db> Definitions<'db> { } /// Apply the "goto declaration" interpretation to these definitions. + /// + /// `class_call` says where the name of a called class, `Circle` in `Circle(3)`, lands pub(crate) fn goto_declaration( self, model: &SemanticModel<'db>, goto_target: &GotoTarget<'_>, + class_call: ClassCallName, ) -> Option> { let mut definitions = self.0; - // When our target is a class constructor, we want to exclude - // navigation targets to its `__init__` or `__new__` methods. - // - // ... unless the cursor is on the parenthesis of the call, - // in which case we want to prefer the constructor methods - // if available. + // a class's call is both a reference to the class and a call of its + // constructor, and the definitions found for it hold both: the class, + // and the `__init__` or `__new__` the call runs // - // See: https://github.com/astral-sh/ty/issues/2218 + // on the parenthesis the constructor methods are preferred, when there + // are any. on the name it depends on `class_call` (see + // https://github.com/astral-sh/ty/issues/2218 for why a client may want + // the class there) if let GotoTarget::Call { on_parenthesis, .. } = *goto_target && goto_target .inferred_type(model) .is_some_and(|ty| ty.is_class_literal()) { + let db = model.db(); let is_constructor_method = |resolved_def: &ty_python_semantic::ResolvedDefinition<'_>| { let Some(def) = resolved_def.definition() else { return false; }; - if !matches!(*def.kind(model.db()), DefinitionKind::Function(_)) { + if !matches!(*def.kind(db), DefinitionKind::Function(_)) { return false; } - def.name(model.db()) + def.name(db) .is_some_and(|name| name == "__init__" || name == "__new__") }; + // the constructor every class has, which says nothing about this one + let is_written_constructor = + |resolved_def: &ty_python_semantic::ResolvedDefinition<'_>| { + is_constructor_method(resolved_def) + && !resolved_def + .definition() + .is_some_and(|def| is_declared_by_object(model, def)) + }; if on_parenthesis { // Only limit to constructor methods if we have at least one. @@ -324,6 +371,10 @@ impl<'db> Definitions<'db> { if definitions.iter().any(is_constructor_method) { definitions.retain(|resolved_def| is_constructor_method(resolved_def)); } + } else if class_call == ClassCallName::Constructor + && definitions.iter().any(is_written_constructor) + { + definitions.retain(|resolved_def| is_written_constructor(resolved_def)); } else { definitions.retain(|resolved_def| !is_constructor_method(resolved_def)); } @@ -363,7 +414,7 @@ impl<'db> Definitions<'db> { Some( definitions - .goto_declaration(model, goto_target)? + .goto_declaration(model, goto_target, ClassCallName::Constructor)? .map_stubs(model.db()), ) } diff --git a/crates/ty_ide/src/goto_declaration.rs b/crates/ty_ide/src/goto_declaration.rs index f20df34040..5f7655f18b 100644 --- a/crates/ty_ide/src/goto_declaration.rs +++ b/crates/ty_ide/src/goto_declaration.rs @@ -1,4 +1,4 @@ -use crate::goto::{django_lookup_definitions, find_goto_target}; +use crate::goto::{ClassCallName, django_lookup_definitions, find_goto_target}; use crate::{Db, NavigationTargets, RangedValue}; use ruff_db::files::FileRange; use ruff_db::parsed::parsed_module; @@ -26,7 +26,7 @@ pub fn goto_declaration( // basedpython: a django lookup path's leading name declares no binding, so // the field it names is the only declaration there is .or_else(|| django_lookup_definitions(&model, &module, &goto_target))? - .goto_declaration(&model, &goto_target)? + .goto_declaration(&model, &goto_target, ClassCallName::Constructor)? .into_navigation_targets(model.db()); Some(RangedValue { @@ -137,10 +137,10 @@ mod tests { 6 | instance = MyClass() | ^^^^^^^ Clicking here info: Found 1 declaration - --> main.py:2:7 + --> main.py:3:9 | - 2 | class MyClass: - | ------- + 3 | def __init__(self): + | -------- "); } diff --git a/crates/ty_ide/src/goto_definition.rs b/crates/ty_ide/src/goto_definition.rs index a1905a8967..2404999ad2 100644 --- a/crates/ty_ide/src/goto_definition.rs +++ b/crates/ty_ide/src/goto_definition.rs @@ -742,10 +742,10 @@ class MyOtherClass: 3 | x = MyClass(0) | ^^^^^^^ Clicking here info: Found 1 definition - --> mymodule.py:2:7 + --> mymodule.py:3:9 | - 2 | class MyClass: - | ------- + 3 | def __init__(self, val): + | -------- "); } @@ -2547,6 +2547,97 @@ TracebackType assert_snapshot!(test.goto_definition(), @"No goto target found"); } + /// goto-definition on the name of a called class goes to the constructor + /// the call runs, as a call of the class is a call of its constructor + #[test] + fn goto_definition_class_call_name_goes_to_init() { + let test = CursorTest::builder() + .source( + "main.py", + " +class MyClass: + def __init__(self, val): + self.val = val +x = MyClass(1) +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.py:5:5 + | + 5 | x = MyClass(1) + | ^^^^^^^ Clicking here + info: Found 1 definition + --> main.py:3:9 + | + 3 | def __init__(self, val): + | -------- + "); + } + + /// a class that writes no constructor of its own runs the one its base + /// writes + #[test] + fn goto_definition_class_call_name_goes_to_inherited_init() { + let test = CursorTest::builder() + .source( + "main.py", + " +class Base: + def __init__(self, val): + self.val = val +class Sub(Base): + pass +x = Sub(1) +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.py:7:5 + | + 7 | x = Sub(1) + | ^^^ Clicking here + info: Found 1 definition + --> main.py:3:9 + | + 3 | def __init__(self, val): + | -------- + "); + } + + /// `object`'s constructor says nothing about a class that writes none, so + /// the class is where its call's name goes + #[test] + fn goto_definition_class_call_name_without_constructor_goes_to_class() { + let test = CursorTest::builder() + .source( + "main.py", + " +class MyClass: + pass +x = MyClass() +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.py:4:5 + | + 4 | x = MyClass() + | ^^^^^^^ Clicking here + info: Found 1 definition + --> main.py:2:7 + | + 2 | class MyClass: + | ------- + "); + } + /// goto-definition on a class init opening parenthesis should go to constructor #[test] fn goto_definition_class_init_parenthesis_opening() { diff --git a/crates/ty_ide/src/references.rs b/crates/ty_ide/src/references.rs index 5aded4fedb..0f2a961948 100644 --- a/crates/ty_ide/src/references.rs +++ b/crates/ty_ide/src/references.rs @@ -10,7 +10,7 @@ //! * Members can be visible outside the module, even when assigned inside a function. //! * Parameters can also have cross-file references through keyword argument labels. -use crate::goto::{Definitions, GotoTarget}; +use crate::goto::{ClassCallName, Definitions, GotoTarget}; use crate::{Db, ReferenceKind, ReferenceTarget}; use rayon::prelude::*; use ruff_db::files::File; @@ -245,7 +245,7 @@ impl<'db> PreparedSearch<'db> { let target_definitions = goto_target .definitions(model, mode.to_import_alias_resolution())? - .goto_declaration(model, goto_target)?; + .goto_declaration(model, goto_target, ClassCallName::Class)?; let import_alias_resolution = mode.to_import_alias_resolution(); // An identifier can have both ordinary Python references and pytest fixture references. // Keep its Python definitions alongside any fixture roots used for the same search. @@ -955,7 +955,9 @@ impl<'a> LocalReferencesFinder<'a> { && self.search.target_definitions.iter().next().is_some() && goto_target .definitions(self.model, self.search.import_alias_resolution) - .and_then(|definitions| definitions.goto_declaration(self.model, &goto_target)) + .and_then(|definitions| { + definitions.goto_declaration(self.model, &goto_target, ClassCallName::Class) + }) .is_some_and(|definitions| self.search.target_definitions.intersects(&definitions)); if !fixture_match && !ordinary_match { diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index 82558f7191..c1762e2c35 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1,4 +1,4 @@ -use crate::goto::{GotoTarget, find_goto_target}; +use crate::goto::{ClassCallName, GotoTarget, find_goto_target}; use crate::references::{ReferencesMode, references}; use crate::{Db, ReferenceTarget}; use ruff_db::files::File; @@ -84,7 +84,9 @@ pub fn prepare_rename(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> P let Some(declarations) = goto_target .definitions(&model, ReferencesMode::Rename.to_import_alias_resolution()) - .and_then(|definitions| definitions.goto_declaration(&model, &goto_target)) + .and_then(|definitions| { + definitions.goto_declaration(&model, &goto_target, ClassCallName::Class) + }) else { // with no declaration there is nothing to rename, and renaming every // other use of the same spelling would be a guess diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index b8a707601f..cbcdb2ac2a 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -584,6 +584,17 @@ the caret goes in the document as the edits leave it: like the other document requests it takes `textHash`, and it is `null` when language services are off +## go to declaration + +`textDocument/definition` and `textDocument/declaration` on the name of a +called class go to the constructor the call runs: on `Circle` in `Circle(3)`, +the `__init__` (or `__new__`) that `Circle` writes, or that the base it +inherits one from writes. a call of a class is a call of its constructor, and +the constructor is where its parameters are. a class that writes none, and so +runs `object`'s, goes to the class. the class itself is always a +`textDocument/typeDefinition` away, and a rename or a search for references +from the same name is about the class + ## go to super `textDocument/implementation` on a method goes down, to the methods that From 1825f4ce45ea31f90908369bd2af3ebbca0bcd93 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:33:22 +1000 Subject: [PATCH 011/110] extend a selection through an element with its comma, a list's elements without its brackets, a statement's whole lines and a body without its header, as well as through the nodes around the caret --- crates/ty_ide/src/selection_range.rs | 343 ++++++++++++++++++++++++--- docs/basedpython/features/editor.md | 13 + 2 files changed, 329 insertions(+), 27 deletions(-) diff --git a/crates/ty_ide/src/selection_range.rs b/crates/ty_ide/src/selection_range.rs index 686ec08ad1..0df3fc5579 100644 --- a/crates/ty_ide/src/selection_range.rs +++ b/crates/ty_ide/src/selection_range.rs @@ -1,41 +1,91 @@ use ruff_db::PythonFile; use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::token::{TokenKind, Tokens}; +use ruff_python_ast::{self as ast, AnyNodeRef}; +use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::Db; +use crate::comma_list::{comma_list, tokens_within}; /// Returns a list of nested selection ranges, where each range contains the next one. /// The first range in the list is the largest range containing the cursor position. +/// +/// besides the nodes around the offset, the chain holds what a reader selects that the tree has +/// no node for: +/// +/// - an element of a comma-separated list with its comma, `"a", ` in `["a", "b"]` (the last +/// element takes the comma before it), and then the whole list between its brackets +/// - a statement as the whole lines it is written on, indentation and newline included, when +/// nothing else shares them +/// - the statements of a suite, a body without its header, as whole lines +/// +/// a range that does not contain the one before it is left out, so that the chain only ever +/// grows: the lines of a `def`'s body run on past the `def`'s own range, to the newline after its +/// last line, and the next range up is the `def` as whole lines pub fn selection_range(db: &dyn Db, file: PythonFile<'_>, offset: TextSize) -> Vec { let parsed = parsed_module(db, file).load(db); - let range = TextRange::empty(offset); + let source = source_text(db, file.file(db)); + let source = source.as_str(); + let tokens = parsed.tokens(); - let covering = covering_node(parsed.syntax().into(), range); + let covering = covering_node(parsed.syntax().into(), TextRange::empty(offset)); - let mut ranges = Vec::new(); - // Start with the largest range (the root), so iterate ancestors backwards - for node in covering.ancestors().rev() { + let mut candidates = Vec::new(); + for node in covering.ancestors() { if should_include_in_selection(node) { - let range = node.range(); - // Eliminate duplicates when parent and child nodes have the same range - if ranges.last() != Some(&range) { - ranges.push(range); - } + candidates.push(node.range()); if let Some(literal_range) = literal_content_range(node, offset) { - ranges.push(literal_range); + candidates.push(literal_range); + } + } + if let Some(list) = comma_list(node, tokens) { + if let Some(element) = list + .elements + .iter() + .position(|element| element.contains_inclusive(offset)) + .and_then(|index| element_with_comma(&list.elements, index, tokens, source)) + { + candidates.push(element); + } + if list.brackets.is_some() + && let (Some(first), Some(last)) = (list.elements.first(), list.elements.last()) + { + candidates.push(first.cover(*last)); + } + } + if node.is_statement() { + candidates.extend(whole_lines(node.range(), source)); + } + for suite in suites(node) { + if let (Some(first), Some(last)) = (suite.first(), suite.last()) + && first.range().cover(last.range()).contains_inclusive(offset) + { + candidates.extend(whole_lines(first.range().cover(last.range()), source)); } } } - ranges + // smallest first, keeping each range that contains the one kept before it + candidates.sort_by_key(|range| (range.len(), range.start())); + let mut chain: Vec = Vec::with_capacity(candidates.len()); + for range in candidates { + if chain + .last() + .is_none_or(|inner| range != *inner && range.contains_range(*inner)) + { + chain.push(range); + } + } + chain.reverse(); + chain } /// Determines if a node should be included in the selection range hierarchy. /// This filters out intermediate nodes that don't provide meaningful selections. -fn should_include_in_selection(node: ruff_python_ast::AnyNodeRef) -> bool { - use ruff_python_ast::AnyNodeRef; - +fn should_include_in_selection(node: AnyNodeRef) -> bool { // We will likely need to tune this based on user feedback. Some users may // prefer finer-grained selections while others may prefer coarser-grained. match node { @@ -47,8 +97,7 @@ fn should_include_in_selection(node: ruff_python_ast::AnyNodeRef) -> bool { } /// Determines the content range of a valid string or byte literal if it's non-empty. -fn literal_content_range(node: ruff_python_ast::AnyNodeRef, offset: TextSize) -> Option { - use ruff_python_ast::AnyNodeRef; +fn literal_content_range(node: AnyNodeRef, offset: TextSize) -> Option { let range = match node { AnyNodeRef::StringLiteral(s) if !s.flags.is_invalid() => Some(s.content_range()), AnyNodeRef::BytesLiteral(b) if !b.flags.is_invalid() => Some(b.content_range()), @@ -57,6 +106,78 @@ fn literal_content_range(node: ruff_python_ast::AnyNodeRef, offset: TextSize) -> range.filter(|&range| !range.is_empty() && range.contains_inclusive(offset)) } +/// the element at `index` of a list with the comma that separates it from the next one, and +/// the spaces after that comma, so that deleting the selection leaves the list well formed. the +/// last element, with no comma after it, takes the comma before it instead. `None` for an +/// element no comma separates from another, the only one of its list +fn element_with_comma( + elements: &[TextRange], + index: usize, + tokens: &Tokens, + source: &str, +) -> Option { + let element = *elements.get(index)?; + let next_bound = elements + .get(index + 1) + .map_or(TextSize::of(source), Ranged::start); + let after = tokens_within(tokens, TextRange::new(element.end(), next_bound)) + .iter() + .find(|token| !token.kind().is_trivia()); + if let Some(comma) = after.filter(|token| token.kind() == TokenKind::Comma) { + let spaces = source[comma.end().to_usize()..] + .bytes() + .take_while(|byte| matches!(byte, b' ' | b'\t')) + .count(); + let end = comma.end() + TextSize::try_from(spaces).ok()?; + return Some(TextRange::new(element.start(), end)); + } + let previous = elements.get(index.checked_sub(1)?)?; + let comma = tokens_within(tokens, TextRange::new(previous.end(), element.start())) + .iter() + .find(|token| token.kind() == TokenKind::Comma)?; + Some(TextRange::new(comma.start(), element.end())) +} + +/// `range` as the whole lines it is written on, from the start of its first line to past the +/// newline that ends its last, when nothing but indentation comes before it on its first line +/// and nothing but a comment after it on its last +fn whole_lines(range: TextRange, source: &str) -> Option { + let line_start = source.line_start(range.start()); + let before = &source[TextRange::new(line_start, range.start())]; + if !before.chars().all(|char| matches!(char, ' ' | '\t')) { + return None; + } + let after = source[TextRange::new(range.end(), source.line_end(range.end()))].trim_start(); + if !(after.is_empty() || after.starts_with('#')) { + return None; + } + Some(TextRange::new( + line_start, + source.full_line_end(range.end()), + )) +} + +/// the suites of a compound statement, or of the clause of one: a body without its header +fn suites(node: AnyNodeRef<'_>) -> Vec<&[ast::Stmt]> { + match node { + AnyNodeRef::StmtFunctionDef(function) => vec![&function.body[..]], + AnyNodeRef::StmtClassDef(class) => vec![&class.body[..]], + AnyNodeRef::StmtFor(for_stmt) => vec![&for_stmt.body[..], &for_stmt.orelse[..]], + AnyNodeRef::StmtWhile(while_stmt) => vec![&while_stmt.body[..], &while_stmt.orelse[..]], + AnyNodeRef::StmtIf(if_stmt) => vec![&if_stmt.body[..]], + AnyNodeRef::ElifElseClause(clause) => vec![&clause.body[..]], + AnyNodeRef::StmtWith(with) => vec![&with.body[..]], + AnyNodeRef::StmtTry(try_stmt) => vec![ + &try_stmt.body[..], + &try_stmt.orelse[..], + &try_stmt.finalbody[..], + ], + AnyNodeRef::ExceptHandlerExceptHandler(handler) => vec![&handler.body[..]], + AnyNodeRef::MatchCase(case) => vec![&case.body[..]], + _ => Vec::new(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -93,12 +214,18 @@ x = 1 + 2 | ^^^^^^^^^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | x = 1 + 2 + | ^^^^^^^^^ + + info[selection-range]: Selection Range 3 --> main.py:2:5 | 2 | x = 1 + 2 | ^^^^^ - info[selection-range]: Selection Range 3 + info[selection-range]: Selection Range 4 --> main.py:2:9 | 2 | x = 1 + 2 @@ -133,18 +260,24 @@ print(\"hello\") | ^^^^^^^^^^^^^^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | print("hello") + | ^^^^^^^^^^^^^^ + + info[selection-range]: Selection Range 3 --> main.py:2:6 | 2 | print("hello") | ^^^^^^^^^ - info[selection-range]: Selection Range 3 + info[selection-range]: Selection Range 4 --> main.py:2:7 | 2 | print("hello") | ^^^^^^^ - info[selection-range]: Selection Range 4 + info[selection-range]: Selection Range 5 --> main.py:2:8 | 2 | print("hello") @@ -177,6 +310,12 @@ r"hello" | 2 | r"hello" | ^^^^^^^^ + + info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | r"hello" + | ^^^^^^^^ "#); } @@ -229,9 +368,16 @@ def my_function(): | 2 | / def my_function(): 3 | | return 42 - | |_____________^ + | |______________^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | / def my_function(): + 3 | | return 42 + | |_____________^ + + info[selection-range]: Selection Range 3 --> main.py:2:5 | 2 | def my_function(): @@ -269,9 +415,17 @@ class MyClass: 2 | / class MyClass: 3 | | def __init__(self): 4 | | self.value = 1 - | |______________________^ + | |_______________________^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | / class MyClass: + 3 | | def __init__(self): + 4 | | self.value = 1 + | |______________________^ + + info[selection-range]: Selection Range 3 --> main.py:2:7 | 2 | class MyClass: @@ -306,36 +460,42 @@ result = [(lambda x: x[key.attr])(item) for item in data if item is not | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + info[selection-range]: Selection Range 3 --> main.py:2:10 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - info[selection-range]: Selection Range 3 + info[selection-range]: Selection Range 4 --> main.py:2:11 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - info[selection-range]: Selection Range 4 + info[selection-range]: Selection Range 5 --> main.py:2:12 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] | ^^^^^^^^^^^^^^^^^^^^^ - info[selection-range]: Selection Range 5 + info[selection-range]: Selection Range 6 --> main.py:2:22 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] | ^^^^^^^^^^^ - info[selection-range]: Selection Range 6 + info[selection-range]: Selection Range 7 --> main.py:2:24 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] | ^^^^^^^^ - info[selection-range]: Selection Range 7 + info[selection-range]: Selection Range 8 --> main.py:2:28 | 2 | result = [(lambda x: x[key.attr])(item) for item in data if item is not None] @@ -368,6 +528,12 @@ result = [(lambda x: x[key.attr])(item) for item in data if item is not | 2 | "" | ^^ + + info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | "" + | ^^ "#); } @@ -398,6 +564,12 @@ b"hello" | ^^^^^^^^ info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | b"hello" + | ^^^^^^^^ + + info[selection-range]: Selection Range 3 --> main.py:2:3 | 2 | b"hello" @@ -430,10 +602,127 @@ b"123a𝐁c" | 2 | b"123a𝐁c" | ^^^^^^^^^ + + info[selection-range]: Selection Range 2 + --> main.py:2:1 + | + 2 | b"123a𝐁c" + | ^^^^^^^^^ + "#); + } + + /// an element of a list is selected with its comma, and the elements together without the + /// brackets, before the whole list + #[test] + fn selection_range_list_element_with_comma() { + let test = CursorTest::builder() + .source( + "main.py", + r#" +specs = ["square", "circle"] +"#, + ) + .build(); + + assert_snapshot!(test.selection_texts(), @r#" + |square| + |"square"| + |"square", | + |"square", "circle"| + |["square", "circle"]| + |specs = ["square", "circle"]| + |specs = ["square", "circle"]⏎| + |⏎specs = ["square", "circle"]⏎| "#); } + /// the last element has no comma after it, and takes the one before it + #[test] + fn selection_range_last_element_takes_the_comma_before_it() { + let test = CursorTest::builder() + .source( + "main.py", + r#" +print(width, height) +"#, + ) + .build(); + + assert_snapshot!(test.selection_texts(), @" + |height| + |, height| + |width, height| + |(width, height)| + |print(width, height)| + |print(width, height)⏎| + |⏎print(width, height)⏎| + "); + } + + /// a statement is selected as its whole line, and a body as its lines without the header, + /// before the decorated method as whole lines + #[test] + fn selection_range_statement_line_and_suite() { + let test = CursorTest::builder() + .source( + "main.py", + r#" +class Shape: + @property + def area(self) -> int: + size = 2 + return size * size # squared +"#, + ) + .build(); + + assert_snapshot!(test.selection_texts(), @" + |size| + |size * size| + |return size * size| + | return size * size # squared⏎| + | size = 2⏎ return size * size # squared⏎| + | @property⏎ def area(self) -> int:⏎ size = 2⏎ return size * size # squared⏎| + |class Shape:⏎ @property⏎ def area(self) -> int:⏎ size = 2⏎ return size * size # squared⏎| + |⏎class Shape:⏎ @property⏎ def area(self) -> int:⏎ size = 2⏎ return size * size # squared⏎| + "); + } + + /// a statement that shares its line with another is not selected as the line + #[test] + fn selection_range_statement_sharing_its_line() { + let test = CursorTest::builder() + .source( + "main.py", + r#" +if ready: start() +"#, + ) + .build(); + + assert_snapshot!(test.selection_texts(), @" + |start| + |start()| + |if ready: start()| + |if ready: start()⏎| + |⏎if ready: start()⏎| + "); + } + impl CursorTest { + /// each range of the chain as the text it selects, innermost first, between `|`s and with + /// each newline written `⏎` + fn selection_texts(&self) -> String { + let python_file = self.program_file(self.cursor.file).python_file(&self.db); + let source = ruff_db::source::source_text(&self.db, self.cursor.file); + selection_range(&self.db, python_file, self.cursor.offset) + .iter() + .rev() + .map(|&range| format!("|{}|", source.as_str()[range].replace('\n', "⏎"))) + .collect::>() + .join("\n") + } + fn selection_range(&self) -> String { let ranges = selection_range( &self.db, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index cbcdb2ac2a..3c0c87fc70 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -584,6 +584,19 @@ the caret goes in the document as the edits leave it: like the other document requests it takes `textHash`, and it is `null` when language services are off +## extend selection + +`textDocument/selectionRange` grows a selection the way a reader reads the +code, not only node by node. on `"a"` in `xs = ["a", "b"]` the ranges are the +string's text, the string, `"a",` with its comma and the space after it, `"a", "b"` without the +brackets, the list, the assignment, and then the assignment's whole line. a +last element takes the comma before it, `, "b"`. a statement is offered as the +whole lines it is written on, indentation and newline included, when it shares +them with nothing but a comment, and the statements of a body are offered +together without their header, so a selection inside a method grows to the +method's body, then to the method with its decorators as whole lines, then to +the class + ## go to declaration `textDocument/definition` and `textDocument/declaration` on the name of a From 2d5d0d64a55f16b562e4066ae7b529577668bed8 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:46:57 +1000 Subject: [PATCH 012/110] open a hover with the hovered name's kind and qualified name, method shapes.base.Circle.area, and show a decorated def as its source writes it, its decorators over its own signature and docstring, rather than as the wrapper its decorator returns --- crates/ty_ide/src/declaration.rs | 192 ++++ crates/ty_ide/src/hover.rs | 994 +++++++++++++++++- crates/ty_ide/src/lib.rs | 2 + ...tests__hover_shadowed_numeric_builtin.snap | 4 + ...numeric_builtin_in_selected_signature.snap | 4 + ...ts__hover_subscript_non_literal_index.snap | 4 + .../src/types/ide_support.rs | 29 + .../ty_server/tests/e2e/closed_documents.rs | 46 +- crates/ty_server/tests/e2e/configuration.rs | 2 +- crates/ty_server/tests/e2e/hover.rs | 8 +- ...otebook__cells_after_cell_magic_hover.snap | 2 +- .../ty_server/tests/e2e/workspace_folders.rs | 27 +- docs/basedpython/features/editor.md | 23 + 13 files changed, 1302 insertions(+), 35 deletions(-) create mode 100644 crates/ty_ide/src/declaration.rs diff --git a/crates/ty_ide/src/declaration.rs b/crates/ty_ide/src/declaration.rs new file mode 100644 index 0000000000..8a45942a5e --- /dev/null +++ b/crates/ty_ide/src/declaration.rs @@ -0,0 +1,192 @@ +//! What a declaration is: its kind, its name, and the dotted name it is reached by from outside +//! its module. +//! +//! ```py +//! # shapes/base.py +//! class Circle: +//! def __init__(self, radius: float) -> None: +//! self.radius = radius +//! +//! def area(self) -> float: ... +//! ``` +//! +//! `Circle` is the class `shapes.base.Circle`, `area` the method `shapes.base.Circle.area`, and +//! `radius` the attribute `shapes.base.Circle.radius`, although it is assigned in `__init__`. A +//! name local to a function, a parameter among them, has no qualified name, and nor does a type +//! parameter: nothing outside the function or the generic reaches it. + +use ruff_db::parsed::parsed_module; +use ty_module_resolver::file_to_module; +use ty_python_core::definition::{Definition, DefinitionKind}; +use ty_python_core::place::PlaceExprRef; +use ty_python_core::scope::{FileScopeId, ScopeKind}; +use ty_python_core::{ProgramFile, semantic_index}; +use ty_python_semantic::ResolvedDefinition; +use ty_python_semantic::types::ide_support::decorated_function; + +use crate::Db; + +/// What kind of thing a declaration declares. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeclaredKind { + Module, + Class, + Function, + /// A `def` in a class body. + Method, + /// A `def` in a class body that its decorators make a property. + Property, + /// A name a class body binds that is not a `def` or a class, or an attribute a method assigns + /// on `self`. + Attribute, + Variable, + Parameter, + TypeParameter, + TypeAlias, +} + +impl DeclaredKind { + /// The kind as a reader names it. + pub const fn as_str(self) -> &'static str { + match self { + DeclaredKind::Module => "module", + DeclaredKind::Class => "class", + DeclaredKind::Function => "function", + DeclaredKind::Method => "method", + DeclaredKind::Property => "property", + DeclaredKind::Attribute => "attribute", + DeclaredKind::Variable => "variable", + DeclaredKind::Parameter => "parameter", + DeclaredKind::TypeParameter => "type parameter", + DeclaredKind::TypeAlias => "type alias", + } + } +} + +/// A declaration, described. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Declaration { + pub kind: DeclaredKind, + /// The declared name. + pub name: String, + /// The module's dotted name, then each class the declaration is made in, then the name: + /// `shapes.base.Circle.area`. `None` for a name local to a function, and for a declaration + /// in a file no module is imported from. + pub qualified_name: Option, +} + +/// What `resolved` declares. `None` for what is not a declaration of a name: an import that was +/// not followed to what it imports, or a range in a file that is not a definition. +pub(crate) fn declaration(db: &dyn Db, resolved: &ResolvedDefinition<'_>) -> Option { + match resolved { + ResolvedDefinition::Module(file) => { + let qualified_name = module_name(db, *file)?; + Some(Declaration { + kind: DeclaredKind::Module, + name: qualified_name + .rsplit('.') + .next() + .unwrap_or(&qualified_name) + .to_string(), + qualified_name: Some(qualified_name), + }) + } + ResolvedDefinition::Definition(definition) => definition_declaration(db, *definition), + ResolvedDefinition::FileWithRange(_) => None, + } +} + +fn definition_declaration(db: &dyn Db, definition: Definition<'_>) -> Option { + let file = definition.program_file(db); + let index = semantic_index(db, file); + let scope = definition.file_scope(db); + let places = index.place_table(scope); + let (name, on_self) = match places.place(definition.place(db)) { + PlaceExprRef::Symbol(symbol) => (symbol.name().to_string(), false), + // `self.radius = radius` in a method declares the class's attribute; an assignment to + // any other object's attribute declares nothing of this scope's + PlaceExprRef::Member(member) => (member.as_instance_attribute()?.to_string(), true), + }; + let in_class = on_self || index.scope(scope).kind() == ScopeKind::Class; + + let kind = match definition.kind(db) { + DefinitionKind::Import(_) + | DefinitionKind::ImportFrom(_) + | DefinitionKind::ImportFromSubmodule(_) + | DefinitionKind::StarImport(_) => return None, + DefinitionKind::Class(_) => DeclaredKind::Class, + DefinitionKind::Function(_) if !in_class => DeclaredKind::Function, + DefinitionKind::Function(_) => { + if decorated_function(db, definition).is_some_and(|function| function.is_property()) { + DeclaredKind::Property + } else { + DeclaredKind::Method + } + } + DefinitionKind::TypeAlias(_) => DeclaredKind::TypeAlias, + DefinitionKind::Parameter(_) | DefinitionKind::LambdaParameter(_) => { + DeclaredKind::Parameter + } + DefinitionKind::TypeVar(_) + | DefinitionKind::ParamSpec(_) + | DefinitionKind::TypeVarTuple(_) => DeclaredKind::TypeParameter, + _ if in_class => DeclaredKind::Attribute, + _ => DeclaredKind::Variable, + }; + + // the scope the declared name lives in: a class's, for an attribute a method assigns + let owner = if on_self { + let class = index.class_definition_of_method(scope)?; + Some((class.file_scope(db), class.name(db)?)) + } else { + None + }; + // a parameter, or a type parameter, is reached only through what it parameterizes + let qualified_name = if matches!(kind, DeclaredKind::Parameter | DeclaredKind::TypeParameter) { + None + } else { + let (start, owner_name) = owner.map_or((scope, None), |(scope, name)| (scope, Some(name))); + containers(db, file, start).and_then(|mut containers| { + containers.extend(owner_name); + containers.push(name.clone()); + let module = module_name(db, file)?; + Some(format!("{module}.{}", containers.join("."))) + }) + }; + + Some(Declaration { + kind, + name, + qualified_name, + }) +} + +/// The names of the classes `scope` is nested in, outermost first, `scope` itself included when +/// it is a class's. `None` when a function encloses it, which makes what it declares local. +fn containers(db: &dyn Db, file: ProgramFile<'_>, scope: FileScopeId) -> Option> { + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + let mut containers = Vec::new(); + for (id, ancestor) in index.ancestor_scopes(scope) { + match ancestor.kind() { + ScopeKind::Module => break, + ScopeKind::Class => { + containers.push(id.to_scope_id(db, file).name(db, &module).to_string()); + } + // the scope of a generic class's or function's type parameters sits between it and + // what encloses it, and adds no name + ScopeKind::TypeParams => {} + ScopeKind::Function + | ScopeKind::Lambda + | ScopeKind::Comprehension + | ScopeKind::TypeAlias => return None, + } + } + containers.reverse(); + Some(containers) +} + +/// The dotted name `file` is imported by. +fn module_name(db: &dyn Db, file: ProgramFile<'_>) -> Option { + file_to_module(db, file.resolver_file(db)).map(|module| module.name(db).to_string()) +} diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index a1493b8f2e..245200b7ab 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -1,4 +1,5 @@ use crate::annotation_expression::is_in_annotation_expression; +use crate::declaration::{Declaration, DeclaredKind, declaration}; use crate::docstring::{Docstring, DocstringFragment}; use crate::goto::{ Definitions, GotoTarget, docstring_for_call_definition, find_goto_target, find_goto_target_impl, @@ -15,14 +16,19 @@ use ruff_python_literal::strftime; use ruff_text_size::{Ranged, TextRange, TextSize}; use std::fmt::{self, Display, Formatter}; use ty_python_core::ProgramFile; -use ty_python_semantic::types::ide_support::{resolved_call_signature, typed_dict_key_hover}; +use ty_python_core::definition::DefinitionKind; +use ty_python_semantic::types::ide_support::{ + decorated_function, resolved_call_signature, typed_dict_key_hover, +}; use ty_python_semantic::types::{ BoundTypeVarInstance, KnownInstanceType, SpecialFormType, Type, TypeVarVariance, }; use ty_python_semantic::{DisplaySettings, ProgramEnvironment}; use ty_python_semantic::types::format::{SpecLanguage, spec_language}; -use ty_python_semantic::{HasType, SemanticModel, TypeQualifiers, with_display_for_file}; +use ty_python_semantic::{ + HasDefinition, HasType, SemanticModel, TypeQualifiers, with_display_for_file, +}; pub fn hover<'db>( db: &'db dyn Db, @@ -107,7 +113,37 @@ fn hover_inner<'db>( let mut alias_docstring: Option = None; - let docs = if keyword_argument.is_some() || typed_dict_key.is_some() { + // what the hovered name declares, read off the name's own definitions rather than off what a + // call of it runs: `Circle` in `Circle(3)` is the class, whose `__init__` the call runs + let declared = match &goto_target { + GotoTarget::Expression(_) | GotoTarget::Call { .. } => goto_target.expression_definitions( + &model, + ty_python_semantic::ImportAliasResolution::ResolveAliases, + ), + // the `def` itself, which a decorator that returns a function of its own would otherwise + // stand in for + GotoTarget::FunctionDef(function) => Some(Definitions::new(vec![ + ty_python_semantic::ResolvedDefinition::Definition(function.definition(&model)), + ])), + _ => goto_target.definitions( + &model, + ty_python_semantic::ImportAliasResolution::ResolveAliases, + ), + }; + let declaration = declared + .as_ref() + .and_then(|declared| single_declaration(db, declared)); + let decorated = if keyword_argument.is_some() || typed_dict_key.is_some() { + None + } else { + declared.as_ref().and_then(|declared| { + decorated_function_hover(db, &goto_target, declared, declaration.as_ref()) + }) + }; + + let docs = if let Some((_, docstring)) = &decorated { + docstring.clone().map(HoverContent::Docstring) + } else if keyword_argument.is_some() || typed_dict_key.is_some() { None } else if let GotoTarget::Call { call, .. } = goto_target { resolved_call_signature(&model, call) @@ -136,8 +172,16 @@ fn hover_inner<'db>( }; let mut contents = Vec::new(); + if keyword_argument.is_none() + && typed_dict_key.is_none() + && let Some(declaration) = declaration + { + contents.push(HoverContent::Declaration(declaration)); + } if let Some(keyword_argument) = keyword_argument { contents.extend(keyword_argument); + } else if let Some((function, _)) = decorated { + contents.push(function); } else if let Some(signature) = goto_target.call_signature(&model) { contents.push(HoverContent::Signature(signature)); } else if let Some(typed_dict_key) = typed_dict_key { @@ -291,6 +335,67 @@ fn annotation_operator( } } +/// The one declaration `declared` makes, when its definitions all declare the same name the same +/// way: the bindings of a variable assigned twice are one variable. +fn single_declaration(db: &dyn Db, declared: &Definitions<'_>) -> Option { + let mut declarations = declared.iter().map(|resolved| declaration(db, resolved)); + let first = declarations.next()??; + declarations + .all(|other| other.as_ref() == Some(&first)) + .then_some(first) +} + +/// A decorated `def` as its source writes it, its decorators over its own signature, and its own +/// docstring: what the name's type would show instead is whatever its decorators return, a +/// `wrapper(*args, **kwargs)` or a `_lru_cache_wrapper[...]`, documented as the wrapper is. +/// +/// `None` for a `def` without decorators, whose type is the function it writes, and for a use of +/// a property, which reads the property's value rather than naming the `def`. +fn decorated_function_hover<'db>( + db: &'db dyn Db, + goto_target: &GotoTarget<'_>, + declared: &Definitions<'db>, + declaration: Option<&Declaration>, +) -> Option<(HoverContent<'db>, Option)> { + let mut definitions = declared.iter(); + let (Some(ty_python_semantic::ResolvedDefinition::Definition(definition)), None) = + (definitions.next(), definitions.next()) + else { + return None; + }; + let DefinitionKind::Function(function) = definition.kind(db) else { + return None; + }; + if !matches!(goto_target, GotoTarget::FunctionDef(_)) + && declaration.is_some_and(|declaration| declaration.kind == DeclaredKind::Property) + { + return None; + } + let module = parsed_module(db, definition.python_file(db)).load(db); + let source = source_text(db, definition.file(db)); + let decorators: Vec = function + .node(&module) + .decorator_list + .iter() + .map(|decorator| format!("@{}", &source[decorator.expression.range()])) + .collect(); + if decorators.is_empty() { + return None; + } + let function = decorated_function(db, *definition)?.undecorated; + let docstring = Definitions::new(vec![ty_python_semantic::ResolvedDefinition::Definition( + *definition, + )]) + .docstring(db); + Some(( + HoverContent::DecoratedFunction { + decorators, + function, + }, + docstring, + )) +} + fn keyword_argument_hover_contents<'db>( db: &'db dyn Db, model: &SemanticModel<'db>, @@ -540,6 +645,13 @@ impl<'a, 'db> IntoIterator for &'a Hover<'db> { #[derive(Debug, Clone)] pub enum HoverContent<'db> { + /// What the hovered name declares: its kind, and its qualified name when it has one. + Declaration(Declaration), + /// A decorated `def`: its decorators, as written, over the function it writes. + DecoratedFunction { + decorators: Vec, + function: Type<'db>, + }, Signature(String), Parameter(String), Type { @@ -628,6 +740,27 @@ fn create_qualifier_suffix(qualifiers: TypeQualifiers) -> String { impl fmt::Display for DisplayHoverContent<'_, '_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.content { + HoverContent::Declaration(declaration) => { + let name = declaration + .qualified_name + .as_deref() + .unwrap_or(&declaration.name); + let kind = declaration.kind.as_str(); + match self.kind { + MarkupKind::PlainText => write!(f, "{kind} {name}"), + MarkupKind::Markdown => write!(f, "{kind} `{name}`"), + } + } + HoverContent::DecoratedFunction { + decorators, + function, + } => { + let (signature, _) = self.ty_string_and_syntax(function); + let mut written = decorators.join("\n"); + written.push('\n'); + written.push_str(&signature); + self.kind.fenced_code_block(written, "python").fmt(f) + } HoverContent::Signature(signature) => { self.kind.fenced_code_block(&signature, "python").fmt(f) } @@ -932,8 +1065,12 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + variable main.width + --------------------------------------------- Literal[12] --------------------------------------------- + variable `main.width` + --- ```python Literal[12] ``` @@ -964,6 +1101,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + variable main.a + --------------------------------------------- Literal[10] --------------------------------------------- This is the docs for this value @@ -971,6 +1110,8 @@ mod tests { Wow these are good docs! --------------------------------------------- + variable `main.a` + --- ```python Literal[10] ``` @@ -1014,7 +1155,12 @@ mod tests { let code = "name\n```\n![image](https://example.invalid/)\n```"; assert_eq!( rendered, - Some((code.to_string(), format!("````python\n{code}\n````"))) + Some(( + format!( + "variable main.value\n---------------------------------------------\n{code}" + ), + format!("variable `main.value`\n---\n````python\n{code}\n````") + )) ); } @@ -1036,6 +1182,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.my_func + --------------------------------------------- def my_func( a, b @@ -1048,6 +1196,8 @@ mod tests { b: coming for `a`'s title --------------------------------------------- + function `main.my_func` + --- ```python def my_func( a, @@ -1091,6 +1241,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.my_func + --------------------------------------------- def my_func( a, b @@ -1103,6 +1255,8 @@ mod tests { b: coming for `a`'s title --------------------------------------------- + function `main.my_func` + --- ```python def my_func( a, @@ -1147,6 +1301,8 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + function main.documented + --------------------------------------------- def documented(value) -> value --------------------------------------------- Return a value. @@ -1155,6 +1311,8 @@ mod tests { :rtype: str --------------------------------------------- + function `main.documented` + --- ```python def documented(value) -> value ``` @@ -1210,6 +1368,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.MyClass + --------------------------------------------- --------------------------------------------- This is such a great class!! @@ -1219,6 +1379,8 @@ mod tests { Everyone loves my class!! --------------------------------------------- + class `main.MyClass` + --- ```xml ``` @@ -1269,6 +1431,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.MyClass + --------------------------------------------- --------------------------------------------- This is such a great class!! @@ -1278,6 +1442,8 @@ mod tests { Everyone loves my class!! --------------------------------------------- + class `main.MyClass` + --- ```xml ``` @@ -1330,11 +1496,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.MyClass + --------------------------------------------- class MyClass(val) --------------------------------------------- initializes MyClass (perfectly) --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass(val) ``` @@ -1383,11 +1553,15 @@ mod tests { .build(); assert_snapshot!(test.hover(), @" + class mymod.MyClass + --------------------------------------------- class MyClass(val) --------------------------------------------- initializes MyClass (perfectly) --------------------------------------------- + class `mymod.MyClass` + --- ```python class MyClass(val) ``` @@ -1435,6 +1609,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.MyClass + --------------------------------------------- class MyClass(val) --------------------------------------------- This is such a great class!! @@ -1444,6 +1620,8 @@ mod tests { Everyone loves my class!! --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass(val) ``` @@ -1479,11 +1657,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- class MyClass( a: int, b: str ) --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass( a: int, @@ -1521,6 +1703,8 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- class MyClass( a: int, b: str @@ -1529,6 +1713,8 @@ mod tests { MyClass docs --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass( a: int, @@ -1561,8 +1747,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.MyClass + --------------------------------------------- class MyClass() --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass() ``` @@ -1592,11 +1782,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- class MyClass( a: int, b: str ) --------------------------------------------- + class `main.MyClass` + --- ```python class MyClass( a: int, @@ -1638,12 +1832,16 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.Shape + --------------------------------------------- class Shape(val: str) class Shape(val: int) --------------------------------------------- Shape docs --------------------------------------------- + class `main.Shape` + --- ```python class Shape(val: str) class Shape(val: int) @@ -1685,11 +1883,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.Shape + --------------------------------------------- class Shape(val: str) --------------------------------------------- Shape docs --------------------------------------------- + class `main.Shape` + --- ```python class Shape(val: str) ``` @@ -1726,6 +1928,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.S + --------------------------------------------- class S( a: int, b: str @@ -1735,6 +1939,8 @@ mod tests { init docs --------------------------------------------- + class `main.S` + --- ```python class S( a: int, @@ -1775,11 +1981,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.S + --------------------------------------------- class S(a: int) --------------------------------------------- new docs --------------------------------------------- + class `main.S` + --- ```python class S(a: int) ``` @@ -1812,8 +2022,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.Handler + --------------------------------------------- class Handler(callback: (int, str, /) -> bool) --------------------------------------------- + class `main.Handler` + --- ```python class Handler(callback: (int, str, /) -> bool) ``` @@ -1844,8 +2058,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.Color + --------------------------------------------- class Color(value: object) --------------------------------------------- + class `main.Color` + --- ```python class Color(value: object) ``` @@ -1876,12 +2094,16 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.Movie + --------------------------------------------- class Movie( *, title: str, year: int ) --------------------------------------------- + class `main.Movie` + --- ```python class Movie( *, @@ -1917,6 +2139,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.Movie + --------------------------------------------- class Movie( map: Movie, /, @@ -1925,6 +2149,8 @@ mod tests { year: int = ... ) --------------------------------------------- + class `main.Movie` + --- ```python class Movie( map: Movie, @@ -1961,6 +2187,8 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.Movie + --------------------------------------------- class Movie( map: Movie, /, @@ -1969,6 +2197,8 @@ mod tests { year: int = ... ) --------------------------------------------- + class `main.Movie` + --- ```python class Movie( map: Movie, @@ -2005,12 +2235,16 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.Movie + --------------------------------------------- class Movie( *, title: str, year: int = ... ) --------------------------------------------- + class `main.Movie` + --- ```python class Movie( *, @@ -2045,12 +2279,16 @@ mod tests { ); assert_snapshot!(test.hover(), @" + class main.Movie + --------------------------------------------- class Movie( *, title: str = ..., year: int = ... ) --------------------------------------------- + class `main.Movie` + --- ```python class Movie( *, @@ -2102,6 +2340,8 @@ mod tests { ); assert_snapshot!(test.hover(), @" + method main.MyClass.my_method + --------------------------------------------- bound method MyClass.my_method( a, b @@ -2114,6 +2354,8 @@ mod tests { b: coming for `a`'s title --------------------------------------------- + method `main.MyClass.my_method` + --- ```python bound method MyClass.my_method( a, @@ -2162,11 +2404,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + method main.MyTestClass.foo + --------------------------------------------- def foo(x: int) -> int --------------------------------------------- Sample docstring --------------------------------------------- + method `main.MyTestClass.foo` + --- ```python def foo(x: int) -> int ``` @@ -2194,6 +2440,8 @@ mod tests { ); assert_snapshot!(string.hover(), @r#" + method builtins.str.removesuffix + --------------------------------------------- bound method str.removesuffix(suffix: str, /) -> str --------------------------------------------- Return a str with the given suffix string removed if present. @@ -2203,6 +2451,8 @@ mod tests { the original string. --------------------------------------------- + method `builtins.str.removesuffix` + --- ```python bound method str.removesuffix(suffix: str, /) -> str ``` @@ -2233,6 +2483,8 @@ mod tests { ); assert_snapshot!(literal_string.hover(), @r#" + method builtins.str.removesuffix + --------------------------------------------- def removesuffix(suffix: LiteralString, /) -> LiteralString --------------------------------------------- Return a str with the given suffix string removed if present. @@ -2242,6 +2494,8 @@ mod tests { the original string. --------------------------------------------- + method `builtins.str.removesuffix` + --- ```python def removesuffix(suffix: LiteralString, /) -> LiteralString ``` @@ -2286,11 +2540,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- def test() -> str --------------------------------------------- A second overload --------------------------------------------- + function `main.test` + --- ```python def test() -> str ``` @@ -2325,13 +2583,19 @@ mod tests { ); assert_snapshot!(test.hover(), @" - Literal[42] + function main.foo + --------------------------------------------- + @decorator + def foo() --------------------------------------------- Foo documentation --------------------------------------------- + function `main.foo` + --- ```python - Literal[42] + @decorator + def foo() ``` --- Foo documentation @@ -2373,11 +2637,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- def test() --------------------------------------------- Implementation docstring --------------------------------------------- + function `main.test` + --- ```python def test() ``` @@ -2429,11 +2697,15 @@ mod tests { .build(); assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- def test() --------------------------------------------- Version 3.10+ implementation --------------------------------------------- + function `main.test` + --- ```python def test() ``` @@ -2481,11 +2753,15 @@ mod tests { // definition, which is the first definition with a docstring found // by the fallback path. assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) --------------------------------------------- Unrelated docstring --------------------------------------------- + function `main.test` + --- ```python (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) ``` @@ -2532,11 +2808,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) --------------------------------------------- The int overload --------------------------------------------- + function `main.test` + --- ```python (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) ``` @@ -2584,11 +2864,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.test + --------------------------------------------- (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) --------------------------------------------- The real implementation --------------------------------------------- + function `main.test` + --- ```python (Overload[(x: int) -> int, (x: str) -> str]) | (def test()) ``` @@ -2627,8 +2911,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- int --------------------------------------------- + attribute `main.Foo.a` + --- ```python int ``` @@ -2655,11 +2943,15 @@ mod tests { ); assert_snapshot!(test.hover(), @" + function main.foo + --------------------------------------------- def foo( a, b ) --------------------------------------------- + function `main.foo` + --- ```python def foo( a, @@ -3005,8 +3297,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + variable main.value + --------------------------------------------- Literal[123] --------------------------------------------- + variable `main.value` + --- ```python Literal[123] ``` @@ -3037,8 +3333,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + parameter ab + --------------------------------------------- int --------------------------------------------- + parameter `ab` + --- ```python int ``` @@ -3077,8 +3377,12 @@ mod tests { ); assert_snapshot!(test.hover(), @" + variable main.a + --------------------------------------------- (def foo(a, b) -> Literal[0]) | (def bar(a, b) -> Literal[1]) --------------------------------------------- + variable `main.a` + --- ```python (def foo(a, b) -> Literal[0]) | (def bar(a, b) -> Literal[1]) ``` @@ -3105,11 +3409,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3139,11 +3447,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3211,11 +3523,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3272,11 +3588,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3332,8 +3652,12 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + variable main.ab + --------------------------------------------- Unknown --------------------------------------------- + variable `main.ab` + --- ```python Unknown ``` @@ -3387,11 +3711,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3421,11 +3749,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3455,11 +3787,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3489,11 +3825,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3523,11 +3863,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3586,11 +3930,15 @@ mod tests { ); assert_snapshot!(test.hover(), @r#" + class main.MyClass + --------------------------------------------- MyClass --------------------------------------------- some docs --------------------------------------------- + class `main.MyClass` + --- ```python MyClass ``` @@ -3643,11 +3991,15 @@ def ab(a: str): ... .build(); assert_snapshot!(test.hover(), @" + function mymodule.ab + --------------------------------------------- def ab(a: int) --------------------------------------------- the int overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab(a: int) ``` @@ -3688,11 +4040,15 @@ def bar() -> None: .build(); assert_snapshot!(test.hover(), @" + function a.bar + --------------------------------------------- def bar() --------------------------------------------- Implementation docstring --------------------------------------------- + function `a.bar` + --- ```python def bar() ``` @@ -3745,11 +4101,15 @@ def ab(a: str): .build(); assert_snapshot!(test.hover(), @r#" + function mymodule.ab + --------------------------------------------- def ab(a: str) --------------------------------------------- the str overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab(a: str) ``` @@ -3802,6 +4162,8 @@ def ab(a: int): .build(); assert_snapshot!(test.hover(), @" + function mymodule.ab + --------------------------------------------- def ab( a: int, b: int @@ -3810,6 +4172,8 @@ def ab(a: int): the two arg overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab( a: int, @@ -3865,11 +4229,15 @@ def ab(a: int): .build(); assert_snapshot!(test.hover(), @" + function mymodule.ab + --------------------------------------------- def ab(a: int) --------------------------------------------- the one arg overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab(a: int) ``` @@ -3926,6 +4294,8 @@ def ab(a: int, *, c: int): .build(); assert_snapshot!(test.hover(), @" + function mymodule.ab + --------------------------------------------- def ab( a: int, *, @@ -3935,6 +4305,8 @@ def ab(a: int, *, c: int): b overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab( a: int, @@ -3995,6 +4367,8 @@ def ab(a: int, *, c: int): .build(); assert_snapshot!(test.hover(), @" + function mymodule.ab + --------------------------------------------- def ab( a: int, *, @@ -4004,6 +4378,8 @@ def ab(a: int, *, c: int): c overload --------------------------------------------- + function `mymodule.ab` + --- ```python def ab( a: int, @@ -4051,6 +4427,8 @@ def ab(a: int, *, c: int): ); assert_snapshot!(test.hover(), @" + function main.foo + --------------------------------------------- def foo( a: int, b @@ -4063,6 +4441,8 @@ def ab(a: int, *, c: int): The first overload --------------------------------------------- + function `main.foo` + --- ```python def foo( a: int, @@ -4112,12 +4492,16 @@ def ab(a: int, *, c: int): ); assert_snapshot!(test.hover(), @" + function main.foo + --------------------------------------------- def foo(a: int) -> Literal[0] def foo(a: str) -> Literal[1] --------------------------------------------- The first overload --------------------------------------------- + function `main.foo` + --- ```python def foo(a: int) -> Literal[0] def foo(a: str) -> Literal[1] @@ -4159,6 +4543,8 @@ def ab(a: int, *, c: int): .unwrap(); assert_snapshot!(test.hover(), @" + module lib + --------------------------------------------- --------------------------------------------- The cool lib_py module! @@ -4166,6 +4552,8 @@ def ab(a: int, *, c: int): Wow this module rocks. --------------------------------------------- + module `lib` + --- ```xml ``` @@ -4203,8 +4591,12 @@ def outer(): // Should find the variable declaration in the outer scope, not the nonlocal statement assert_snapshot!(test.hover(), @r#" + variable x + --------------------------------------------- Literal["modified"] --------------------------------------------- + variable `x` + --- ```python Literal["modified"] ``` @@ -4236,7 +4628,20 @@ def outer(): ); // Should find the variable declaration in the outer scope, not the nonlocal statement - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @" + variable xy + --------------------------------------------- + variable `xy` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:6:18 + | + 6 | nonlocal xy + | ^- + | || + | |Cursor offset + | source + "); } #[test] @@ -4254,8 +4659,12 @@ def function(): // Should find the global variable declaration, not the global statement assert_snapshot!(test.hover(), @r#" + variable main.global_var + --------------------------------------------- Literal["modified"] --------------------------------------------- + variable `main.global_var` + --- ```python Literal["modified"] ``` @@ -4285,7 +4694,20 @@ def function(): ); // Should find the global variable declaration, not the global statement - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @" + variable main.global_var + --------------------------------------------- + variable `main.global_var` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:5:12 + | + 5 | global global_var + | ^^^^^^^-^^ + | | | + | | Cursor offset + | source + "); } #[test] @@ -4299,7 +4721,20 @@ def function(): "#, ); - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @r#" + variable ab + --------------------------------------------- + variable `ab` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:4:22 + | + 4 | case ["get", ab]: + | ^- + | || + | |Cursor offset + | source + "#); } #[test] @@ -4314,8 +4749,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable ab + --------------------------------------------- str --------------------------------------------- + variable `ab` + --- ```python str ``` @@ -4342,7 +4781,20 @@ def function(): "#, ); - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @r#" + variable ab + --------------------------------------------- + variable `ab` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:4:23 + | + 4 | case ["get", *ab]: + | ^- + | || + | |Cursor offset + | source + "#); } #[test] @@ -4357,8 +4809,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable ab + --------------------------------------------- list[str] --------------------------------------------- + variable `ab` + --- ```python list[str] ``` @@ -4385,7 +4841,20 @@ def function(): "#, ); - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @r#" + variable ab + --------------------------------------------- + variable `ab` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:4:37 + | + 4 | case ["get", ("a" | "b") as ab]: + | ^- + | || + | |Cursor offset + | source + "#); } #[test] @@ -4400,8 +4869,12 @@ def function(): ); assert_snapshot!(test.hover(), @r#" + variable ab + --------------------------------------------- Literal["a", "b"] --------------------------------------------- + variable `ab` + --- ```python Literal["a", "b"] ``` @@ -4434,7 +4907,20 @@ def function(): "#, ); - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @" + variable ab + --------------------------------------------- + variable `ab` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:10:30 + | + 10 | case Click(x, button=ab): + | ^- + | || + | |Cursor offset + | source + "); } #[test] @@ -4455,8 +4941,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable ab + --------------------------------------------- str --------------------------------------------- + variable `ab` + --- ```python str ``` @@ -4490,8 +4980,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + class main.Click + --------------------------------------------- --------------------------------------------- + class `main.Click` + --- ```xml ``` @@ -4536,8 +5030,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- AB (invariant) --------------------------------------------- + type parameter `AB` + --- ```python AB (invariant) ``` @@ -4562,8 +5060,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- AB (invariant) --------------------------------------------- + type parameter `AB` + --- ```python AB (invariant) ``` @@ -4589,8 +5091,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- AB (contravariant) --------------------------------------------- + type parameter `AB` + --- ```python AB (contravariant) ``` @@ -4618,8 +5124,12 @@ def function(): // TODO: This should just be `**AB@Alias2 ()` // https://github.com/astral-sh/ty/issues/1581 assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- (**AB) -> tuple[Unknown] --------------------------------------------- + type parameter `AB` + --- ```python (**AB) -> tuple[Unknown] ``` @@ -4643,7 +5153,20 @@ def function(): "#, ); - assert_snapshot!(test.hover(), @"Hover provided no content"); + assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- + type parameter `AB` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:2:14 + | + 2 | type Alias3[*AB = ()] = tuple[tuple[*AB], tuple[*AB]] + | ^- + | || + | |Cursor offset + | source + "); } #[test] @@ -4655,8 +5178,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter AB + --------------------------------------------- AB (covariant) --------------------------------------------- + type parameter `AB` + --- ```python AB (covariant) ``` @@ -4696,6 +5223,8 @@ def function(): .unwrap(); assert_snapshot!(test.hover(), @" + module lib + --------------------------------------------- --------------------------------------------- The cool lib_py module! @@ -4703,6 +5232,8 @@ def function(): Wow this module rocks. --------------------------------------------- + module `lib` + --- ```xml ``` @@ -4722,6 +5253,99 @@ def function(): "); } + /// a decorated method is shown as its `def` writes it, not as the wrapper its decorator + /// returns, and with its own docstring rather than the wrapper's + #[test] + fn hover_decorated_method_call() { + let test = hover_test( + r#" + def logged(func): + def wrapper(*args, **kwargs): + """Call the function.""" + return func(*args, **kwargs) + return wrapper + + class Circle: + @logged + def area(self, scale: int) -> float: + """The area of the circle.""" + return 1.0 + + Circle().area(2) + "#, + ); + + assert_snapshot!(test.hover(), @" + method main.Circle.area + --------------------------------------------- + @logged + def area( + self, + scale: int + ) -> int | float + --------------------------------------------- + The area of the circle. + + --------------------------------------------- + method `main.Circle.area` + --- + ```python + @logged + def area( + self, + scale: int + ) -> int | float + ``` + --- + The area of the circle. + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:14:10 + | + 14 | Circle().area(2) + | ^^-^ + | | | + | | Cursor offset + | source + "); + } + + /// a property's `def` is shown with its decorator, and its kind says it is a property + #[test] + fn hover_property_def() { + let test = hover_test( + r#" + class Circle: + @property + def diameter(self) -> float: + return 2.0 + "#, + ); + + assert_snapshot!(test.hover(), @" + property main.Circle.diameter + --------------------------------------------- + @property + def diameter(self) -> int | float + --------------------------------------------- + property `main.Circle.diameter` + --- + ```python + @property + def diameter(self) -> int | float + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:4:9 + | + 4 | def diameter(self) -> float: + | ^^^^-^^^ + | | | + | | Cursor offset + | source + "); + } + #[test] fn hover_callable_special_forms() { let test = hover_test( @@ -4734,7 +5358,7 @@ def function(): let hover = test.hover(); assert!( - hover.starts_with(""), + hover.contains("\n"), "{hover}" ); @@ -4748,7 +5372,7 @@ def function(): let hover = test.hover(); assert!( - hover.starts_with(""), + hover.contains("\n"), "{hover}" ); } @@ -4762,8 +5386,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (invariant) --------------------------------------------- + type parameter `T` + --- ```python T (invariant) ``` @@ -4788,8 +5416,12 @@ def function(): // TODO: Should this be constravariant instead? assert_snapshot!(test.hover(), @" + type parameter P + --------------------------------------------- P (covariant) --------------------------------------------- + type parameter `P` + --- ```python P (covariant) ``` @@ -4813,8 +5445,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter Ts + --------------------------------------------- Ts (covariant) --------------------------------------------- + type parameter `Ts` + --- ```python Ts (covariant) ``` @@ -4839,8 +5475,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable a + --------------------------------------------- T --------------------------------------------- + variable `a` + --- ```python T ``` @@ -4864,8 +5504,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter t + --------------------------------------------- T --------------------------------------------- + parameter `t` + --- ```python T ``` @@ -4889,8 +5533,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T --------------------------------------------- + type parameter `T` + --- ```python T ``` @@ -4918,8 +5566,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter t + --------------------------------------------- T --------------------------------------------- + parameter `t` + --- ```python T ``` @@ -4948,8 +5600,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable a + --------------------------------------------- T --------------------------------------------- + variable `a` + --- ```python T ``` @@ -4975,8 +5631,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable a + --------------------------------------------- tuple[T@f, T] --------------------------------------------- + variable `a` + --- ```python tuple[T@f, T] ``` @@ -5001,8 +5661,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable a + --------------------------------------------- P.args --------------------------------------------- + variable `a` + --- ```python P.args ``` @@ -5030,6 +5694,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.value + --------------------------------------------- Literal[1] --------------------------------------------- This is the docs for this value @@ -5037,6 +5703,8 @@ def function(): Wow these are good docs! --------------------------------------------- + variable `main.value` + --- ```python Literal[1] ``` @@ -5077,6 +5745,8 @@ def function(): // of the `value` symbol here in read-context is `1`. This comment mainly exists to // signal that it might be okay to revisit this in the future and reveal 3 instead. assert_snapshot!(test.hover(), @" + variable main.value + --------------------------------------------- Literal[1] --------------------------------------------- This is the docs for this value @@ -5084,6 +5754,8 @@ def function(): Wow these are good docs! --------------------------------------------- + variable `main.value` + --- ```python Literal[1] ``` @@ -5122,6 +5794,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.C.attr + --------------------------------------------- Literal[2] --------------------------------------------- This is the docs for this value @@ -5129,6 +5803,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.C.attr` + --- ```python Literal[2] ``` @@ -5169,6 +5845,8 @@ def function(): // See the comment in the `hover_augmented_assignment` test above. The same // reasoning applies here. assert_snapshot!(test.hover(), @" + attribute main.C.attr + --------------------------------------------- int --------------------------------------------- This is the docs for this value @@ -5176,6 +5854,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.C.attr` + --- ```python int ``` @@ -5208,6 +5888,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- int --------------------------------------------- This is the docs for this value @@ -5215,6 +5897,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.Foo.a` + --- ```python int ``` @@ -5247,6 +5931,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- Literal[1] --------------------------------------------- This is the docs for this value @@ -5254,6 +5940,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.Foo.a` + --- ```python Literal[1] ``` @@ -5289,6 +5977,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- int --------------------------------------------- This is the docs for this value @@ -5296,6 +5986,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.Foo.a` + --- ```python int ``` @@ -5329,6 +6021,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- int --------------------------------------------- This is the docs for this value @@ -5336,6 +6030,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.Foo.a` + --- ```python int ``` @@ -5372,6 +6068,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- int --------------------------------------------- This is the docs for this value @@ -5379,6 +6077,8 @@ def function(): Wow these are good docs! --------------------------------------------- + attribute `main.Foo.a` + --- ```python int ``` @@ -5410,8 +6110,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.a + --------------------------------------------- str (Final) --------------------------------------------- + attribute `main.Foo.a` + --- ```python str (Final) ``` @@ -5437,8 +6141,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.x + --------------------------------------------- Literal[1] (Final) --------------------------------------------- + variable `main.x` + --- ```python Literal[1] (Final) ``` @@ -5465,8 +6173,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.x + --------------------------------------------- Literal[1] (Final) --------------------------------------------- + variable `main.x` + --- ```python Literal[1] (Final) ``` @@ -5496,8 +6208,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + attribute main.Foo.x + --------------------------------------------- int (ClassVar) --------------------------------------------- + attribute `main.Foo.x` + --- ```python int (ClassVar) ``` @@ -5527,8 +6243,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.x + --------------------------------------------- int (Final) --------------------------------------------- + variable `main.x` + --- ```python int (Final) ``` @@ -5560,8 +6280,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter a + --------------------------------------------- str --------------------------------------------- + parameter `a` + --- ```python str ``` @@ -5822,12 +6546,16 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab( x: int, y: (int, int, /) -> Any, z: list[int] ) -> int --------------------------------------------- + function `main.ab` + --- ```python def ab( x: int, @@ -5859,8 +6587,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.ab + --------------------------------------------- tuple[Any, int, (int, int, /) -> Any] --------------------------------------------- + variable `main.ab` + --- ```python tuple[Any, int, (int, int, /) -> Any] ``` @@ -5888,8 +6620,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.ab + --------------------------------------------- ((int, int, /) -> Any) | None --------------------------------------------- + variable `main.ab` + --- ```python ((int, int, /) -> Any) | None ``` @@ -5928,11 +6664,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docsand docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -5963,12 +6703,16 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.urlparse + --------------------------------------------- def urlparse() --------------------------------------------- Parse a URL into components: :///;?# --------------------------------------------- + function `main.urlparse` + --- ```python def urlparse() ``` @@ -5998,8 +6742,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6027,11 +6775,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docsand docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6061,11 +6813,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6096,11 +6852,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6132,11 +6892,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docsand docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6169,11 +6933,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + function main.ab + --------------------------------------------- def ab() --------------------------------------------- wow cool docsand docs --------------------------------------------- + function `main.ab` + --- ```python def ab() ``` @@ -6202,8 +6970,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.ab + --------------------------------------------- Literal[1] --------------------------------------------- + variable `main.ab` + --- ```python Literal[1] ``` @@ -6230,8 +7002,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (covariant) --------------------------------------------- + type parameter `T` + --- ```python T (covariant) ``` @@ -6254,8 +7030,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (covariant) --------------------------------------------- + type parameter `T` + --- ```python T (covariant) ``` @@ -6278,8 +7058,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (contravariant) --------------------------------------------- + type parameter `T` + --- ```python T (contravariant) ``` @@ -6302,8 +7086,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (contravariant) --------------------------------------------- + type parameter `T` + --- ```python T (contravariant) ``` @@ -6329,8 +7117,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (contravariant) --------------------------------------------- + type parameter `T` + --- ```python T (contravariant) ``` @@ -6353,8 +7145,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter U + --------------------------------------------- U --------------------------------------------- + type parameter `U` + --- ```python U ``` @@ -6379,8 +7175,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T --------------------------------------------- + type parameter `T` + --- ```python T ``` @@ -6402,8 +7202,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T --------------------------------------------- + type parameter `T` + --- ```python T ``` @@ -6425,8 +7229,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T --------------------------------------------- + type parameter `T` + --- ```python T ``` @@ -6448,8 +7256,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T --------------------------------------------- + type parameter `T` + --- ```python T ``` @@ -6473,8 +7285,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (invariant) --------------------------------------------- + type parameter `T` + --- ```python T (invariant) ``` @@ -6495,8 +7311,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (invariant) --------------------------------------------- + type parameter `T` + --- ```python T (invariant) ``` @@ -6517,8 +7337,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (covariant) --------------------------------------------- + type parameter `T` + --- ```python T (covariant) ``` @@ -6539,8 +7363,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type parameter T + --------------------------------------------- T (covariant) --------------------------------------------- + type parameter `T` + --- ```python T (covariant) ``` @@ -6564,8 +7392,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + type alias main.Box + --------------------------------------------- type Box = int | None --------------------------------------------- + type alias `main.Box` + --- ```python type Box = int | None ``` @@ -6590,8 +7422,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.Tree + --------------------------------------------- Tree = int | list[Tree] --------------------------------------------- + variable `main.Tree` + --- ```python Tree = int | list[Tree] ``` @@ -6618,8 +7454,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter value + --------------------------------------------- Tree = int | list[Tree] --------------------------------------------- + parameter `value` + --- ```python Tree = int | list[Tree] ``` @@ -6648,8 +7488,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter value + --------------------------------------------- Tree = T | list[Tree[T]] --------------------------------------------- + parameter `value` + --- ```python Tree = T | list[Tree[T]] ``` @@ -6678,8 +7522,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter value + --------------------------------------------- Tree = T | list[Tree[list[T]]] --------------------------------------------- + parameter `value` + --- ```python Tree = T | list[Tree[list[T]]] ``` @@ -6708,11 +7556,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + parameter value + --------------------------------------------- Tree = Link[Tree] --------------------------------------------- A linked value. --------------------------------------------- + parameter `value` + --- ```python Tree = Link[Tree] ``` @@ -6739,6 +7591,8 @@ def function(): ); assert_snapshot!(test.hover(), @" + type alias main.Wrapper + --------------------------------------------- type Wrapper[T] = list[T] --------------------------------------------- Built-in mutable sequence. @@ -6747,6 +7601,8 @@ def function(): The argument must be an iterable if specified. --------------------------------------------- + type alias `main.Wrapper` + --- ```python type Wrapper[T] = list[T] ``` @@ -6804,8 +7660,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.T + --------------------------------------------- TypeVar --------------------------------------------- + variable `main.T` + --- ```python TypeVar ``` @@ -6831,8 +7691,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.T + --------------------------------------------- T@covariant --------------------------------------------- + variable `main.T` + --- ```python T@covariant ``` @@ -6858,8 +7722,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.T + --------------------------------------------- TypeVar --------------------------------------------- + variable `main.T` + --- ```python TypeVar ``` @@ -6885,8 +7753,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.T + --------------------------------------------- T@contravariant --------------------------------------------- + variable `main.T` + --- ```python T@contravariant ``` @@ -6913,8 +7785,12 @@ def function(): ); assert_snapshot!(test.hover(), @" + variable main.T + --------------------------------------------- T (contravariant) --------------------------------------------- + variable `main.T` + --- ```python T (contravariant) ``` @@ -6938,11 +7814,15 @@ def function(): ); assert_snapshot!(test.hover(), @" + method builtins.int.__add__ + --------------------------------------------- bound method int.__add__(value: int, /) -> int --------------------------------------------- Return self+value. --------------------------------------------- + method `builtins.int.__add__` + --- ```python bound method int.__add__(value: int, /) -> int ``` @@ -7174,9 +8054,13 @@ Alias: TypeAlias = "int & str" // TODO: We should only show the matching overload here. // https://github.com/astral-sh/ty/issues/73 assert_snapshot!(test.hover(), @" + method main.Test.__add__ + --------------------------------------------- def __add__(other: Test, /) -> Test def __add__(other: Other, /) -> Test --------------------------------------------- + method `main.Test.__add__` + --- ```python def __add__(other: Test, /) -> Test def __add__(other: Other, /) -> Test @@ -7325,6 +8209,8 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + variable typing.Final + --------------------------------------------- --------------------------------------------- Special typing construct to indicate final names to type checkers. @@ -7345,6 +8231,8 @@ Alias: TypeAlias = "int & str" There is no runtime checking of these properties. --------------------------------------------- + variable `typing.Final` + --- ```xml ``` @@ -7385,8 +8273,12 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + variable main.x + --------------------------------------------- Literal[1] (Final) --------------------------------------------- + variable `main.x` + --- ```python Literal[1] (Final) ``` @@ -7461,8 +8353,12 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + parameter x + --------------------------------------------- int --------------------------------------------- + parameter `x` + --- ```python int ``` @@ -7484,8 +8380,12 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + parameter x + --------------------------------------------- int --------------------------------------------- + parameter `x` + --- ```python int ``` @@ -7510,8 +8410,12 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + variable _ + --------------------------------------------- list[int] --------------------------------------------- + variable `_` + --- ```python list[int] ``` @@ -7533,8 +8437,12 @@ Alias: TypeAlias = "int & str" ); assert_snapshot!(test.hover(), @" + variable _ + --------------------------------------------- list[int] --------------------------------------------- + variable `_` + --- ```python list[int] ``` @@ -7610,8 +8518,12 @@ Alias: TypeAlias = "int & str" // The module is correct assert_snapshot!(test.hover(), @" + module mypackage.subpkg + --------------------------------------------- --------------------------------------------- + module `mypackage.subpkg` + --- ```xml ``` @@ -7688,8 +8600,12 @@ Alias: TypeAlias = "int & str" // The submodule is correct assert_snapshot!(test.hover(), @" + module mypackage.subpkg.submod + --------------------------------------------- --------------------------------------------- + module `mypackage.subpkg.submod` + --- ```xml ``` @@ -7726,8 +8642,12 @@ Alias: TypeAlias = "int & str" // The module is correct assert_snapshot!(test.hover(), @" + module mypackage.subpkg + --------------------------------------------- --------------------------------------------- + module `mypackage.subpkg` + --- ```xml ``` @@ -7764,8 +8684,12 @@ Alias: TypeAlias = "int & str" // int is correct assert_snapshot!(test.hover(), @" + variable mypackage.subpkg.subpkg + --------------------------------------------- int --------------------------------------------- + variable `mypackage.subpkg.subpkg` + --- ```python int ``` @@ -7802,8 +8726,12 @@ Alias: TypeAlias = "int & str" // int is correct assert_snapshot!(test.hover(), @" + variable mypackage.subpkg.subpkg + --------------------------------------------- int --------------------------------------------- + variable `mypackage.subpkg.subpkg` + --- ```python int ``` @@ -7846,8 +8774,12 @@ Alias: TypeAlias = "int & str" .build(); assert_snapshot!(test.hover(), @" + variable mymod.a + --------------------------------------------- Literal[10] --------------------------------------------- + variable `mymod.a` + --- ```python Literal[10] ``` @@ -7876,8 +8808,12 @@ Alias: TypeAlias = "int & str" .build(); assert_snapshot!(test.hover(), @" + variable main.a + --------------------------------------------- 1 --------------------------------------------- + variable `main.a` + --- ```python 1 ``` @@ -7911,8 +8847,12 @@ Alias: TypeAlias = "int & str" .build(); assert_snapshot!(test.hover(), @" + function main.f + --------------------------------------------- def f(x: 1) --------------------------------------------- + function `main.f` + --- ```python def f(x: 1) ``` @@ -7942,8 +8882,12 @@ Alias: TypeAlias = "int & str" .build(); assert_snapshot!(test.hover(), @" + type parameter Kwargs + --------------------------------------------- KeywordPack --------------------------------------------- + type parameter `Kwargs` + --- ```python KeywordPack ``` @@ -8018,11 +8962,15 @@ class CoolType(str): ); assert_snapshot!(test.hover(), @" + type alias main.U + --------------------------------------------- type U = MyType --------------------------------------------- Awesome docs --------------------------------------------- + type alias `main.U` + --- ```python type U = MyType ``` @@ -8053,11 +9001,15 @@ type U = MyType ); assert_snapshot!(test.hover(), @" + type alias main.U + --------------------------------------------- type U = MyType --------------------------------------------- Awesome docs --------------------------------------------- + type alias `main.U` + --- ```python type U = MyType ``` @@ -8101,11 +9053,15 @@ type U = MyType .build(); assert_snapshot!(test.hover(), @" + type alias library.Alias + --------------------------------------------- type Alias = Target --------------------------------------------- Target implementation docs. --------------------------------------------- + type alias `library.Alias` + --- ```python type Alias = Target ``` @@ -8150,11 +9106,15 @@ type U = MyType .build(); assert_snapshot!(test.hover(), @" + type alias library.Alias + --------------------------------------------- type Alias = Target --------------------------------------------- Target implementation docs. --------------------------------------------- + type alias `library.Alias` + --- ```python type Alias = Target ``` @@ -8191,6 +9151,8 @@ type U = MyType .build(); assert_snapshot!(test.hover(), @" + variable library.Copy + --------------------------------------------- type Alias = Target --------------------------------------------- Copy Docs @@ -8199,6 +9161,8 @@ type U = MyType Target Docs --------------------------------------------- + variable `library.Copy` + --- ```python type Alias = Target ``` diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 972681e2e2..922ddedca7 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -13,6 +13,7 @@ mod common_aliases; mod complete_statement; mod completion; mod data_flow; +mod declaration; mod django_template; mod doc_highlights; mod docstring; @@ -67,6 +68,7 @@ pub use completion::{ CompletionKind, CompletionSettings, completion, }; pub use data_flow::{Finding, FindingKind, data_flow_at}; +pub use declaration::{Declaration, DeclaredKind}; pub use django_template::{ DisplayTemplateHover, DjangoChecker, DjangoCodeLens, DjangoLensAction, DjangoLensTarget, PreparedTemplateRename, TemplateCompletion, TemplateEdit, TemplateHover, TemplateInlayHint, diff --git a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin.snap b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin.snap index cc74217ade..4f6e045bd5 100644 --- a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin.snap +++ b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin.snap @@ -2,8 +2,12 @@ source: crates/ty_ide/src/hover.rs expression: test.hover() --- +parameter x +--------------------------------------------- int | builtins.float | main.float --------------------------------------------- +parameter `x` +--- ```python int | builtins.float | main.float ``` diff --git a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin_in_selected_signature.snap b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin_in_selected_signature.snap index 1d2f763862..0b836306b5 100644 --- a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin_in_selected_signature.snap +++ b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_shadowed_numeric_builtin_in_selected_signature.snap @@ -2,8 +2,12 @@ source: crates/ty_ide/src/hover.rs expression: test.hover() --- +function main.choose +--------------------------------------------- def choose(value: int | builtins.float | main.float) --------------------------------------------- +function `main.choose` +--- ```python def choose(value: int | builtins.float | main.float) ``` diff --git a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_subscript_non_literal_index.snap b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_subscript_non_literal_index.snap index 983b0d938a..4a7fa523fc 100644 --- a/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_subscript_non_literal_index.snap +++ b/crates/ty_ide/src/snapshots/ty_ide__hover__tests__hover_subscript_non_literal_index.snap @@ -2,8 +2,12 @@ source: crates/ty_ide/src/hover.rs expression: test.hover() --- +variable main.idx +--------------------------------------------- int --------------------------------------------- +variable `main.idx` +--- ```python int ``` diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 3987d12aeb..0c79634ee9 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -5039,3 +5039,32 @@ cast(*args) Ok(()) } } + +/// basedpython: a `def` as its source writes it and as its decorators leave it. +#[derive(Debug, Clone, Copy)] +pub struct DecoratedFunction<'db> { + /// the function the `def` makes, before any decorator applies to it + pub undecorated: Type<'db>, + /// what the decorators bind the name to, which is the function itself when there are none + pub decorated: Type<'db>, +} + +impl DecoratedFunction<'_> { + /// whether the decorators bind the name to a property + pub fn is_property(&self) -> bool { + matches!(self.decorated, Type::PropertyInstance(_)) + } +} + +/// basedpython: what the `def` `definition` makes, before and after its decorators. `None` for +/// a definition that is not a `def` +pub fn decorated_function<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> Option> { + let function = infer_definition_types(db, definition).function_type(definition)?; + Some(DecoratedFunction { + undecorated: Type::FunctionLiteral(function), + decorated: binding_type(db, definition), + }) +} diff --git a/crates/ty_server/tests/e2e/closed_documents.rs b/crates/ty_server/tests/e2e/closed_documents.rs index 7d088f8014..d282a36223 100644 --- a/crates/ty_server/tests/e2e/closed_documents.rs +++ b/crates/ty_server/tests/e2e/closed_documents.rs @@ -109,10 +109,20 @@ value .wait_until_workspaces_are_initialized(); let position = Position::new(5, 0); - assert_hover(&mut server, path, position, "Literal[\"workspace\"]"); + assert_hover( + &mut server, + path, + position, + "variable dependency.value\n---------------------------------------------\nLiteral[\"workspace\"]", + ); server.open_text_document(path, source, 1); - assert_hover(&mut server, path, position, "Literal[\"script\"]"); + assert_hover( + &mut server, + path, + position, + "variable dependency.value\n---------------------------------------------\nLiteral[\"script\"]", + ); Ok(()) } @@ -133,7 +143,12 @@ value ])? .build() .wait_until_workspaces_are_initialized(); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"disk\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable library.value\n---------------------------------------------\nLiteral[\"disk\"]", + ); // Opening the file must replace the cached disk contents with unsaved editor contents. let unsaved = "\ @@ -145,11 +160,16 @@ value &mut server, path, Position::new(1, 0), - "Literal[\"unsaved\"]", + "variable library.value\n---------------------------------------------\nLiteral[\"unsaved\"]", ); // Closing without saving must restore the disk contents. server.close_text_document(path); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"disk\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable library.value\n---------------------------------------------\nLiteral[\"disk\"]", + ); // While the file is closed, filesystem notifications must refresh the cached contents. let edited = "\ @@ -162,14 +182,19 @@ value &mut server, path, Position::new(1, 0), - "Literal[\"edited\"]", + "variable library.value\n---------------------------------------------\nLiteral[\"edited\"]", ); std::fs::remove_file(server.file_path(path))?; server.did_change_watched_file(path, FileChangeType::Deleted); assert!(server.hover_request(path, Position::new(1, 0)).is_none()); server.write_file(path, source)?; server.did_change_watched_file(path, FileChangeType::Created); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"disk\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable library.value\n---------------------------------------------\nLiteral[\"disk\"]", + ); Ok(()) } @@ -193,7 +218,12 @@ value .wait_until_workspaces_are_initialized(); // Neither project contains the file; the first workspace by path supplies its environment. - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"a\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable project_value.value\n---------------------------------------------\nLiteral[\"a\"]", + ); Ok(()) } diff --git a/crates/ty_server/tests/e2e/configuration.rs b/crates/ty_server/tests/e2e/configuration.rs index 71863b7f18..0c9826a748 100644 --- a/crates/ty_server/tests/e2e/configuration.rs +++ b/crates/ty_server/tests/e2e/configuration.rs @@ -176,7 +176,7 @@ fn invalid_python_environment_preserves_valid_configuration() -> Result<()> { { "contents": { "kind": "plaintext", - "value": "Literal[\"win32\"]" + "value": "variable sys.platform\n---------------------------------------------\nLiteral[\"win32\"]" }, "range": { "start": { diff --git a/crates/ty_server/tests/e2e/hover.rs b/crates/ty_server/tests/e2e/hover.rs index ecf8c6ef7d..f24989f7fa 100644 --- a/crates/ty_server/tests/e2e/hover.rs +++ b/crates/ty_server/tests/e2e/hover.rs @@ -74,7 +74,7 @@ python-platform = "linux" { "contents": { "kind": "plaintext", - "value": "Literal[\"win32\"]" + "value": "variable sys.platform\n---------------------------------------------\nLiteral[\"win32\"]" }, "range": { "start": { @@ -125,7 +125,7 @@ python-platform = "linux" { "contents": { "kind": "plaintext", - "value": "Literal[\"win32\"]" + "value": "variable sys.platform\n---------------------------------------------\nLiteral[\"win32\"]" }, "range": { "start": { @@ -198,7 +198,7 @@ python-version = "3.12" { "contents": { "kind": "plaintext", - "value": "Literal[12]" + "value": "variable shared.value\n---------------------------------------------\nLiteral[12]" }, "range": { "start": { @@ -218,7 +218,7 @@ python-version = "3.12" { "contents": { "kind": "plaintext", - "value": "Literal[13]" + "value": "variable shared.value\n---------------------------------------------\nLiteral[13]" }, "range": { "start": { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__cells_after_cell_magic_hover.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__cells_after_cell_magic_hover.snap index 862b56272d..be9040f127 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__cells_after_cell_magic_hover.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__cells_after_cell_magic_hover.snap @@ -5,7 +5,7 @@ expression: hover { "contents": { "kind": "plaintext", - "value": "Literal[1]" + "value": "variable x\n---------------------------------------------\nLiteral[1]" }, "range": { "start": { diff --git a/crates/ty_server/tests/e2e/workspace_folders.rs b/crates/ty_server/tests/e2e/workspace_folders.rs index 8fa9b7f591..2a01b97d3c 100644 --- a/crates/ty_server/tests/e2e/workspace_folders.rs +++ b/crates/ty_server/tests/e2e/workspace_folders.rs @@ -675,7 +675,12 @@ extra-paths = ["../shared"] .wait_until_workspaces_are_initialized(); server.open_text_document(path, source, 1); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"web\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable project_value.value\n---------------------------------------------\nLiteral[\"web\"]", + ); // Adding api's search path moves the shared file from web's database to api's. let config = SystemPath::new("api/ty.toml"); @@ -684,7 +689,12 @@ extra-paths = ["../shared"] uri: server.file_uri(config), kind: lsp_types::FileChangeType::Changed, }]); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"api\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable project_value.value\n---------------------------------------------\nLiteral[\"api\"]", + ); server.change_text_document( path, @@ -705,7 +715,7 @@ value &mut server, path, Position::new(1, 0), - "Literal[\"edited\"]", + "variable library.value\n---------------------------------------------\nLiteral[\"edited\"]", ); // The old database must also see the edit when importing the shared file. @@ -714,7 +724,7 @@ value &mut server, importer, Position::new(0, 20), - "Literal[\"edited\"]", + "variable library.value\n---------------------------------------------\nLiteral[\"edited\"]", ); Ok(()) @@ -784,7 +794,7 @@ extra-paths = ["imports"] &mut server, path, Position::new(1, 0), - "Literal[\"library\"]", + "variable project_value.value\n---------------------------------------------\nLiteral[\"library\"]", ); Ok(()) @@ -825,7 +835,12 @@ extra-paths = ["../shared"] .wait_until_workspaces_are_initialized(); server.open_text_document(path, source, 1); - assert_hover(&mut server, path, Position::new(1, 0), "Literal[\"tools\"]"); + assert_hover( + &mut server, + path, + Position::new(1, 0), + "variable project_value.value\n---------------------------------------------\nLiteral[\"tools\"]", + ); Ok(()) } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 3c0c87fc70..ef834e33ed 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -584,6 +584,29 @@ the caret goes in the document as the edits leave it: like the other document requests it takes `textHash`, and it is `null` when language services are off +## hover + +a hover opens with what the name is: its kind and the dotted name it is +reached by from outside its module, `method shapes.base.Circle.area`. the kinds +are `module`, `class`, `function`, `method`, `property`, `attribute` (a name a +class body binds, or one a method assigns on `self`), `variable`, `parameter`, +`type parameter` and `type alias`. a name local to a function has only its own +name, and so does a parameter or a type parameter + +a decorated `def` is shown as its source writes it, its decorators over its own +signature and with its own docstring: + +```by +@functools.cache +def discount(self, x: int) -> int +``` + +and the docstring is the def's, not that of the type its decorators bind the name +to. when that type is not the def's own function, which for a decorator returning +a wrapper of its own is `def wrapper(*args, **kwargs)`, it is shown first, before +the def as written. that holds wherever the name is hovered, a call of it included, +except a use of a property, which reads the property's value + ## extend selection `textDocument/selectionRange` grows a selection the way a reader reads the From ec39c708ea77b0e53908df15c2d75f20f79cf376 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:52:18 +1000 Subject: [PATCH 013/110] answer by/declaration, the kind, name and qualified name of what the name at a position declares or refers to with where it is, shapes.base.Circle.area on area in circle.area(), for an editor's copy reference and the target of a usage search --- crates/ty_ide/src/declaration.rs | 70 +++++- crates/ty_ide/src/hover.rs | 228 ++++++++++++++---- crates/ty_ide/src/lib.rs | 2 +- .../src/types/ide_support.rs | 19 +- crates/ty_server/src/server/api.rs | 3 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/declaration.rs | 132 ++++++++++ crates/ty_server/tests/e2e/declaration.rs | 127 ++++++++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 46 ++++ 10 files changed, 581 insertions(+), 49 deletions(-) create mode 100644 crates/ty_server/src/server/api/requests/declaration.rs create mode 100644 crates/ty_server/tests/e2e/declaration.rs diff --git a/crates/ty_ide/src/declaration.rs b/crates/ty_ide/src/declaration.rs index 8a45942a5e..bd6a22e157 100644 --- a/crates/ty_ide/src/declaration.rs +++ b/crates/ty_ide/src/declaration.rs @@ -15,16 +15,19 @@ //! name local to a function, a parameter among them, has no qualified name, and nor does a type //! parameter: nothing outside the function or the generic reaches it. +use ruff_db::files::FileRange; use ruff_db::parsed::parsed_module; +use ruff_text_size::{Ranged, TextSize}; use ty_module_resolver::file_to_module; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::place::PlaceExprRef; use ty_python_core::scope::{FileScopeId, ScopeKind}; use ty_python_core::{ProgramFile, semantic_index}; -use ty_python_semantic::ResolvedDefinition; use ty_python_semantic::types::ide_support::decorated_function; +use ty_python_semantic::{HasDefinition, ImportAliasResolution, ResolvedDefinition, SemanticModel}; -use crate::Db; +use crate::goto::{Definitions, GotoTarget, find_goto_target}; +use crate::{Db, NavigationTarget, RangedValue}; /// What kind of thing a declaration declares. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -75,6 +78,69 @@ pub struct Declaration { pub qualified_name: Option, } +/// the declaration the name at `offset` refers to, or declares, with where it is +#[derive(Debug, Clone)] +pub struct DeclarationAt { + pub declaration: Declaration, + /// where the declaration is: the whole of it, and the name it declares + pub target: NavigationTarget, +} + +/// what the name at `offset` declares, or refers to: on `total_area` in `total_area(shapes)` +/// the function `shapes.util.total_area`, on `Circle` in `Circle(3)` the class rather than the +/// `__init__` the call runs. `None` where there is no name, and for a name whose definitions +/// declare different things +pub fn declaration_at( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option> { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + let goto_target = find_goto_target(&model, &parsed, offset)?; + let declared = declared_definitions(&model, &goto_target)?; + let declaration = single_declaration(db, &declared)?; + let target = declared.into_navigation_targets(db).into_iter().next()?; + Some(RangedValue { + range: FileRange::new(file.file(db), goto_target.range()), + value: DeclarationAt { + declaration, + target, + }, + }) +} + +/// the definitions of what the name at `goto_target` declares, read off the name's own +/// definitions rather than off what a call of it runs: `Circle` in `Circle(3)` is the class, +/// whose `__init__` the call runs, and the name of a decorated `def` is the `def`, which a +/// decorator returning a function of its own would otherwise stand in for +pub(crate) fn declared_definitions<'db>( + model: &SemanticModel<'db>, + goto_target: &GotoTarget<'_>, +) -> Option> { + match goto_target { + GotoTarget::Expression(_) | GotoTarget::Call { .. } => { + goto_target.expression_definitions(model, ImportAliasResolution::ResolveAliases) + } + GotoTarget::FunctionDef(function) => { + Some(Definitions::new(vec![ResolvedDefinition::Definition( + function.definition(model), + )])) + } + _ => goto_target.definitions(model, ImportAliasResolution::ResolveAliases), + } +} + +/// the one declaration `declared` makes, when its definitions all declare the same name the same +/// way: the bindings of a variable assigned twice are one variable +pub(crate) fn single_declaration(db: &dyn Db, declared: &Definitions<'_>) -> Option { + let mut declarations = declared.iter().map(|resolved| declaration(db, resolved)); + let first = declarations.next()??; + declarations + .all(|other| other.as_ref() == Some(&first)) + .then_some(first) +} + /// What `resolved` declares. `None` for what is not a declaration of a name: an import that was /// not followed to what it imports, or a range in a file that is not a definition. pub(crate) fn declaration(db: &dyn Db, resolved: &ResolvedDefinition<'_>) -> Option { diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index 245200b7ab..2e826eeb93 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -1,5 +1,5 @@ use crate::annotation_expression::is_in_annotation_expression; -use crate::declaration::{Declaration, DeclaredKind, declaration}; +use crate::declaration::{Declaration, DeclaredKind, declared_definitions, single_declaration}; use crate::docstring::{Docstring, DocstringFragment}; use crate::goto::{ Definitions, GotoTarget, docstring_for_call_definition, find_goto_target, find_goto_target_impl, @@ -26,9 +26,7 @@ use ty_python_semantic::types::{ use ty_python_semantic::{DisplaySettings, ProgramEnvironment}; use ty_python_semantic::types::format::{SpecLanguage, spec_language}; -use ty_python_semantic::{ - HasDefinition, HasType, SemanticModel, TypeQualifiers, with_display_for_file, -}; +use ty_python_semantic::{HasType, SemanticModel, TypeQualifiers, with_display_for_file}; pub fn hover<'db>( db: &'db dyn Db, @@ -113,23 +111,7 @@ fn hover_inner<'db>( let mut alias_docstring: Option = None; - // what the hovered name declares, read off the name's own definitions rather than off what a - // call of it runs: `Circle` in `Circle(3)` is the class, whose `__init__` the call runs - let declared = match &goto_target { - GotoTarget::Expression(_) | GotoTarget::Call { .. } => goto_target.expression_definitions( - &model, - ty_python_semantic::ImportAliasResolution::ResolveAliases, - ), - // the `def` itself, which a decorator that returns a function of its own would otherwise - // stand in for - GotoTarget::FunctionDef(function) => Some(Definitions::new(vec![ - ty_python_semantic::ResolvedDefinition::Definition(function.definition(&model)), - ])), - _ => goto_target.definitions( - &model, - ty_python_semantic::ImportAliasResolution::ResolveAliases, - ), - }; + let declared = declared_definitions(&model, &goto_target); let declaration = declared .as_ref() .and_then(|declared| single_declaration(db, declared)); @@ -141,8 +123,8 @@ fn hover_inner<'db>( }) }; - let docs = if let Some((_, docstring)) = &decorated { - docstring.clone().map(HoverContent::Docstring) + let docs = if let Some(decorated) = &decorated { + decorated.docstring.clone().map(HoverContent::Docstring) } else if keyword_argument.is_some() || typed_dict_key.is_some() { None } else if let GotoTarget::Call { call, .. } = goto_target { @@ -180,8 +162,15 @@ fn hover_inner<'db>( } if let Some(keyword_argument) = keyword_argument { contents.extend(keyword_argument); - } else if let Some((function, _)) = decorated { - contents.push(function); + } else if let Some(decorated) = decorated { + if let Some(ty) = decorated.replacement { + contents.push(HoverContent::Type { + ty, + variance: None, + qualifiers: TypeQualifiers::empty(), + }); + } + contents.push(decorated.written); } else if let Some(signature) = goto_target.call_signature(&model) { contents.push(HoverContent::Signature(signature)); } else if let Some(typed_dict_key) = typed_dict_key { @@ -335,20 +324,16 @@ fn annotation_operator( } } -/// The one declaration `declared` makes, when its definitions all declare the same name the same -/// way: the bindings of a variable assigned twice are one variable. -fn single_declaration(db: &dyn Db, declared: &Definitions<'_>) -> Option { - let mut declarations = declared.iter().map(|resolved| declaration(db, resolved)); - let first = declarations.next()??; - declarations - .all(|other| other.as_ref() == Some(&first)) - .then_some(first) +/// a decorated `def`: the type its decorators bind the name to, a `wrapper(*args, **kwargs)` or a +/// `_lru_cache_wrapper[...]`, when that is not the function the `def` writes, then the `def` as +/// its source writes it, its decorators over its own signature, and its own docstring rather than +/// the wrapper's +struct DecoratedFunctionHover<'db> { + replacement: Option>, + written: HoverContent<'db>, + docstring: Option, } -/// A decorated `def` as its source writes it, its decorators over its own signature, and its own -/// docstring: what the name's type would show instead is whatever its decorators return, a -/// `wrapper(*args, **kwargs)` or a `_lru_cache_wrapper[...]`, documented as the wrapper is. -/// /// `None` for a `def` without decorators, whose type is the function it writes, and for a use of /// a property, which reads the property's value rather than naming the `def`. fn decorated_function_hover<'db>( @@ -356,7 +341,7 @@ fn decorated_function_hover<'db>( goto_target: &GotoTarget<'_>, declared: &Definitions<'db>, declaration: Option<&Declaration>, -) -> Option<(HoverContent<'db>, Option)> { +) -> Option> { let mut definitions = declared.iter(); let (Some(ty_python_semantic::ResolvedDefinition::Definition(definition)), None) = (definitions.next(), definitions.next()) @@ -382,18 +367,19 @@ fn decorated_function_hover<'db>( if decorators.is_empty() { return None; } - let function = decorated_function(db, *definition)?.undecorated; + let function = decorated_function(db, *definition)?; let docstring = Definitions::new(vec![ty_python_semantic::ResolvedDefinition::Definition( *definition, )]) .docstring(db); - Some(( - HoverContent::DecoratedFunction { + Some(DecoratedFunctionHover { + replacement: function.replacement(db), + written: HoverContent::DecoratedFunction { decorators, - function, + function: function.undecorated, }, docstring, - )) + }) } fn keyword_argument_hover_contents<'db>( @@ -2585,6 +2571,8 @@ mod tests { assert_snapshot!(test.hover(), @" function main.foo --------------------------------------------- + Literal[42] + --------------------------------------------- @decorator def foo() --------------------------------------------- @@ -2594,6 +2582,10 @@ mod tests { function `main.foo` --- ```python + Literal[42] + ``` + --- + ```python @decorator def foo() ``` @@ -5253,8 +5245,8 @@ def function(): "); } - /// a decorated method is shown as its `def` writes it, not as the wrapper its decorator - /// returns, and with its own docstring rather than the wrapper's + /// a decorated method is shown as the wrapper its decorator returns, then as its `def` writes + /// it, with its own docstring rather than the wrapper's #[test] fn hover_decorated_method_call() { let test = hover_test( @@ -5278,6 +5270,8 @@ def function(): assert_snapshot!(test.hover(), @" method main.Circle.area --------------------------------------------- + def wrapper(...) -> Unknown + --------------------------------------------- @logged def area( self, @@ -5290,6 +5284,10 @@ def function(): method `main.Circle.area` --- ```python + def wrapper(...) -> Unknown + ``` + --- + ```python @logged def area( self, @@ -5310,6 +5308,140 @@ def function(): "); } + /// a `def` whose decorator returns something other than a function is shown as that, then as + /// the `def` writes it + #[test] + fn hover_context_manager_def_use() { + let test = hover_test( + r#" + from contextlib import contextmanager + from collections.abc import Iterator + + @contextmanager + def opened() -> Iterator[int]: + """Open it.""" + yield 1 + + opened() + "#, + ); + + assert_snapshot!(test.hover(), @" + function main.opened + --------------------------------------------- + () -> _GeneratorContextManager[int, None, None] + --------------------------------------------- + @contextmanager + def opened() -> Iterator[int] + --------------------------------------------- + Open it. + + --------------------------------------------- + function `main.opened` + --- + ```python + () -> _GeneratorContextManager[int, None, None] + ``` + --- + ```python + @contextmanager + def opened() -> Iterator[int] + ``` + --- + Open it. + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:10:1 + | + 10 | opened() + | ^^^^-^ + | | | + | | Cursor offset + | source + "); + } + + #[test] + fn hover_cached_def() { + let test = hover_test( + r#" + import functools + + @functools.cache + def computed(x: int) -> str: + return "" + "#, + ); + + assert_snapshot!(test.hover(), @" + function main.computed + --------------------------------------------- + _lru_cache_wrapper[def computed(x: int) -> str] + --------------------------------------------- + @functools.cache + def computed(x: int) -> str + --------------------------------------------- + function `main.computed` + --- + ```python + _lru_cache_wrapper[def computed(x: int) -> str] + ``` + --- + ```python + @functools.cache + def computed(x: int) -> str + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:5:5 + | + 5 | def computed(x: int) -> str: + | ^^^^-^^^ + | | | + | | Cursor offset + | source + "); + } + + /// a decorator that returns the function it is given leaves the name bound to the `def`, so + /// the hover shows only the `def` as it is written + #[test] + fn hover_def_decorated_by_identity_decorator() { + let test = hover_test( + r#" + def registered[F](function: F) -> F: + return function + + @registered + def handler(x: int) -> str: + return "" + "#, + ); + + assert_snapshot!(test.hover(), @" + function main.handler + --------------------------------------------- + @registered + def handler(x: int) -> str + --------------------------------------------- + function `main.handler` + --- + ```python + @registered + def handler(x: int) -> str + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:6:5 + | + 6 | def handler(x: int) -> str: + | ^^^-^^^ + | | | + | | Cursor offset + | source + "); + } + /// a property's `def` is shown with its decorator, and its kind says it is a property #[test] fn hover_property_def() { @@ -5325,12 +5457,18 @@ def function(): assert_snapshot!(test.hover(), @" property main.Circle.diameter --------------------------------------------- + property + --------------------------------------------- @property def diameter(self) -> int | float --------------------------------------------- property `main.Circle.diameter` --- ```python + property + ``` + --- + ```python @property def diameter(self) -> int | float ``` diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 922ddedca7..9f60eff0d6 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -68,7 +68,7 @@ pub use completion::{ CompletionKind, CompletionSettings, completion, }; pub use data_flow::{Finding, FindingKind, data_flow_at}; -pub use declaration::{Declaration, DeclaredKind}; +pub use declaration::{Declaration, DeclarationAt, DeclaredKind, declaration_at}; pub use django_template::{ DisplayTemplateHover, DjangoChecker, DjangoCodeLens, DjangoLensAction, DjangoLensTarget, PreparedTemplateRename, TemplateCompletion, TemplateEdit, TemplateHover, TemplateInlayHint, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 0c79634ee9..de7af87fef 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -5047,13 +5047,29 @@ pub struct DecoratedFunction<'db> { pub undecorated: Type<'db>, /// what the decorators bind the name to, which is the function itself when there are none pub decorated: Type<'db>, + definition: Definition<'db>, } -impl DecoratedFunction<'_> { +impl<'db> DecoratedFunction<'db> { /// whether the decorators bind the name to a property pub fn is_property(&self) -> bool { matches!(self.decorated, Type::PropertyInstance(_)) } + + /// what the decorators bind the name to when it is not the function the `def` writes but + /// something a decorator returned in its place, a wrapper, a `_lru_cache_wrapper[...]` or an + /// `int`. `None` when the decorators leave the name bound to the `def`'s own function, as + /// `@staticmethod`, `@final` and a decorator that returns its argument do + pub fn replacement(&self, db: &'db dyn Db) -> Option> { + match self.decorated { + Type::FunctionLiteral(function) + if function.contains_definition(db, self.definition) => + { + None + } + decorated => Some(decorated), + } + } } /// basedpython: what the `def` `definition` makes, before and after its decorators. `None` for @@ -5066,5 +5082,6 @@ pub fn decorated_function<'db>( Some(DecoratedFunction { undecorated: Type::FunctionLiteral(function), decorated: binding_type(db, definition), + definition, }) } diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index e6ae75a446..d3e51d52f1 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -237,6 +237,9 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::LatencySensitive ), + requests::DeclarationRequestHandler::METHOD => background_document_request_task::< + requests::DeclarationRequestHandler, + >(req, BackgroundSchedule::Worker), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index b58876d9ef..de80e4b2bb 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -24,6 +24,7 @@ mod complete_statement; mod completion; mod copied_imports; mod data_flow; +mod declaration; mod detached_transpile; mod diagnostic; mod doc_highlights; @@ -85,6 +86,7 @@ pub(super) use complete_statement::CompleteStatementRequestHandler; pub(super) use completion::CompletionRequestHandler; pub(super) use copied_imports::CopiedImportsRequestHandler; pub(super) use data_flow::DataFlowRequestHandler; +pub(super) use declaration::DeclarationRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; pub(super) use document_super_members::DocumentSuperMembersRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/declaration.rs b/crates/ty_server/src/server/api/requests/declaration.rs new file mode 100644 index 0000000000..da9d8af090 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/declaration.rs @@ -0,0 +1,132 @@ +//! `by/declaration`: what the name at a position declares, or refers to +//! +//! its kind, its name, and the dotted name it is reached by from outside its module: +//! `shapes.base.Circle.area` on `area` in `circle.area()`. what an editor's *copy reference* +//! copies, and how it names the target of a search for usages. LSP has no request for it: +//! `textDocument/hover` is prose for a reader, and `workspace/symbol` answers a query rather than +//! a position. see [`ty_ide::declaration_at`] for what is answered + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ruff_db::files::FileRange; +use ty_ide::declaration_at; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToLink}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DeclarationRequest {} + +impl Request for DeclarationRequest { + type Params = DeclarationParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/declaration"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the position of a name, where it is declared or where it is used +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DeclarationParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// what the name declares, and where +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Declaration { + /// `module`, `class`, `function`, `method`, `property`, `attribute`, `variable`, + /// `parameter`, `type parameter` or `type alias` + kind: String, + /// the declared name + name: String, + /// the module's dotted name, each class the declaration is made in, and the name. `null` for + /// a name local to a function, a parameter, a type parameter, and a declaration in a file no + /// module is imported from + qualified_name: Option, + /// the name at the position the request asked about + origin_selection_range: Option, + /// the file the declaration is in + uri: Uri, + /// the whole declaration + range: Range, + /// the declared name + selection_range: Range, +} + +pub(crate) struct DeclarationRequestHandler; + +impl RequestHandler for DeclarationRequestHandler { + type RequestType = DeclarationRequest; +} + +impl BackgroundDocumentRequestHandler for DeclarationRequestHandler { + // what the project's files say about a position; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &DeclarationParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a template, where there is no name, and for a + /// name whose definitions declare different things + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: DeclarationParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.is_django_template() { + return Ok(None); + } + + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let Some(found) = declaration_at(db, db.program_file(file), offset) else { + return Ok(None); + }; + let origin: FileRange = found.range; + let Some(link) = found + .value + .target + .to_link(db, Some(origin), snapshot.encoding()) + else { + return Ok(None); + }; + let declaration = found.value.declaration; + Ok(Some(Declaration { + kind: declaration.kind.as_str().to_string(), + name: declaration.name, + qualified_name: declaration.qualified_name, + origin_selection_range: link.origin_selection_range, + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + })) + } +} + +impl RetriableRequestHandler for DeclarationRequestHandler {} diff --git a/crates/ty_server/tests/e2e/declaration.rs b/crates/ty_server/tests/e2e/declaration.rs new file mode 100644 index 0000000000..6d8c6b0332 --- /dev/null +++ b/crates/ty_server/tests/e2e/declaration.rs @@ -0,0 +1,127 @@ +//! `by/declaration` over the wire +//! +//! what a name declares is `ty_ide`'s and its unit tests cover it. what these cover is the +//! contract a client reads: a position in, and out the declaration's kind, name and qualified +//! name with where it is, or `null` where there is no name + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// the request as a client sends it, in json throughout +enum Declaration {} + +impl Request for Declaration { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/declaration"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const BASE: &str = "\ +class Circle: + def __init__(self, radius: float) -> None: + self.radius = radius + + def area(self) -> float: + return self.radius +"; + +const MAIN: &str = "\ +from shapes.base import Circle + +circle = Circle(2) +size = circle.area() + +def scaled() -> float: + factor = 2 + return factor * circle.radius +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes/__init__.by"), "")? + .with_file(SystemPath::new("src/shapes/base.by"), BASE)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, line: u32, character: u32) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + })) +} + +fn range(line: u32, start: u32, end: u32) -> serde_json::Value { + serde_json::json!({ + "start": { "line": line, "character": start }, + "end": { "line": line, "character": end }, + }) +} + +#[test] +fn a_method_used_elsewhere_is_named_by_its_module_and_class() -> Result<()> { + let mut server = server()?; + let base = server.file_uri(SystemPath::new("src/shapes/base.by")); + + let answer = ask(&mut server, 3, 16).expect("a declaration"); + assert_eq!(answer["kind"], "method"); + assert_eq!(answer["name"], "area"); + assert_eq!(answer["qualifiedName"], "shapes.base.Circle.area"); + assert_eq!(answer["originSelectionRange"], range(3, 14, 18)); + assert_eq!(answer["uri"], serde_json::json!(base)); + assert_eq!(answer["selectionRange"], range(4, 8, 12)); + Ok(()) +} + +#[test] +fn a_called_class_is_the_class() -> Result<()> { + let mut server = server()?; + + let answer = ask(&mut server, 2, 10).expect("a declaration"); + assert_eq!(answer["kind"], "class"); + assert_eq!(answer["qualifiedName"], "shapes.base.Circle"); + Ok(()) +} + +#[test] +fn an_attribute_assigned_in_a_method_belongs_to_the_class() -> Result<()> { + let mut server = server()?; + + let answer = ask(&mut server, 7, 30).expect("a declaration"); + assert_eq!(answer["kind"], "attribute"); + assert_eq!(answer["qualifiedName"], "shapes.base.Circle.radius"); + Ok(()) +} + +#[test] +fn a_local_has_no_qualified_name() -> Result<()> { + let mut server = server()?; + + let answer = ask(&mut server, 7, 12).expect("a declaration"); + assert_eq!(answer["kind"], "variable"); + assert_eq!(answer["name"], "factor"); + assert_eq!(answer["qualifiedName"], serde_json::Value::Null); + + let module_level = ask(&mut server, 3, 1).expect("a declaration"); + assert_eq!(module_level["qualifiedName"], "main.size"); + Ok(()) +} + +#[test] +fn no_name_answers_null() -> Result<()> { + let mut server = server()?; + + assert_eq!(ask(&mut server, 1, 0), None); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 76ca637e18..0f0d25ad9d 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -39,6 +39,7 @@ mod complete_statement; mod completions; mod configuration; mod data_flow; +mod declaration; mod diagnostic_snapshots; mod django_templates; mod evaluatable_expression; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ef834e33ed..4fde29b571 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -607,6 +607,52 @@ a wrapper of its own is `def wrapper(*args, **kwargs)`, it is shown first, befor the def as written. that holds wherever the name is hovered, a call of it included, except a use of a property, which reads the property's value +## what a name declares + +an editor's *copy reference* copies a declaration's dotted name, and a search +for usages names what it searched for. both ask `by/declaration`, with the +position of a name where it is declared or where it is used: + +```json +{ + "textDocument": { "uri": "file:///src/main.by" }, + "position": { "line": 3, "character": 16 } +} +``` + +and the answer is what the name declares, or refers to, and where: + +```json +{ + "kind": "method", + "name": "area", + "qualifiedName": "shapes.base.Circle.area", + "originSelectionRange": { + "start": { "line": 3, "character": 14 }, + "end": { "line": 3, "character": 18 } + }, + "uri": "file:///src/shapes/base.by", + "range": { + "start": { "line": 4, "character": 4 }, + "end": { "line": 5, "character": 26 } + }, + "selectionRange": { + "start": { "line": 4, "character": 8 }, + "end": { "line": 4, "character": 12 } + } +} +``` + +`kind` is one of the kinds a hover names. `qualifiedName` is the module's +dotted name, each class the declaration is made in, and the name; an attribute +a method assigns on `self` is the class's, `shapes.base.Circle.radius`. it is +`null` for a name local to a function, a parameter, a type parameter, and a +declaration in a file no module is imported from. `originSelectionRange` is the +name at the position, `range` the whole declaration and `selectionRange` its +name. a called class is the class, not the `__init__` the call runs, and a +decorated `def` is the `def`. the answer is `null` where there is no name, and +for a name whose definitions declare different things + ## extend selection `textDocument/selectionRange` grows a selection the way a reader reads the From a1230b837d977810d407e0a78fb3b527739df1a4 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:58:46 +1000 Subject: [PATCH 014/110] name a workspace symbol's module and classes as its containerName, match a dotted query's parts before the name against them, and find an attribute a method assigns on self as a field of its class, in workspace symbols and the outline both --- crates/ty_ide/src/document_symbols.rs | 7 + crates/ty_ide/src/symbols.rs | 120 +++++++++++ crates/ty_ide/src/workspace_symbols.rs | 200 ++++++++++++++++-- .../server/api/requests/workspace_symbols.rs | 11 +- crates/ty_server/src/server/api/symbols.rs | 5 +- docs/basedpython/features/editor.md | 21 ++ 6 files changed, 343 insertions(+), 21 deletions(-) diff --git a/crates/ty_ide/src/document_symbols.rs b/crates/ty_ide/src/document_symbols.rs index e9ad617a3f..766f8d2a91 100644 --- a/crates/ty_ide/src/document_symbols.rs +++ b/crates/ty_ide/src/document_symbols.rs @@ -218,6 +218,13 @@ def standalone_function(): | ^^^^^^^^^^^^^^^ info: Method _private_method + info[document-symbols]: SymbolInfo + --> main.py:16:14 + | + 16 | self.instance_var = 0 + | ^^^^^^^^^^^^ + info: Field instance_var + info[document-symbols]: SymbolInfo --> main.py:24:5 | diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index 541939051e..f7253b9b79 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -158,6 +158,22 @@ impl FlatSymbols { .filter(|(_, symbol)| query.is_match_symbol(symbol)) } + /// The names of the symbols the symbol `id` is nested in, outermost first: `["Circle"]` for + /// a method of `Circle`. + pub(crate) fn container_names(&self, id: SymbolId) -> Vec<&str> { + let mut names = Vec::new(); + let mut parent = self.symbols.get(id).and_then(|symbol| symbol.parent); + while let Some(id) = parent { + let Some(symbol) = self.symbols.get(id) else { + break; + }; + names.push(symbol.name.as_str()); + parent = symbol.parent; + } + names.reverse(); + names + } + /// Turns this flat sequence of symbols into a hierarchy of symbols. pub fn to_hierarchical(&self) -> HierarchicalSymbols { let mut children_ids: IndexVec> = IndexVec::new(); @@ -1239,6 +1255,72 @@ impl<'db> SymbolVisitor<'db> { name.chars().all(|c| c.is_ascii_uppercase() || c == '_') } + /// Adds the attributes `class_def`'s methods assign on their receiver, `self.radius` in + /// `self.radius = radius`, as fields of the class: a reader looks for them among its members, + /// where the class body itself never declares them. The first assignment of each is the + /// declaration, and a name the class body declares is its own already. + fn add_instance_attributes(&mut self, class_id: SymbolId, class_def: &'db ast::StmtClassDef) { + let mut seen: FxHashSet = self + .symbols + .iter() + .filter(|symbol| symbol.parent == Some(class_id)) + .map(|symbol| symbol.name.clone()) + .collect(); + // basedpython: what a property construct is lowered into, its backing field among it, + // stands for source the property already covers + let constructs: Vec = class_def + .body + .iter() + .filter_map(|stmt| stmt.as_function_def_stmt()?.property_construct_range()) + .collect(); + for stmt in &class_def.body { + let ast::Stmt::FunctionDef(method) = stmt else { + continue; + }; + if constructs + .iter() + .any(|construct| construct.contains_range(stmt.range())) + { + continue; + } + // a static method's first parameter is not the instance + let is_static = method.decorator_list.iter().any(|decorator| { + UnqualifiedName::from_expr(&decorator.expression) + .is_some_and(|name| name.segments().last() == Some(&"staticmethod")) + }); + if is_static { + continue; + } + let Some(receiver) = method + .parameters + .posonlyargs + .first() + .or_else(|| method.parameters.args.first()) + else { + continue; + }; + let mut collector = ReceiverAttributes { + receiver: receiver.name().as_str(), + current_stmt: None, + found: Vec::new(), + }; + collector.visit_body(&method.body); + for (name, name_range, full_range) in collector.found { + if seen.insert(name.to_string()) { + self.add_symbol(SymbolTree { + parent: None, + name: name.to_string(), + kind: SymbolKind::Field, + deprecated: false, + name_range, + full_range, + imported_from: None, + }); + } + } + } + } + fn has_deprecated_decorator(decorator_list: &[ast::Decorator]) -> bool { decorator_list.iter().any(|decorator| { let expr = match &decorator.expression { @@ -1412,6 +1494,7 @@ impl<'db> SymbolVisitor<'db> { self.symbol_stack.push(symbol_id); self.visit_body(&class_def.body); + self.add_instance_attributes(symbol_id, class_def); self.pop_symbol(); // Restore the previous class scope state @@ -1691,6 +1774,43 @@ fn create_all_name(expr: &ast::Expr) -> Option { Some(expr.as_string_literal_expr()?.value.to_str().into()) } +/// The attributes a method assigns on its receiver, each with the statement that assigns it, in +/// the order they are assigned. A nested function or class has receivers of its own. +struct ReceiverAttributes<'a> { + receiver: &'a str, + current_stmt: Option, + found: Vec<(&'a Name, TextRange, TextRange)>, +} + +impl<'a> SourceOrderVisitor<'a> for ReceiverAttributes<'a> { + fn visit_stmt(&mut self, stmt: &'a ast::Stmt) { + if matches!(stmt, ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_)) { + return; + } + let previous = self.current_stmt.replace(stmt.range()); + source_order::walk_stmt(self, stmt); + self.current_stmt = previous; + } + + fn visit_expr(&mut self, expr: &'a ast::Expr) { + match expr { + ast::Expr::Lambda(_) => {} + ast::Expr::Attribute(attribute) + if attribute.ctx.is_store() + && attribute + .value + .as_name_expr() + .is_some_and(|name| name.id.as_str() == self.receiver) + && let Some(stmt) = self.current_stmt => + { + self.found + .push((&attribute.attr.id, attribute.attr.range(), stmt)); + } + _ => source_order::walk_expr(self, expr), + } + } +} + #[cfg(test)] mod tests { use camino::Utf8Component; diff --git a/crates/ty_ide/src/workspace_symbols.rs b/crates/ty_ide/src/workspace_symbols.rs index eab323d7d6..b67e365428 100644 --- a/crates/ty_ide/src/workspace_symbols.rs +++ b/crates/ty_ide/src/workspace_symbols.rs @@ -2,10 +2,19 @@ use crate::django_template::django_workspace_symbols; use crate::symbols::{QueryPattern, SymbolInfo, symbols_for_file}; use rayon::prelude::*; use ruff_db::files::File; +use ty_module_resolver::file_to_module; use ty_project::{Db, parallel::ParallelIteratorExt}; /// Get all workspace symbols matching the query string. /// Returns symbols from all files in the workspace, filtered by the query. +/// +/// A query with dots in it is a qualified name: its last part is matched against the symbol's +/// name, and the parts before it against what the symbol is declared in, the module's dotted +/// name and the classes around it. `Shape.describe` finds `describe` in the class `Shape`, and +/// `base.Sq` the class `Square` of the module `shapes.base`. Each part matches as the whole query +/// would, as a subsequence of its letters; the last of the parts before the name matches what +/// the symbol is directly in, and the rest match, in order, what that is in. A name with dots of +/// its own, a django template's `base.html`, takes as many of the query's parts as it has. pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { // If the query is empty, return immediately to avoid expensive file scanning if query.is_empty() { @@ -15,8 +24,15 @@ pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { let workspace_symbols_span = tracing::debug_span!("workspace_symbols"); let _span = workspace_symbols_span.enter(); + let (containers_query, name_query) = query.rsplit_once('.').unwrap_or(("", query)); + let containers_query: Vec = containers_query + .split('.') + .filter(|part| !part.is_empty()) + .map(QueryPattern::fuzzy) + .collect(); + let name_query = QueryPattern::fuzzy(name_query); + let project = db.project(); - let query = QueryPattern::fuzzy(query); let files = project.files(db); let files: Vec<_> = files.iter().collect(); @@ -30,12 +46,27 @@ pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { ); let _entered = symbols_for_file_span.entered(); - symbols_for_file(db, db.program_file(file)) - .search(&query) - .map(|(_, symbol)| WorkspaceSymbolInfo { - symbol: symbol.to_owned(), - file, - container: None, + let program_file = db.program_file(file); + let module = file_to_module(db, program_file.resolver_file(db)) + .map(|module| module.name(db).to_string()); + let symbols = symbols_for_file(db, program_file); + symbols + .search(&name_query) + .filter_map(|(id, symbol)| { + let mut containers: Vec<&str> = module + .as_deref() + .map(|module| module.split('.').collect()) + .unwrap_or_default(); + containers.extend(symbols.container_names(id)); + if !containers_match(&containers_query, &containers) { + return None; + } + Some(WorkspaceSymbolInfo { + symbol: symbol.to_owned(), + file, + declared_in: (!containers.is_empty()).then(|| containers.join(".")), + container: None, + }) }) .collect::>() }) @@ -44,13 +75,20 @@ pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { // a django project is made of things python has no symbol for — a route // name, a template — and of classes python offers without saying what django - // does with them. both are added to the python answer rather than changing it + // does with them. both are added to the python answer rather than changing it. + // what django names is declared in nothing a query can name, and its own name + // can have dots in it, a template's `base.html`: the query's parts are the + // name's, matched against the whole of it as an undotted query is against a + // python name, while there are no more of them than the name has + let query_parts = query.split('.').count(); found.extend( - django_workspace_symbols(db, &query) + django_workspace_symbols(db, &QueryPattern::fuzzy(query)) .into_iter() + .filter(|django| django.symbol.name.split('.').count() >= query_parts) .map(|django| WorkspaceSymbolInfo { symbol: django.symbol, file: django.file, + declared_in: None, container: Some(django.container), }), ); @@ -58,6 +96,31 @@ pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { found } +/// Whether the parts of a qualified query before the name match `containers`, what a symbol is +/// declared in, outermost first: the last part matches the innermost container, and each part +/// before it a container further out than the one the part after it matched. +fn containers_match(query: &[QueryPattern], containers: &[&str]) -> bool { + let Some((innermost, outer)) = query.split_last() else { + return true; + }; + let Some((&direct, mut remaining)) = containers.split_last() else { + return false; + }; + if !innermost.is_match_symbol_name(direct) { + return false; + } + for part in outer.iter().rev() { + let Some(position) = remaining + .iter() + .rposition(|container| part.is_match_symbol_name(container)) + else { + return false; + }; + remaining = &remaining[..position]; + } + true +} + /// A symbol found in the workspace, including the file it was found in. #[derive(Debug, Clone, PartialEq, Eq)] pub struct WorkspaceSymbolInfo { @@ -65,6 +128,10 @@ pub struct WorkspaceSymbolInfo { pub symbol: SymbolInfo<'static>, /// The file containing the symbol pub file: File, + /// What a python symbol is declared in: the module's dotted name and the classes around it, + /// `shapes.base.Circle` for a method of `Circle`. `None` for a symbol of a file no module is + /// imported from, and for a symbol django contributed. + pub declared_in: Option, /// what kind of django thing this is, for a symbol django contributed /// /// `None` for everything python contributes, which is every symbol that was @@ -75,6 +142,7 @@ pub struct WorkspaceSymbolInfo { #[cfg(test)] mod tests { use super::*; + use crate::django_template::tests::{DJANGO_ADMIN, TemplateTest}; use crate::tests::CursorTest; use crate::tests::IntoDiagnostic; use insta::assert_snapshot; @@ -117,7 +185,7 @@ API_BASE_URL = 'https://api.example.com' | 2 | def utility_function(): | ^^^^^^^^^^^^^^^^ - info: Function utility_function + info: Function utility_function in utils "); assert_snapshot!(test.workspace_symbols("data"), @" @@ -126,7 +194,7 @@ API_BASE_URL = 'https://api.example.com' | 2 | class DataModel: | ^^^^^^^^^ - info: Class DataModel + info: Class DataModel in models "); assert_snapshot!(test.workspace_symbols("apibase"), @" @@ -135,7 +203,7 @@ API_BASE_URL = 'https://api.example.com' | 2 | API_BASE_URL = 'https://api.example.com' | ^^^^^^^^^^^^ - info: Constant API_BASE_URL + info: Constant API_BASE_URL in constants "); } @@ -157,7 +225,7 @@ class Test: | 3 | def from_path(): ... | ^^^^^^^^^ - info: Method from_path + info: Method from_path in utils.Test "); } @@ -180,7 +248,7 @@ class Test: | 4 | def from_path(): ... | ^^^^^^^^^ - info: Method from_path + info: Method from_path in utils.Test "); } @@ -204,13 +272,109 @@ foo = 1 | 5 | foo = 1 | ^^^ - info: Variable foo + info: Variable foo in utils "); assert_snapshot!(test.workspace_symbols("re"), @"No symbols found"); assert_snapshot!(test.workspace_symbols("json"), @"No symbols found"); assert_snapshot!(test.workspace_symbols("default"), @"No symbols found"); } + /// a query with dots in it names what the symbol is declared in, and an attribute a method + /// assigns on `self` is a member of the class + #[test] + fn qualified_queries_and_instance_attributes() { + let test = CursorTest::builder() + .source( + "shapes/base.py", + " +class Shape: + def describe(self) -> str: + return '' + +class Circle(Shape): + def __init__(self, radius: float) -> None: + self.radius = radius + + def describe(self) -> str: + return '' +", + ) + .source("main.py", "") + .build(); + + assert_snapshot!(test.workspace_symbols("Shape.describe"), @" + info[workspace-symbols]: WorkspaceSymbolInfo + --> shapes/base.py:3:9 + | + 3 | def describe(self) -> str: + | ^^^^^^^^ + info: Method describe in shapes.base.Shape + "); + assert_snapshot!(test.workspace_symbols("base.Circ"), @" + info[workspace-symbols]: WorkspaceSymbolInfo + --> shapes/base.py:6:7 + | + 6 | class Circle(Shape): + | ^^^^^^ + info: Class Circle in shapes.base + "); + assert_snapshot!(test.workspace_symbols("radius"), @" + info[workspace-symbols]: WorkspaceSymbolInfo + --> shapes/base.py:8:14 + | + 8 | self.radius = radius + | ^^^^^^ + info: Field radius in shapes.base.Circle + "); + assert_snapshot!(test.workspace_symbols("other.describe"), @"No symbols found"); + } + + /// a django template's name has dots of its own, so a dotted query is matched against the + /// whole of it, as an undotted query is matched against a python name + #[test] + fn a_dotted_query_finds_a_django_template() { + let test = TemplateTest::with_site_packages( + &[ + ( + "manage.py", + " + import os + + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') + ", + ), + ("project/__init__.py", ""), + ( + "project/settings.py", + " + INSTALLED_APPS = ['blog'] + + TEMPLATES = [{'DIRS': [], 'APP_DIRS': True, 'OPTIONS': {}}] + ", + ), + ("blog/__init__.py", ""), + ("blog/templates/base.html", "a"), + ("blog/templates/blog/post.html", "b"), + ], + DJANGO_ADMIN, + ); + + assert_eq!( + test.workspace_symbols("base.html"), + ["django template base.html [/src/blog/templates/base.html]"] + ); + assert_eq!( + test.workspace_symbols("post.ht"), + ["django template blog/post.html [/src/blog/templates/blog/post.html]"] + ); + // a part more than the name has would be what the template is declared in, and a + // template is declared in nothing + assert_eq!( + test.workspace_symbols("templates.base.html"), + Vec::::new() + ); + } + impl CursorTest { fn workspace_symbols(&self, query: &str) -> String { let symbols = workspace_symbols(&self.db, query); @@ -237,7 +401,11 @@ foo = 1 fn into_diagnostic(self) -> Diagnostic { let symbol_kind_str = self.symbol_info.symbol.kind.to_string(); - let info_text = format!("{} {}", symbol_kind_str, self.symbol_info.symbol.name); + let mut info_text = format!("{} {}", symbol_kind_str, self.symbol_info.symbol.name); + if let Some(declared_in) = &self.symbol_info.declared_in { + info_text.push_str(" in "); + info_text.push_str(declared_in); + } let sub = SubDiagnostic::new(SubDiagnosticSeverity::Info, info_text); diff --git a/crates/ty_server/src/server/api/requests/workspace_symbols.rs b/crates/ty_server/src/server/api/requests/workspace_symbols.rs index 75f03f988b..b8d5f4859d 100644 --- a/crates/ty_server/src/server/api/requests/workspace_symbols.rs +++ b/crates/ty_server/src/server/api/requests/workspace_symbols.rs @@ -40,15 +40,20 @@ impl BackgroundRequestHandler for WorkspaceSymbolRequestHandler { let WorkspaceSymbolInfo { symbol, file, + declared_in, container, } = workspace_symbol_info; // Get position encoding from session let encoding = snapshot.position_encoding(); - let Some(symbol) = - convert_to_lsp_symbol_information(db, file, symbol, container, encoding) - else { + let Some(symbol) = convert_to_lsp_symbol_information( + db, + file, + symbol, + container.or(declared_in.as_deref()), + encoding, + ) else { tracing::debug!( "Failed to convert symbol '{}' to LSP symbol information", file.path(db) diff --git a/crates/ty_server/src/server/api/symbols.rs b/crates/ty_server/src/server/api/symbols.rs index 06f7e87da4..6e946dcb5f 100644 --- a/crates/ty_server/src/server/api/symbols.rs +++ b/crates/ty_server/src/server/api/symbols.rs @@ -27,8 +27,9 @@ pub(crate) fn convert_symbol_kind(kind: ty_ide::SymbolKind) -> SymbolKind { /// Convert a `ty_ide` `SymbolInfo` to LSP `SymbolInformation` /// -/// `container` is what the symbol belongs to, which is how a django route or -/// model says what it is beside the python symbol of the same name. +/// `container` is what the symbol belongs to: for a python symbol the module and the classes it +/// is declared in, `shapes.base.Circle`, and for a django route or model what it is, which is +/// how it says so beside the python symbol of the same name. /// /// Returns `None` if the symbol's range cannot be converted to a location /// (e.g., if the file cannot be converted to a URI). diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 4fde29b571..f69d20c3a7 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -332,6 +332,27 @@ a [property](properties.md) is one member in the source, so it is one entry in the outline — not the getter, backing field and setter it lowers into. enum variants and an extension's methods appear under their declarations +an attribute a method assigns on its receiver, `self.radius = radius`, is a +field of the class, listed after the members the class body declares. the first +assignment is its declaration, and a name the class body declares already is not +listed again + +## workspace symbols + +`workspace/symbol` gives each python symbol the module and the classes it is +declared in as its `containerName`, `shapes.base.Circle` for the method `area` +of `Circle`, so that the three `area`s of a class hierarchy can be told apart. +the attributes a method assigns on `self` are found as fields of their class + +a query with dots in it is a qualified name. its last part is matched against +the symbol's name and the parts before it against what the symbol is declared +in: `Shape.describe` finds `describe` in the class `Shape`, and `base.Sq` the +class `Square` of the module `shapes.base`. every part matches the way a whole +query does, as a subsequence of its letters. the last part before the name +matches what the symbol is directly in, and each part before that something +further out. a name with dots of its own, like a template's `base.html`, takes as +many of the query's parts as it has + ## folding `textDocument/foldingRange` gives every range a kind saying what it folds, so From 074b19ed5275e029fa268969da74e50ae8a11965 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:03:00 +1000 Subject: [PATCH 015/110] give each document symbol a detail, a function's parameters and return annotation and a class's bases as the source writes them on one line, and outline a @property or cached_property def as a property whose setter and deleter are the same member --- crates/ruff_python_codegen/src/generator.rs | 105 +++++++---- crates/ty_ide/src/document_symbols.rs | 177 +++++++++++++++++- crates/ty_ide/src/lib.rs | 2 +- crates/ty_ide/src/symbols.rs | 103 +++++++--- crates/ty_ide/src/workspace_symbols.rs | 16 +- .../server/api/requests/document_symbols.rs | 5 +- crates/ty_server/src/server/api/symbols.rs | 2 +- docs/basedpython/features/editor.md | 8 + 8 files changed, 348 insertions(+), 70 deletions(-) diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index 1ee8a07955..9c11e4a100 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -183,6 +183,24 @@ impl<'a> Generator<'a> { self.generate() } + /// basedpython: generate the parameter list and return annotation of a `def`, + /// `(self, radius: float) -> None`, as its header writes them + pub fn signature(mut self, function: &ast::StmtFunctionDef) -> String { + self.unparse_signature( + &function.parameters, + function.returns.as_deref(), + function.is_asserts_return, + ); + self.generate() + } + + /// basedpython: generate a class's bases and keywords, `(Shape, metaclass=Meta)`, as its + /// header writes them + pub fn class_arguments(mut self, arguments: &ast::Arguments) -> String { + self.unparse_class_arguments(arguments); + self.generate() + } + fn newline(&mut self) { if !self.initial { self.num_newlines = std::cmp::max(self.num_newlines, 1); @@ -405,21 +423,7 @@ impl<'a> Generator<'a> { if let Some(type_params) = type_params { self.unparse_type_params(type_params); } - self.p("("); - self.unparse_parameters(parameters); - self.p(")"); - if *is_asserts_return { - // basedpython: `-> asserts x` names a place, not a type. the - // generator emits python, and such a function returns `None` - self.p(" -> None"); - } else if let Some(returns) = returns { - self.p(" -> "); - // render at the same precedence as a variable annotation - // so a union return type stays unparenthesised - // (`-> int | None`) while a bare tuple still groups - // (`-> (int, str)`) - self.unparse_expr(returns, precedence::COMMA); - } + self.unparse_signature(parameters, returns.as_deref(), *is_asserts_return); self.p(":"); }); self.body(body); @@ -450,27 +454,7 @@ impl<'a> Generator<'a> { self.unparse_type_params(type_params); } if let Some(arguments) = arguments { - self.p("("); - let mut first = true; - for arg_or_keyword in arguments.iter_source_order() { - match arg_or_keyword { - ArgOrKeyword::Arg(arg) => { - self.p_delim(&mut first, ", "); - self.unparse_expr(arg, precedence::MAX); - } - ArgOrKeyword::Keyword(keyword) => { - self.p_delim(&mut first, ", "); - if let Some(arg) = &keyword.arg { - self.p_keyword_name(arg, keyword.key); - self.p("="); - } else { - self.p("**"); - } - self.unparse_expr(&keyword.value, precedence::MAX); - } - } - } - self.p(")"); + self.unparse_class_arguments(arguments); } self.p(":"); }); @@ -1916,6 +1900,55 @@ impl<'a> Generator<'a> { } } + /// a `def`'s parameter list and return annotation, `(self, radius: float) -> None` + fn unparse_signature( + &mut self, + parameters: &Parameters, + returns: Option<&Expr>, + is_asserts_return: bool, + ) { + self.p("("); + self.unparse_parameters(parameters); + self.p(")"); + if is_asserts_return { + // basedpython: `-> asserts x` names a place, not a type. the + // generator emits python, and such a function returns `None` + self.p(" -> None"); + } else if let Some(returns) = returns { + self.p(" -> "); + // render at the same precedence as a variable annotation + // so a union return type stays unparenthesised + // (`-> int | None`) while a bare tuple still groups + // (`-> (int, str)`) + self.unparse_expr(returns, precedence::COMMA); + } + } + + /// a class's bases and keywords, `(Shape, metaclass=Meta)` + fn unparse_class_arguments(&mut self, arguments: &ast::Arguments) { + self.p("("); + let mut first = true; + for arg_or_keyword in arguments.iter_source_order() { + match arg_or_keyword { + ArgOrKeyword::Arg(arg) => { + self.p_delim(&mut first, ", "); + self.unparse_expr(arg, precedence::MAX); + } + ArgOrKeyword::Keyword(keyword) => { + self.p_delim(&mut first, ", "); + if let Some(arg) = &keyword.arg { + self.p_keyword_name(arg, keyword.key); + self.p("="); + } else { + self.p("**"); + } + self.unparse_expr(&keyword.value, precedence::MAX); + } + } + } + self.p(")"); + } + fn unparse_parameters(&mut self, parameters: &Parameters) { let mut first = true; for (i, parameter_with_default) in parameters diff --git a/crates/ty_ide/src/document_symbols.rs b/crates/ty_ide/src/document_symbols.rs index 766f8d2a91..494c8f71fa 100644 --- a/crates/ty_ide/src/document_symbols.rs +++ b/crates/ty_ide/src/document_symbols.rs @@ -1,4 +1,10 @@ -use crate::symbols::{FlatSymbols, symbols_for_file}; +use crate::symbols::{FlatSymbols, SymbolInfo, symbols_for_file}; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_python_ast::AnyNodeRef; +use ruff_python_ast::find_node::covering_node; +use ruff_python_codegen::{Generator, Mode, Stylist}; +use ruff_text_size::Ranged; use ty_project::Db; use ty_python_core::ProgramFile; @@ -7,6 +13,37 @@ pub fn document_symbols<'db>(db: &'db dyn Db, file: ProgramFile<'db>) -> &'db Fl symbols_for_file(db, file) } +/// what an outline shows beside a symbol's name: a function's parameters and return annotation, +/// `(self, radius: float) -> None`, and a class's bases, `(Shape)`, each printed from the syntax +/// tree on one line, a `.by` file's in basedpython's syntax. `None` for any other symbol, and for +/// a class without bases +pub fn symbol_detail( + db: &dyn Db, + file: ProgramFile<'_>, + symbol: &SymbolInfo<'_>, +) -> Option { + let module = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let stylist = Stylist::from_tokens(module.tokens(), source.as_str()); + let mode = if file.file(db).source_type(db).is_basedpython() { + Mode::BasedPython + } else { + Mode::Default + }; + let generator = || Generator::from(&stylist).with_mode(mode); + let covering = covering_node(module.syntax().into(), symbol.name_range); + covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtFunctionDef(function) if function.name.range() == symbol.name_range => { + Some(generator().signature(function)) + } + AnyNodeRef::StmtClassDef(class) if class.name.range() == symbol.name_range => class + .arguments + .as_ref() + .map(|arguments| generator().class_arguments(arguments)), + _ => None, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -18,6 +55,7 @@ mod tests { SubDiagnosticSeverity, }; use ruff_db::files::File; + use std::fmt::Write as _; /// basedpython: an accessor property is one member in the source, so it is /// one symbol in the outline — not the getter, backing field and setter the @@ -71,6 +109,42 @@ class P: ); } + /// basedpython: the attributes a method assigns on its first parameter are members of the + /// class only when that parameter is the receiver, which a `static def`'s is not; a + /// classmethod's receiver is the class, so what it assigns is a class attribute + #[test] + fn receiver_attributes_follow_the_method_kind() { + let test = CursorTest::builder() + .source( + "main.by", + "\ +class Seed: + pass + +class Counter: + def __init__(self, start: int) -> None: + self.value = start + + static def make(seed: Seed) -> None: + seed.planted = True + + class def reset(cls) -> None: + cls.total = 0 +", + ) + .build(); + + assert_snapshot!(test.outline_with_details(), @" + Class Seed + Class Counter + Constructor __init__(self, start: int) -> None + Method make(seed: Seed) -> None + Method reset(cls) -> None + Field value + Field total + "); + } + #[test] fn test_document_symbols_simple() { let test = cursor_test( @@ -656,7 +730,108 @@ class Example((class_base := Base)): ); } + /// a function's outline entry carries its signature as written and a class's its bases, a + /// `@property` is a property, and its setter is the same property rather than a second member + #[test] + fn details_and_properties() { + let test = cursor_test( + " +class Shape(ABC, metaclass=Meta): + def shifted( + self, + dx: int, + dy: int, + ) -> 'Shape': + return self + + @property + def diameter(self) -> float: + return self._d + + @diameter.setter + def diameter(self, value: float) -> None: + self._d = value + + @functools.cached_property + def area(self): + return 1 + +class Plain: + pass +", + ); + + assert_snapshot!(test.outline_with_details(), @" + Class Shape(ABC, metaclass=Meta) + Method shifted(self, dx: int, dy: int) -> 'Shape' + Property diameter(self) -> float + Property area(self) + Field _d + Class Plain + "); + } + + /// a detail is printed from the syntax tree, so a comment in a signature written over several + /// lines is left out, and a string default keeps the spaces inside it + #[test] + fn details_are_printed_from_the_syntax_tree() { + let test = cursor_test( + " +def greet( + name: str = 'a b', # who to greet + *, + loud: bool, +) -> str: + return name + +class Loud( # the base + str, +): + pass +", + ); + + assert_snapshot!(test.outline_with_details(), @" + Function greet(name: str='a b', *, loud: bool) -> str + Class Loud(str) + "); + } + impl CursorTest { + /// the outline as one line per symbol: its kind, its name and its detail, indented + /// under what it is in + fn outline_with_details(&self) -> String { + fn walk( + test: &CursorTest, + symbols: &HierarchicalSymbols, + id: SymbolId, + symbol: &SymbolInfo<'_>, + depth: usize, + out: &mut String, + ) { + let file = test.program_file(test.cursor.file); + let detail = symbol_detail(&test.db, file, symbol).unwrap_or_default(); + let _ = writeln!( + out, + "{}{} {}{}", + " ".repeat(depth), + symbol.kind.to_string(), + symbol.name, + detail + ); + for (child_id, child) in symbols.children(id) { + walk(test, symbols, child_id, &child, depth + 1, out); + } + } + let symbols = + document_symbols(&self.db, self.program_file(self.cursor.file)).to_hierarchical(); + let mut out = String::new(); + for (id, symbol) in symbols.iter() { + walk(self, &symbols, id, &symbol, 0, &mut out); + } + out + } + fn document_symbols(&self) -> String { let symbols = document_symbols(&self.db, self.program_file(self.cursor.file)).to_hierarchical(); diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 9f60eff0d6..c7fb117522 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -80,7 +80,7 @@ pub use django_template::{ django_template_semantic_tokens, django_template_signature_help, is_django_template_path, }; pub use doc_highlights::document_highlights; -pub use document_symbols::document_symbols; +pub use document_symbols::{document_symbols, symbol_detail}; pub use evaluatable_expression::evaluatable_expression; pub use find_references::find_references; pub use folding_range::{FoldingRange, FoldingRangeKind, folding_ranges}; diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index f7253b9b79..8d10f64abd 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -17,6 +17,9 @@ use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ImportingFile, ModuleName, resolve_module}; use ty_project::Db; use ty_python_core::ProgramFile; +use ty_python_semantic::types::MethodDecorator; +use ty_python_semantic::types::ide_support::decorated_function; +use ty_python_semantic::{HasDefinition, SemanticModel}; use crate::completion::CompletionKind; @@ -158,8 +161,8 @@ impl FlatSymbols { .filter(|(_, symbol)| query.is_match_symbol(symbol)) } - /// The names of the symbols the symbol `id` is nested in, outermost first: `["Circle"]` for - /// a method of `Circle`. + /// the names of the symbols the symbol `id` is nested in, outermost first: `["Circle"]` for + /// a method of `Circle` pub(crate) fn container_names(&self, id: SymbolId) -> Vec<&str> { let mut names = Vec::new(); let mut parent = self.symbols.get(id).and_then(|symbol| symbol.parent); @@ -1255,10 +1258,12 @@ impl<'db> SymbolVisitor<'db> { name.chars().all(|c| c.is_ascii_uppercase() || c == '_') } - /// Adds the attributes `class_def`'s methods assign on their receiver, `self.radius` in + /// adds the attributes `class_def`'s methods assign on their receiver, `self.radius` in /// `self.radius = radius`, as fields of the class: a reader looks for them among its members, - /// where the class body itself never declares them. The first assignment of each is the - /// declaration, and a name the class body declares is its own already. + /// where the class body itself never declares them. a classmethod's receiver is the class, + /// and what it assigns is a class attribute, a field as one the class body assigns is; a + /// static method has no receiver. the first assignment of each is the declaration, and a + /// name the class body declares is its own already fn add_instance_attributes(&mut self, class_id: SymbolId, class_def: &'db ast::StmtClassDef) { let mut seen: FxHashSet = self .symbols @@ -1283,14 +1288,6 @@ impl<'db> SymbolVisitor<'db> { { continue; } - // a static method's first parameter is not the instance - let is_static = method.decorator_list.iter().any(|decorator| { - UnqualifiedName::from_expr(&decorator.expression) - .is_some_and(|name| name.segments().last() == Some(&"staticmethod")) - }); - if is_static { - continue; - } let Some(receiver) = method .parameters .posonlyargs @@ -1305,6 +1302,13 @@ impl<'db> SymbolVisitor<'db> { found: Vec::new(), }; collector.visit_body(&method.body); + // the semantic model is asked for the method's kind only when its body assigns on + // its first parameter: a method that assigns nothing adds nothing whatever its kind + if collector.found.is_empty() + || matches!(self.method_kind(method), MethodDecorator::StaticMethod) + { + continue; + } for (name, name_range, full_range) in collector.found { if seen.insert(name.to_string()) { self.add_symbol(SymbolTree { @@ -1321,6 +1325,45 @@ impl<'db> SymbolVisitor<'db> { } } + /// whether `method` is an instance method, a classmethod or a static method, as the semantic + /// model reads its decorators, `@staticmethod`, basedpython's `static def`, or an implicit + /// one such as `__new__`'s. A method the model calls both is read as an instance method + fn method_kind(&self, method: &ast::StmtFunctionDef) -> MethodDecorator { + let model = SemanticModel::new(self.db, self.file); + decorated_function(self.db, method.definition(&model)) + .and_then(|function| function.undecorated.as_function_literal()) + .and_then(|function| MethodDecorator::try_from_fn_type(self.db, function)) + .unwrap_or_default() + } + + /// what a `property` decorator among `decorator_list` makes of the `def`: a property, as + /// `@property` and `@functools.cached_property` do, or another accessor of one, as + /// `@radius.setter` does + fn property_decorator(decorator_list: &[ast::Decorator]) -> Option> { + decorator_list.iter().find_map(|decorator| { + let name = UnqualifiedName::from_expr(&decorator.expression)?; + match name.segments() { + [.., "property" | "cached_property"] => Some(PropertyDecorator::Property), + [property, "setter" | "getter" | "deleter"] => { + Some(PropertyDecorator::Accessor(property)) + } + _ => None, + } + }) + } + + /// whether the class being walked has listed a property named `name` + fn class_lists_property(&self, name: &str) -> bool { + let Some(&class) = self.symbol_stack.last() else { + return false; + }; + self.symbols.iter().any(|symbol| { + symbol.parent == Some(class) + && symbol.kind == SymbolKind::Property + && symbol.name == name + }) + } + fn has_deprecated_decorator(decorator_list: &[ast::Decorator]) -> bool { decorator_list.iter().any(|decorator| { let expr = match &decorator.expression { @@ -1398,12 +1441,23 @@ impl<'db> SymbolVisitor<'db> { }); return; } - let kind = SymbolKind::function_kind( - &func_def.name, - self.iter_symbol_stack() - .last() - .is_some_and(|tree| tree.kind == SymbolKind::Class), - ); + let in_class = self + .iter_symbol_stack() + .last() + .is_some_and(|tree| tree.kind == SymbolKind::Class); + let kind = match Self::property_decorator(&func_def.decorator_list) { + Some(PropertyDecorator::Property) if in_class => SymbolKind::Property, + // `@radius.setter` declares no member of its own: it is the property + // `radius` the class has already listed, given another accessor + Some(PropertyDecorator::Accessor(property)) + if in_class + && property == func_def.name.as_str() + && self.class_lists_property(property) => + { + return; + } + _ => SymbolKind::function_kind(&func_def.name, in_class), + }; let symbol = SymbolTree { parent: None, @@ -1774,8 +1828,15 @@ fn create_all_name(expr: &ast::Expr) -> Option { Some(expr.as_string_literal_expr()?.value.to_str().into()) } -/// The attributes a method assigns on its receiver, each with the statement that assigns it, in -/// the order they are assigned. A nested function or class has receivers of its own. +/// what a decorator makes of the `def` it decorates, when it is a property's +enum PropertyDecorator<'a> { + Property, + /// another accessor of the property with this name + Accessor(&'a str), +} + +/// the attributes a method assigns on its receiver, each with the statement that assigns it, in +/// the order they are assigned. a nested function or class has receivers of its own struct ReceiverAttributes<'a> { receiver: &'a str, current_stmt: Option, diff --git a/crates/ty_ide/src/workspace_symbols.rs b/crates/ty_ide/src/workspace_symbols.rs index b67e365428..8311946de6 100644 --- a/crates/ty_ide/src/workspace_symbols.rs +++ b/crates/ty_ide/src/workspace_symbols.rs @@ -8,13 +8,13 @@ use ty_project::{Db, parallel::ParallelIteratorExt}; /// Get all workspace symbols matching the query string. /// Returns symbols from all files in the workspace, filtered by the query. /// -/// A query with dots in it is a qualified name: its last part is matched against the symbol's +/// a query with dots in it is a qualified name: its last part is matched against the symbol's /// name, and the parts before it against what the symbol is declared in, the module's dotted /// name and the classes around it. `Shape.describe` finds `describe` in the class `Shape`, and -/// `base.Sq` the class `Square` of the module `shapes.base`. Each part matches as the whole query +/// `base.Sq` the class `Square` of the module `shapes.base`. each part matches as the whole query /// would, as a subsequence of its letters; the last of the parts before the name matches what -/// the symbol is directly in, and the rest match, in order, what that is in. A name with dots of -/// its own, a django template's `base.html`, takes as many of the query's parts as it has. +/// the symbol is directly in, and the rest match, in order, what that is in. a name with dots of +/// its own, a django template's `base.html`, takes as many of the query's parts as it has pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { // If the query is empty, return immediately to avoid expensive file scanning if query.is_empty() { @@ -96,9 +96,9 @@ pub fn workspace_symbols(db: &dyn Db, query: &str) -> Vec { found } -/// Whether the parts of a qualified query before the name match `containers`, what a symbol is +/// whether the parts of a qualified query before the name match `containers`, what a symbol is /// declared in, outermost first: the last part matches the innermost container, and each part -/// before it a container further out than the one the part after it matched. +/// before it a container further out than the one the part after it matched fn containers_match(query: &[QueryPattern], containers: &[&str]) -> bool { let Some((innermost, outer)) = query.split_last() else { return true; @@ -128,9 +128,9 @@ pub struct WorkspaceSymbolInfo { pub symbol: SymbolInfo<'static>, /// The file containing the symbol pub file: File, - /// What a python symbol is declared in: the module's dotted name and the classes around it, + /// what a python symbol is declared in: the module's dotted name and the classes around it, /// `shapes.base.Circle` for a method of `Circle`. `None` for a symbol of a file no module is - /// imported from, and for a symbol django contributed. + /// imported from, and for a symbol django contributed pub declared_in: Option, /// what kind of django thing this is, for a symbol django contributed /// diff --git a/crates/ty_server/src/server/api/requests/document_symbols.rs b/crates/ty_server/src/server/api/requests/document_symbols.rs index 53c5394c6c..ca66463fc3 100644 --- a/crates/ty_server/src/server/api/requests/document_symbols.rs +++ b/crates/ty_server/src/server/api/requests/document_symbols.rs @@ -5,7 +5,7 @@ use lsp_types::{DocumentSymbol, DocumentSymbolParams, Uri}; use ruff_db::files::File; use ty_ide::{ HierarchicalSymbols, SymbolId, SymbolInfo, TemplateSymbol, django_template_document_symbols, - document_symbols, + document_symbols, symbol_detail, }; use ty_project::{ProjectDatabase, SemanticDb as _}; @@ -192,10 +192,11 @@ fn convert_to_lsp_document_symbol( encoding: PositionEncoding, ) -> Option { let symbol_kind = convert_symbol_kind(symbol.kind); + let detail = symbol_detail(db, db.program_file(file), &symbol); Some(DocumentSymbol { name: symbol.name.into_owned(), - detail: None, + detail, kind: symbol_kind, tags: None, #[allow(deprecated)] diff --git a/crates/ty_server/src/server/api/symbols.rs b/crates/ty_server/src/server/api/symbols.rs index 6e946dcb5f..1e4e54d66a 100644 --- a/crates/ty_server/src/server/api/symbols.rs +++ b/crates/ty_server/src/server/api/symbols.rs @@ -29,7 +29,7 @@ pub(crate) fn convert_symbol_kind(kind: ty_ide::SymbolKind) -> SymbolKind { /// /// `container` is what the symbol belongs to: for a python symbol the module and the classes it /// is declared in, `shapes.base.Circle`, and for a django route or model what it is, which is -/// how it says so beside the python symbol of the same name. +/// how it says so beside the python symbol of the same name /// /// Returns `None` if the symbol's range cannot be converted to a location /// (e.g., if the file cannot be converted to a URI). diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index f69d20c3a7..0aa2d03a24 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -337,6 +337,14 @@ field of the class, listed after the members the class body declares. the first assignment is its declaration, and a name the class body declares already is not listed again +a `def` decorated `@property`, `@cached_property` or `@functools.cached_property` +is a property, and its `@diameter.setter` or `@diameter.deleter` is the same +property given another accessor, not a second member. each document symbol's +`detail` is what the source writes beside the name, on one line and printed +from the syntax tree rather than copied from the source: a function's +parameters and return annotation, `(self, radius: float) -> None`, and a class's +bases, `(Shape)` + ## workspace symbols `workspace/symbol` gives each python symbol the module and the classes it is From f0a3b5bdb8f84e69910b1ebce687cf3425b98595 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:05:22 +1000 Subject: [PATCH 016/110] answer by/expressionTypes, every expression of the statement a selection or caret is in from the innermost out with its type, for an editor's type info to offer the call and the expression around a name as well as the name --- crates/ty_ide/src/expression_types.rs | 122 ++++++++++++++++++ crates/ty_ide/src/lib.rs | 2 + crates/ty_server/src/server/api.rs | 7 + crates/ty_server/src/server/api/requests.rs | 2 + .../server/api/requests/expression_types.rs | 110 ++++++++++++++++ .../ty_server/tests/e2e/expression_types.rs | 96 ++++++++++++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 36 ++++++ 8 files changed, 376 insertions(+) create mode 100644 crates/ty_ide/src/expression_types.rs create mode 100644 crates/ty_server/src/server/api/requests/expression_types.rs create mode 100644 crates/ty_server/tests/e2e/expression_types.rs diff --git a/crates/ty_ide/src/expression_types.rs b/crates/ty_ide/src/expression_types.rs new file mode 100644 index 0000000000..423278a5df --- /dev/null +++ b/crates/ty_ide/src/expression_types.rs @@ -0,0 +1,122 @@ +//! The expressions a range is in, each with its type: what an editor's *type info* offers. +//! +//! ```py +//! total = total_area(shapes) * 2 +//! ``` +//! +//! On `shapes` the answer is `shapes`, `total_area(shapes)` and `total_area(shapes) * 2`, from the +//! innermost out, each with the type ty inferred for it: `list[Shape]`, `float`, `float`. On a +//! selection of `total_area(shapes)` it starts from that call. + +use ruff_db::parsed::parsed_module; +use ruff_python_ast::find_node::covering_node; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::ProgramFile; +use ty_python_semantic::{HasType, SemanticModel, with_display_for_file}; + +use crate::Db; + +/// An expression and its type. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExpressionType { + pub range: TextRange, + /// The type as the file would spell it: `1` in a basedpython file, `Literal[1]` in python. + pub ty: String, +} + +/// The expressions of the statement `range` is in that contain all of it, innermost first, each +/// with its type. An expression ty has no type for, such as one inside a string annotation, is +/// left out, and so is everything outside the statement: a statement has no type, and what +/// contains it is not an expression. +pub fn expression_types( + db: &dyn Db, + file: ProgramFile<'_>, + range: TextRange, +) -> Vec { + let module = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + let env = model.program_environment(); + let covering = covering_node(module.syntax().into(), range); + + with_display_for_file(db, file.file(db), || { + let mut found: Vec = Vec::new(); + for node in covering.ancestors() { + if node.is_statement() { + break; + } + let Some(expression) = node.as_expr_ref() else { + continue; + }; + if found + .last() + .is_some_and(|inner| inner.range == expression.range()) + { + continue; + } + let Some(ty) = expression.inferred_type(&model) else { + continue; + }; + found.push(ExpressionType { + range: expression.range(), + ty: ty.display(db, &env).to_string(), + }); + } + found + }) +} + +#[cfg(test)] +mod tests { + use super::expression_types; + use crate::tests::CursorTest; + use insta::assert_snapshot; + use ruff_text_size::TextRange; + + fn types(test: &CursorTest, range: TextRange) -> String { + let source = ruff_db::source::source_text(&test.db, test.cursor.file); + expression_types(&test.db, test.program_file(test.cursor.file), range) + .iter() + .map(|found| format!("{} : {}", &source.as_str()[found.range], found.ty)) + .collect::>() + .join("\n") + } + + #[test] + fn a_caret_answers_every_expression_around_it() { + let test = CursorTest::builder() + .source( + "main.py", + " +def total_area(shapes: list[int]) -> float: + return 1.0 + +def main(shapes: list[int]) -> None: + total = total_area(shapes) * 2 +", + ) + .build(); + + assert_snapshot!(types(&test, TextRange::empty(test.cursor.offset)), @" + shapes : list[int] + total_area(shapes) : int | float + total_area(shapes) * 2 : int | float + "); + } + + #[test] + fn a_selection_starts_from_the_expression_it_spans() { + let test = CursorTest::builder() + .source( + "main.py", + " +def main(text: str) -> None: + head, _, tail = text.partition(':') +", + ) + .build(); + let start = test.cursor.offset; + let range = TextRange::new(start, start + ruff_text_size::TextSize::from(19)); + + assert_snapshot!(types(&test, range), @"text.partition(':') : tuple[str, str, str]"); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index c7fb117522..cd6ad21d1e 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -19,6 +19,7 @@ mod doc_highlights; mod docstring; mod document_symbols; mod evaluatable_expression; +mod expression_types; mod find_references; mod folding_range; mod goto; @@ -82,6 +83,7 @@ pub use django_template::{ pub use doc_highlights::document_highlights; pub use document_symbols::{document_symbols, symbol_detail}; pub use evaluatable_expression::evaluatable_expression; +pub use expression_types::{ExpressionType, expression_types}; pub use find_references::find_references; pub use folding_range::{FoldingRange, FoldingRangeKind, folding_ranges}; pub use goto::{goto_declaration, goto_definition, goto_type_definition}; diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index d3e51d52f1..d72bc349de 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -243,6 +243,13 @@ pub(super) fn request(req: server::Request) -> Task { requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), + // asked while a person waits on the popup, like hover + requests::ExpressionTypesRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } // asked while a person holds the mouse over a value, like hover requests::EvaluatableExpressionRequestHandler::METHOD => { background_document_request_task::( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index de80e4b2bb..14c24b3394 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -36,6 +36,7 @@ mod evaluatable_expression; mod execute_command; mod explain_rule; mod explain_transpilation; +mod expression_types; mod folding_range; mod goto_declaration; mod goto_definition; @@ -97,6 +98,7 @@ pub(super) use evaluatable_expression::EvaluatableExpressionRequestHandler; pub(super) use execute_command::ExecuteCommand; pub(super) use explain_rule::ExplainRuleHandler; pub(super) use explain_transpilation::ExplainTranspilationHandler; +pub(super) use expression_types::ExpressionTypesRequestHandler; pub(super) use folding_range::FoldingRangeRequestHandler; pub(super) use goto_declaration::GotoDeclarationRequestHandler; pub(super) use goto_definition::GotoDefinitionRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/expression_types.rs b/crates/ty_server/src/server/api/requests/expression_types.rs new file mode 100644 index 0000000000..f065857a7f --- /dev/null +++ b/crates/ty_server/src/server/api/requests/expression_types.rs @@ -0,0 +1,110 @@ +//! `by/expressionTypes`: the expressions a range is in, each with its type +//! +//! what an editor's *type info* offers: on `shapes` in `total_area(shapes) * 2` the argument, the +//! call and the product, so that a reader can pick the one they meant. LSP has no request for it: +//! `textDocument/hover` answers for the name under the cursor, never for the call or the +//! expression around it. see [`ty_ide::expression_types`] for what is answered + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ruff_text_size::TextRange; +use ty_ide::expression_types; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum ExpressionTypesRequest {} + +impl Request for ExpressionTypesRequest { + type Params = ExpressionTypesParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/expressionTypes"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the range to answer for: the selection, or an empty range at the caret +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ExpressionTypesParams { + text_document: TextDocumentIdentifier, + range: Range, +} + +/// an expression that contains the whole range, and its type +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ExpressionType { + range: Range, + /// the type as the document's language spells it + #[serde(rename = "type")] + ty: String, +} + +pub(crate) struct ExpressionTypesRequestHandler; + +impl RequestHandler for ExpressionTypesRequestHandler { + type RequestType = ExpressionTypesRequest; +} + +impl BackgroundDocumentRequestHandler for ExpressionTypesRequestHandler { + fn document_uri(params: &ExpressionTypesParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a template; otherwise the expressions, + /// innermost first, an empty list where the range is in none + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: ExpressionTypesParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.is_django_template() { + return Ok(None); + } + + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let to_offset = |position: Position| { + position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + }; + let (Some(start), Some(end)) = (to_offset(params.range.start), to_offset(params.range.end)) + else { + return Ok(None); + }; + // a range the client sent end first is the same selection + let range = TextRange::new(start.min(end), start.max(end)); + + Ok(Some( + expression_types(db, db.program_file(file), range) + .into_iter() + .filter_map(|found| { + Some(ExpressionType { + range: found + .range + .to_lsp_range(db, file, snapshot.encoding())? + .local_range(), + ty: found.ty, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for ExpressionTypesRequestHandler {} diff --git a/crates/ty_server/tests/e2e/expression_types.rs b/crates/ty_server/tests/e2e/expression_types.rs new file mode 100644 index 0000000000..d04662147f --- /dev/null +++ b/crates/ty_server/tests/e2e/expression_types.rs @@ -0,0 +1,96 @@ +//! `by/expressionTypes` over the wire +//! +//! which expressions a range is in, and their types, is `ty_ide`'s and its unit tests cover it. +//! what these cover is the contract a client reads: a range in, and out the expressions that +//! contain it, innermost first, each with its range and its type as the document spells it + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// the request as a client sends it, in json throughout +enum ExpressionTypes {} + +impl Request for ExpressionTypes { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/expressionTypes"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const MAIN: &str = "\ +def size(text: str) -> int: + return len(text) + 1 +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, start: u32, end: u32) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "range": { + "start": { "line": 1, "character": start }, + "end": { "line": 1, "character": end }, + }, + })) +} + +fn found(start: u32, end: u32, ty: &str) -> serde_json::Value { + serde_json::json!({ + "range": { + "start": { "line": 1, "character": start }, + "end": { "line": 1, "character": end }, + }, + "type": ty, + }) +} + +#[test] +fn a_caret_answers_every_expression_around_it_innermost_first() -> Result<()> { + let mut server = server()?; + + assert_eq!( + ask(&mut server, 17, 17), + Some(serde_json::json!([ + found(15, 19, "str"), + found(11, 20, "int"), + found(11, 24, "int"), + ])) + ); + Ok(()) +} + +#[test] +fn a_selection_starts_from_the_expression_it_spans() -> Result<()> { + let mut server = server()?; + + assert_eq!( + ask(&mut server, 11, 20), + Some(serde_json::json!([ + found(11, 20, "int"), + found(11, 24, "int") + ])) + ); + Ok(()) +} + +#[test] +fn outside_an_expression_answers_nothing() -> Result<()> { + let mut server = server()?; + + assert_eq!(ask(&mut server, 0, 0), Some(serde_json::json!([]))); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 0f0d25ad9d..59e6f0bb09 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -43,6 +43,7 @@ mod declaration; mod diagnostic_snapshots; mod django_templates; mod evaluatable_expression; +mod expression_types; mod file_watching; mod folding_range; mod goto_definition; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 0aa2d03a24..12a1ac2db3 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -682,6 +682,42 @@ name. a called class is the class, not the `__init__` the call runs, and a decorated `def` is the `def`. the answer is `null` where there is no name, and for a name whose definitions declare different things +## type info + +an editor's *type info* shows the type of an expression, and which expression +is meant depends on more than the name under the cursor: on `text` in +`len(text) + 1`, a reader may be asking about `text`, about `len(text)` or about +the sum. the editor asks `by/expressionTypes` with the selection, or an empty +range at the caret: + +```json +{ + "textDocument": { "uri": "file:///src/main.by" }, + "range": { + "start": { "line": 1, "character": 17 }, + "end": { "line": 1, "character": 17 } + } +} +``` + +and the answer is every expression of the statement that contains the whole +range, innermost first, each with its `range` and its `type` as the document's +language spells it: + +```json +[ + { "range": { "start": { "line": 1, "character": 15 }, "end": { "line": 1, "character": 19 } }, "type": "str" }, + { "range": { "start": { "line": 1, "character": 11 }, "end": { "line": 1, "character": 20 } }, "type": "int" }, + { "range": { "start": { "line": 1, "character": 11 }, "end": { "line": 1, "character": 24 } }, "type": "int" } +] +``` + +with a selection the first is the smallest expression containing it, which is +the selected one when the selection is exactly an expression. an expression ty +has no type for, such as one inside a string annotation, is left out. the answer +is an empty list where the range is in no expression, a `def`'s name or a +keyword, and `null` when language services are off + ## extend selection `textDocument/selectionRange` grows a selection the way a reader reads the From 4e623ff8a9de3834e3ff09291efc9a03d3c2daeb Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:23:12 +1000 Subject: [PATCH 017/110] complete a def's name in a class body with what the class can override, writing override before the def in a .by file, and offer an import only for a name whose word the typed text starts, so ret no longer offers secrets. document that a plain dict's keys complete from the display it was bound to --- .../completion-evaluation-tasks.csv | 5 +- .../completion.toml | 2 + .../keyword-over-fuzzy-auto-import/main.py | 5 + .../pyproject.toml | 5 + .../keyword-over-fuzzy-auto-import/uv.lock | 8 + crates/ty_ide/src/completion.rs | 294 +++++++++++++++++- docs/basedpython/features/editor.md | 21 ++ 7 files changed, 333 insertions(+), 7 deletions(-) create mode 100644 crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/completion.toml create mode 100644 crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/main.py create mode 100644 crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/uv.lock diff --git a/crates/ty_completion_eval/completion-evaluation-tasks.csv b/crates/ty_completion_eval/completion-evaluation-tasks.csv index 04d45b6473..806952ec91 100644 --- a/crates/ty_completion_eval/completion-evaluation-tasks.csv +++ b/crates/ty_completion_eval/completion-evaluation-tasks.csv @@ -37,10 +37,11 @@ import-deprioritizes-type_check_only,main.pyi,0,1 import-deprioritizes-type_check_only,main.pyi,1,1 import-keyword-completion,main.py,0,1 internal-typeshed-hidden,main.py,0,1 +keyword-over-fuzzy-auto-import,main.py,0,1 local-over-auto-import,main.py,0,1 modules-over-other-symbols,main.py,0,1 none-completion,main.py,0,1 -numpy-array,main.py,0,11 +numpy-array,main.py,0,10 numpy-array,main.py,1,1 object-attr-instance-methods,main.py,0,1 object-attr-instance-methods,main.py,1,1 @@ -63,7 +64,7 @@ typing-gets-priority,main.py,4,1 typing-only-auto-import-ranking,main.py,0,1 typing-only-auto-import-ranking,main.py,1,1 typing-only-auto-import-ranking,main.py,2,1 -typing-only-auto-import-ranking,main.py,3,2 +typing-only-auto-import-ranking,main.py,3,1 typing-only-auto-import-ranking,main.py,4,1 typing-only-auto-import-ranking,main.py,5,1 typing-only-auto-import-ranking,main.py,6,1 diff --git a/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/completion.toml b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/main.py b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/main.py new file mode 100644 index 0000000000..d70151d818 --- /dev/null +++ b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/main.py @@ -0,0 +1,5 @@ +def total(values: list[int]) -> int: + # `return` should come first, ahead of the imports of + # modules like `secrets` or `urlretrieve` that only + # contain the letters `r`, `e` and `t` in order. + ret diff --git a/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/pyproject.toml b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/uv.lock b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/keyword-over-fuzzy-auto-import/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." } diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 54d419a546..30c61b846c 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -121,6 +121,17 @@ pub fn completion<'db>( .add(CompletionBuilder::keyword(keyword.as_str()).context_specific(true)); } } + ContextKind::OverrideName { class, def_keyword } => { + add_override_name_completions( + &model, + class, + def_keyword, + &context.cursor, + source_type, + capabilities, + &mut completions, + ); + } ContextKind::TypeParamModifiers(modifiers) => { for &modifier in modifiers { completions.add(CompletionBuilder::keyword(modifier).context_specific(true)); @@ -1001,6 +1012,13 @@ struct Context<'m> { #[derive(Debug)] enum ContextKind<'m> { Keywords(&'static [ContextualKeyword]), + /// the name of a `def` written directly in a class body, which names + /// either a new member or one the class overrides + OverrideName { + class: &'m ast::StmtClassDef, + /// the `def` keyword, before which an `override` modifier goes + def_keyword: TextRange, + }, /// basedpython: the modifiers a type parameter opens with. A type /// parameter is being named, so nothing else is valid there. TypeParamModifiers(&'static [&'static str]), @@ -1066,6 +1084,8 @@ impl<'m> Context<'m> { ContextKind::Keywords(keywords) } else if let Some(modifiers) = cursor.type_param_modifiers(file.file(db).source_type(db)) { ContextKind::TypeParamModifiers(modifiers) + } else if let Some((class, def_keyword)) = cursor.class_body_def_name() { + ContextKind::OverrideName { class, def_keyword } } else if cursor.is_in_definition_place() { return None; } else if let Some(import) = ImportStatement::detect( @@ -1096,6 +1116,7 @@ impl<'m> Context<'m> { match self.kind { ContextKind::Keywords(_) + | ContextKind::OverrideName { .. } | ContextKind::TypeParamModifiers(_) | ContextKind::Import(_) => CollectionContext { type_checking_block, @@ -1440,6 +1461,40 @@ impl<'m> ContextCursor<'m> { }) } + /// the class, and the `def` keyword, when the cursor is on the name of a + /// `def` written directly in that class's body: `def are` + fn class_body_def_name(&self) -> Option<(&'m ast::StmtClassDef, TextRange)> { + let def_keyword = match self.tokens_before { + [.., penultimate, _] if self.typed.is_some() => penultimate, + [.., last] if self.typed.is_none() => last, + _ => return None, + }; + if def_keyword.kind() != TokenKind::Def { + return None; + } + // the `def` being written is a statement of its own, around the cursor + // or, before its name is, just ahead of it + let mut ancestors = self.covering_node.ancestors().skip_while(|node| { + !matches!( + node, + ast::AnyNodeRef::StmtClassDef(_) | ast::AnyNodeRef::StmtFunctionDef(_) + ) + }); + let mut next = ancestors.next()?; + if let ast::AnyNodeRef::StmtFunctionDef(function) = next { + if function.range().start() > def_keyword.start() + || !function.name.range().contains_range(self.range) + { + return None; + } + next = ancestors.next()?; + } + match next { + ast::AnyNodeRef::StmtClassDef(class) => Some((class, def_keyword.range())), + _ => None, + } + } + /// Returns true when the tokens indicate that the definition of a new /// name is being introduced at the end. fn is_in_definition_place(&self) -> bool { @@ -3131,6 +3186,59 @@ fn add_override_completions<'db>( } } +/// the superclass members a class could override, offered at the name of a +/// `def` written in its body, each as the rest of the header it would be +/// written with: `def are` completes to `def area(self) -> float:` +/// +/// in a basedpython file an override says so, and `missing-override-decorator` +/// holds one that does not to it, so the completion also writes `override` +/// before the `def` when it is not there already +fn add_override_name_completions<'db>( + model: &SemanticModel<'db>, + class: &ast::StmtClassDef, + def_keyword: TextRange, + cursor: &ContextCursor<'_>, + source_type: PySourceType, + capabilities: CompletionCapabilities, + completions: &mut Completions<'db>, +) { + let env = &model.program_environment(); + let Some(class) = class.inferred_type(model) else { + return; + }; + let says_override = cursor + .tokens_before + .iter() + .rev() + .skip_while(|token| token.range() != def_keyword) + .skip(1) + .find(|token| !token.kind().is_trivia()) + .is_some_and(|token| &cursor.source[token.range()] == "override"); + let write_override = source_type.is_basedpython() && !says_override; + + for member in overridable_members(model.db(), env, class) { + let header = format!("{}{}:", member.name, member.signature); + let mut builder = Completion::builder(member.name.as_str()) + .kind(CompletionKind::Method) + .label(format!("{}{}", member.name, member.signature)) + .ty(member.ty) + .detail(format!("override {}", member.superclass)) + .documentation(format!("override `{}.{}`", member.superclass, member.name)) + .context_specific(true); + if write_override { + builder = builder.additional_edit(Edit::insertion( + "override ".to_string(), + def_keyword.start(), + )); + } + completions.add(if capabilities.snippets { + builder.snippet(format!("{header}\n$0")) + } else { + builder.insert(header) + }); + } +} + /// A statement opener the completion list has to carry itself. /// /// Either a construct spelled with several keywords, each of which the list @@ -4108,6 +4216,17 @@ fn add_unimported_completions<'db>( (name, ImportRequest::module(name)) }); + // basedpython: a name the file has not imported is one the user is + // recalling by how it begins, so the letters typed have to start where a + // word of it does. `ret` offers `return` and `reveal_type` but not the + // import of `secrets` or `urlretrieve`, which only contain the letters in + // order, and which would otherwise crowd out the names in scope + if let Some(typed) = completions.query.exact.as_deref() + && !matches_from_a_word_start(typed, name) + { + continue; + } + // Don't suggest symbols that are already imported. if members.satisfies(db, importing_file, &request) { continue; @@ -4143,6 +4262,55 @@ fn add_unimported_completions<'db>( } } +/// whether `typed` matches `name` from the start of one of its words, ignoring case: its first +/// letter is where a word of `name` starts, and the rest of `typed` follows in order. `rt` +/// matches `reveal_type` and `NoReturn` but not `secrets` +/// +/// a word starts at the start of `name` and after a character that is not a letter or a digit, +/// as `_` and `.` are. within a run of letters and digits, a word starts at an upper-case letter +/// after a lower-case one (`NoReturn`'s `R`), at the last of a run of upper-case letters when a +/// lower-case one follows it (`HTTPServer`'s `S`), and at a letter after a digit (`md5sum`'s +/// `s`) +fn matches_from_a_word_start(typed: &str, name: &str) -> bool { + let mut typed = typed.chars(); + let Some(first) = typed.next() else { + return true; + }; + let rest: Vec = typed.collect(); + let chars: Vec = name.chars().collect(); + (0..chars.len()).any(|start| { + if !is_word_start(&chars, start) || !chars[start].to_lowercase().eq(first.to_lowercase()) { + return false; + } + let mut remaining = chars[start + 1..].iter(); + rest.iter().all(|wanted| { + remaining.any(|candidate| candidate.to_lowercase().eq(wanted.to_lowercase())) + }) + }) +} + +/// whether a word of `chars` starts at `index`, as [`matches_from_a_word_start`] splits words +fn is_word_start(chars: &[char], index: usize) -> bool { + let current = chars[index]; + let Some(&previous) = index + .checked_sub(1) + .and_then(|previous| chars.get(previous)) + else { + return true; + }; + if !previous.is_alphanumeric() { + return true; + } + if previous.is_numeric() { + return current.is_alphabetic(); + } + if !current.is_uppercase() { + return false; + } + previous.is_lowercase() + || (previous.is_uppercase() && chars.get(index + 1).is_some_and(|next| next.is_lowercase())) +} + /// The aliases the user could be part-way through writing. /// /// An alias is only ever offered as something to import, so this answers with nothing where @@ -7949,6 +8117,39 @@ if sys.platform == \"not-my-current-platform\": ); } + /// an unimported name is offered when the letters typed start where one of its words does, + /// not when it merely contains them in order + #[test] + fn auto_import_matches_from_a_word_start() { + let builder = completion_test_builder( + "\ +ret +", + ); + + let test = builder.build(); + test.contains("return"); + assert!( + test.completions() + .iter() + .filter(|completion| completion.module_name.is_some()) + .all(|completion| super::matches_from_a_word_start("ret", &completion.name)), + "an import was offered for a name `ret` does not start a word of", + ); + assert!(super::matches_from_a_word_start("rt", "reveal_type")); + assert!(super::matches_from_a_word_start("ret", "NoReturn")); + assert!(!super::matches_from_a_word_start("ret", "secrets")); + assert!(!super::matches_from_a_word_start("ret", "urlretrieve")); + // the last capital of a run of them starts the word it leads, and a letter after a digit + // starts one + assert!(super::matches_from_a_word_start("ser", "HTTPServer")); + assert!(super::matches_from_a_word_start("http", "HTTPServer")); + assert!(!super::matches_from_a_word_start("tps", "HTTPServer")); + assert!(super::matches_from_a_word_start("sum", "md5sum")); + assert!(super::matches_from_a_word_start("ser", "Base64Serializer")); + assert!(!super::matches_from_a_word_start("um", "md5sum")); + } + #[test] fn star_import() { let builder = completion_test_builder( @@ -7959,11 +8160,19 @@ Re ", ); - // `ReadableBuffer` is a symbol in `typing`, but it is not re-exported - builder - .build() - .contains("Reversible") - .not_contains("ReadableBuffer"); + // `ReadableBuffer` is a symbol in `typing`, but it is not re-exported, so the star import + // does not bring it into scope: it is offered only as an import from `_typeshed` + let test = builder.build(); + test.contains("Reversible"); + assert!( + test.completions() + .iter() + .all(|completion| completion.name != "ReadableBuffer" + || completion + .module_name + .is_some_and(|module| module.as_str() == "_typeshed")), + "Expected `ReadableBuffer` to be offered only as an import from `_typeshed`", + ); } #[test] @@ -13318,6 +13527,81 @@ if foo: ); } + /// the name of a `def` in a class body offers what the class could override, + /// as the rest of the header, and writes `override` before the `def` in a + /// basedpython file + #[test] + fn override_completions_at_a_def_name() { + let test = CursorTest::builder() + .source( + "main.by", + "class A:\n def total(self, rate: float) -> float:\n return rate\n\nclass B(A):\n def to", + ) + .completion_test_builder() + .skip_auto_import(); + let test = test.build(); + let total = test + .completions() + .iter() + .find(|completion| completion.name == "total") + .expect("`total` is offered"); + assert_eq!( + total.insert.as_deref(), + Some("total(self, rate: float) -> float:") + ); + assert_eq!( + total + .additional_edit + .as_ref() + .and_then(|edit| edit.content()), + Some("override ") + ); + } + + /// in python the header is all there is to write + #[test] + fn override_completions_at_a_def_name_in_python() { + let test = completion_test_builder( + "class A:\n def total(self) -> int:\n return 1\n\nclass B(A):\n def to", + ) + .skip_auto_import(); + let test = test.build(); + let total = test + .completions() + .iter() + .find(|completion| completion.name == "total") + .expect("`total` is offered"); + assert_eq!(total.insert.as_deref(), Some("total(self) -> int:")); + assert!(total.additional_edit.is_none()); + } + + /// a `def` that already says `override` is not given a second one, and a + /// `def` outside a class body is still a new name + #[test] + fn override_completions_at_a_def_name_already_override() { + let test = CursorTest::builder() + .source( + "main.by", + "class A:\n def total(self) -> int:\n return 1\n\nclass B(A):\n override def to", + ) + .completion_test_builder() + .skip_auto_import(); + let test = test.build(); + let total = test + .completions() + .iter() + .find(|completion| completion.name == "total") + .expect("`total` is offered"); + assert!(total.additional_edit.is_none()); + + assert!( + completion_test_builder("def to") + .build() + .completions() + .is_empty() + ); + } + /// A member the class already defines is not one it can add. #[test] fn basedpython_override_completions_skip_defined_members() { diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 12a1ac2db3..014e524ba0 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -77,6 +77,11 @@ class B(A): `object`'s members are left out +the name of a `def` written in a class body completes the same way, as the rest +of the header: `def gre` offers `greet(self, name: str) -> str:`. in a `.by` +file, taking it also writes `override` before the `def`, unless it is there +already + ### types a type position offers the words basedpython spells types with — `literal`, @@ -122,6 +127,15 @@ off too ### unimported names +a name the file has not imported is offered for import only when what you have +typed starts where one of its words does — at its start, after a `_`, a `.` or +any other character that is not a letter or a digit, at an upper-case letter that +follows a lower-case one, at the last of a run of capitals that a lower-case letter +follows (`ser` offers `HTTPServer`), or at a letter that follows a digit (`sum` +offers `md5sum`) — and the rest follows in order. `ret` offers `reveal_type` and `NoReturn` to import, but not +`secrets` or `urlretrieve`, which merely contain the letters, and which would +otherwise crowd out `return` and the names in scope + a name the file has not imported still completes past the dot. `Asdf.` offers the members of the `Asdf` an import would bind, and accepting one writes that import: @@ -176,6 +190,13 @@ a docstring, a `case` pattern and a `str.format` template are not offered the conversion, and neither is a string written where a type belongs: an f-string is a different thing in each of those places, or no longer legal at all +### dict keys + +the key of a subscript completes from what the dict is known to hold. a +`TypedDict`'s keys come from its type, and a plain dict's from the dict display +the name was bound to: after `d = {"alpha": 1, "beta": 2}`, `d["` offers +`alpha` and `beta` + ### enum members and extensions a bare [enum member](enums.md) is offered where the expected type admits one — From 6253b8ff3c809f5fe6cdc77ce294518db17d9686 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:26:55 +1000 Subject: [PATCH 018/110] read an upper-case auto() member of an Enum subclass in a .by file, which the class body reads as final, as the auto() member it is, so its value is counted out as python counts it, Level.LOW.value revealing 1 rather than int, and its enumValues hint is drawn --- crates/ty_ide/src/inlay_hints.rs | 24 +++++++++++++++++++ .../resources/mdtest/basedpython_enums.md | 16 +++++++++++++ crates/ty_python_semantic/src/types/enums.rs | 4 ++++ 3 files changed, 44 insertions(+) diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index b57397be3e..60139f9ca3 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -6381,6 +6381,30 @@ Source with applied edits: "); } + /// basedpython reads an upper-case name a class body assigns as `final`, and + /// an `auto()` member written that way is hinted all the same + #[test] + fn basedpython_enum_auto_values() { + let mut test = basedpython_inlay_hint_test( + r#" + from enum import IntEnum, auto + + class Color(IntEnum): + RED = auto() + GREEN = auto() + "#, + ); + + assert_snapshot!(test.inlay_hints(), @" + + from enum import IntEnum, auto + + class Color(IntEnum): + RED = auto()[ 1] + GREEN = auto()[ 2] + "); + } + /// annotating an enum member is an error, and ty reads the value off the /// annotation rather than the `auto()` — so there is nothing to show #[test] diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md index 712801c996..409182e17e 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md @@ -498,6 +498,22 @@ enum class Shape: reveal_type(Shape.Point) # revealed: Shape.Point ``` +## an upper-case `auto()` member of an `Enum` subclass + +a class body reads an upper-case name it assigns as `final`, and an `auto()` member written that way +is still an `auto()` member: its value is counted out as python's `Enum` counts it + +```by +from enum import IntEnum, auto + +class Level(IntEnum): + LOW = auto() + HIGH = auto() + +reveal_type(Level.LOW.value) # revealed: 1 +reveal_type(Level.HIGH.value) # revealed: 2 +``` + ## constants in an enum body stay constants an assignment member disqualifies the idiomatic-`Enum` lowering (python's `Enum` would turn the diff --git a/crates/ty_python_semantic/src/types/enums.rs b/crates/ty_python_semantic/src/types/enums.rs index 2ca7471d59..0f55e854e3 100644 --- a/crates/ty_python_semantic/src/types/enums.rs +++ b/crates/ty_python_semantic/src/types/enums.rs @@ -1313,6 +1313,10 @@ pub(crate) fn enum_metadata<'db>( return None; } Place::Defined(DefinedPlace { ty, .. }) => { + // basedpython: an upper-case name a class body assigns is read + // as `final`, and the member is the value the restriction wraps: + // `RED = auto()` is an `auto()` member however it is qualified + let ty = ty.erase_restriction(db); let special_case = match ty { Type::Callable(_) | Type::FunctionLiteral(_) => { // Some types are specifically disallowed for enum members. From 2ee4e7df8f4616ee0d9173d4922317c237afe2e9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:31:51 +1000 Subject: [PATCH 019/110] name buff as buff in what its server shows, its code actions' titles, its diagnostics' source and its error messages, rather than as Ruff --- crates/ruff_server/src/lib.rs | 4 ++-- crates/ruff_server/src/server/api.rs | 12 ++++++------ .../src/server/api/requests/execute_command.rs | 4 ++-- crates/ruff_server/src/session/index.rs | 2 +- crates/ruff_server/src/session/options.rs | 2 +- crates/ruff_server/src/session/settings.rs | 2 +- crates/ruff_server/tests/e2e/capabilities.rs | 16 ++++++++++++++++ crates/ruff_server/tests/e2e/code_action.rs | 16 ++++++++-------- crates/ruff_server/tests/e2e/diagnostics.rs | 6 +++--- ..._optimize_imports_is_answered_when_named.snap | 2 +- ..._organize_imports_is_answered_when_named.snap | 2 +- ..._optimize_imports_is_answered_when_named.snap | 2 +- ...k__notebook_without_ipynb_extension_open.snap | 4 ++-- ...otebook__super_resolution_overview_final.snap | 4 ++-- ...notebook__super_resolution_overview_open.snap | 4 ++-- crates/ruff_server/tests/e2e/workspace.rs | 2 +- 16 files changed, 50 insertions(+), 34 deletions(-) diff --git a/crates/ruff_server/src/lib.rs b/crates/ruff_server/src/lib.rs index da06eb312e..396041a361 100644 --- a/crates/ruff_server/src/lib.rs +++ b/crates/ruff_server/src/lib.rs @@ -21,8 +21,8 @@ mod server; mod session; mod workspace; -const SERVER_NAME: &str = "ruff"; -pub(crate) const DIAGNOSTIC_NAME: &str = "Ruff"; +const SERVER_NAME: &str = "buff"; +pub(crate) const DIAGNOSTIC_NAME: &str = "buff"; const SOURCE_FIX_ALL_RUFF: CodeActionKind = CodeActionKind::new("source.fixAll.ruff"); const SOURCE_ORGANIZE_IMPORTS_RUFF: CodeActionKind = diff --git a/crates/ruff_server/src/server/api.rs b/crates/ruff_server/src/server/api.rs index b62e181f87..6c02e6b19a 100644 --- a/crates/ruff_server/src/server/api.rs +++ b/crates/ruff_server/src/server/api.rs @@ -85,7 +85,7 @@ pub(super) fn request(req: server::Request) -> Task { Task::sync(move |_session, client| { client.show_error_message( - "Ruff failed to handle a request from the editor. Check the logs for more details.", + "buff failed to handle a request from the editor. Check the logs for more details.", ); respond_silent_error( id, @@ -139,7 +139,7 @@ pub(super) fn notification(notif: server::Notification) -> Task { tracing::error!("Encountered error when routing notification: {err}"); Task::sync(|_session, client| { client.show_error_message( - "Ruff failed to handle a notification from the editor. \ + "buff failed to handle a notification from the editor. \ Check the logs for more details.", ); }) @@ -235,7 +235,7 @@ fn sync_notification_task(notif: server::Notificatio if let Err(err) = N::run(session, client, params) { tracing::error!("An error occurred while running {id}: {err}"); client - .show_error_message("Ruff encountered a problem. Check the logs for more details."); + .show_error_message("buff encountered a problem. Check the logs for more details."); } })) } @@ -274,7 +274,7 @@ where tracing::error!(message); client.show_error_message( - "Ruff encountered a panic. Check the logs for more details.", + "buff encountered a panic. Check the logs for more details.", ); return; } @@ -283,7 +283,7 @@ where if let Err(err) = result { tracing::error!("An error occurred while running {id}: {err}"); client.show_error_message( - "Ruff encountered a problem. Check the logs for more details.", + "buff encountered a problem. Check the logs for more details.", ); } }) @@ -331,7 +331,7 @@ fn respond( { if let Err(err) = &result { tracing::error!("An error occurred with request ID {id}: {err}"); - client.show_error_message("Ruff encountered a problem. Check the logs for more details."); + client.show_error_message("buff encountered a problem. Check the logs for more details."); } if let Err(err) = client.respond(id, result) { tracing::error!("Failed to send response: {err}"); diff --git a/crates/ruff_server/src/server/api/requests/execute_command.rs b/crates/ruff_server/src/server/api/requests/execute_command.rs index ab59b70d99..3e8d58c0f6 100644 --- a/crates/ruff_server/src/server/api/requests/execute_command.rs +++ b/crates/ruff_server/src/server/api/requests/execute_command.rs @@ -89,7 +89,7 @@ impl super::SyncRequestHandler for ExecuteCommand { for Argument { uri, version } in arguments { let Some(snapshot) = session.take_snapshot(uri.clone()) else { tracing::error!("Document at {uri} could not be opened"); - client.show_error_message("Ruff does not recognize this file"); + client.show_error_message("buff does not recognize this file"); return Ok(None); }; match command { @@ -179,7 +179,7 @@ fn apply_edit( .failure_reason .unwrap_or_else(|| String::from("unspecified reason")); tracing::error!("Failed to apply workspace edit: {reason}"); - client.show_error_message(format_args!("Ruff was unable to apply edits: {reason}")); + client.show_error_message(format_args!("buff was unable to apply edits: {reason}")); } }, ) diff --git a/crates/ruff_server/src/session/index.rs b/crates/ruff_server/src/session/index.rs index 846770e869..219ce75cf8 100644 --- a/crates/ruff_server/src/session/index.rs +++ b/crates/ruff_server/src/session/index.rs @@ -423,7 +423,7 @@ impl WorkspaceSettingsIndex { if workspace_uri.scheme() != "file" { tracing::info!("Ignoring non-file workspace URI: {workspace_uri}"); client.show_warning_message(format_args!( - "Ruff does not support non-file workspaces; Ignoring {workspace_uri}" + "buff does not support non-file workspaces; Ignoring {workspace_uri}" )); return Ok(()); } diff --git a/crates/ruff_server/src/session/options.rs b/crates/ruff_server/src/session/options.rs index 98807c3ec8..32485a6fbc 100644 --- a/crates/ruff_server/src/session/options.rs +++ b/crates/ruff_server/src/session/options.rs @@ -355,7 +355,7 @@ impl AllOptions { Falling back to default client settings..." ); client.show_error_message( - "Ruff received invalid client settings - \ + "buff received invalid client settings - \ falling back to default client settings.", ); }) diff --git a/crates/ruff_server/src/session/settings.rs b/crates/ruff_server/src/session/settings.rs index 9f8b5c9b23..c0c2a51e5b 100644 --- a/crates/ruff_server/src/session/settings.rs +++ b/crates/ruff_server/src/session/settings.rs @@ -41,7 +41,7 @@ impl GlobalClientSettings { Ok(settings) => settings, Err(settings) => { self.client.show_error_message( - "Ruff received invalid settings from the editor. \ + "buff received invalid settings from the editor. \ Refer to the logs for more information.", ); settings diff --git a/crates/ruff_server/tests/e2e/capabilities.rs b/crates/ruff_server/tests/e2e/capabilities.rs index fe47aac6ec..b84179db6e 100644 --- a/crates/ruff_server/tests/e2e/capabilities.rs +++ b/crates/ruff_server/tests/e2e/capabilities.rs @@ -5,6 +5,22 @@ use lsp_types::{DocumentFormattingRequest, DocumentRangeFormattingRequest, Regis use crate::TestServerBuilder; +/// the server names itself as buff, the name it shows in everything else +#[test] +fn server_is_named_buff() -> Result<()> { + let server = TestServerBuilder::new()?.build(); + let server_info = server + .initialization_result() + .expect("Server should return initialization capabilities") + .server_info + .as_ref() + .expect("the server names itself"); + + assert_eq!(server_info.name, "buff"); + + Ok(()) +} + #[test] fn statically_registers_formatting_when_dynamic_registration_is_unsupported() -> Result<()> { let server = TestServerBuilder::new()?.build(); diff --git a/crates/ruff_server/tests/e2e/code_action.rs b/crates/ruff_server/tests/e2e/code_action.rs index 1e0c0fc52c..6473c22113 100644 --- a/crates/ruff_server/tests/e2e/code_action.rs +++ b/crates/ruff_server/tests/e2e/code_action.rs @@ -45,7 +45,7 @@ fn code_actions_for_python() -> Result<()> { @r#" [ { - "title": "Ruff: Fix all auto-fixable problems", + "title": "buff: Fix all auto-fixable problems", "kind": "source.fixAll.ruff", "edit": { "changes": { @@ -68,7 +68,7 @@ fn code_actions_for_python() -> Result<()> { } }, { - "title": "Ruff: Organize imports", + "title": "buff: Organize imports", "kind": "source.organizeImports.ruff", "edit": { "changes": {} @@ -111,7 +111,7 @@ extend-select = ["F401"] assert_json_snapshot!(actions, @r#" [ { - "title": "Ruff (rule-codes-in-selectors): Replace rule code with `unused-import`", + "title": "buff (rule-codes-in-selectors): Replace rule code with `unused-import`", "kind": "quickfix", "diagnostics": [ { @@ -130,7 +130,7 @@ extend-select = ["F401"] "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/rule-codes-in-selectors" }, - "source": "Ruff", + "source": "buff", "message": "Rule code used instead of name in `lint.extend-select`\n\nhelp: Replace rule code with `unused-import`", "tags": [] } @@ -158,7 +158,7 @@ extend-select = ["F401"] "data": "file:///ruff.toml" }, { - "title": "Ruff: Fix all auto-fixable problems", + "title": "buff: Fix all auto-fixable problems", "kind": "source.fixAll.ruff", "edit": { "changes": { @@ -225,8 +225,8 @@ preview = true }) .collect(); - assert!(titles.contains(&"Ruff (unused-import): Remove unused import: `os`")); - assert!(titles.contains(&"Ruff (unused-import): Disable for this line")); + assert!(titles.contains(&"buff (unused-import): Remove unused import: `os`")); + assert!(titles.contains(&"buff (unused-import): Disable for this line")); Ok(()) } @@ -251,7 +251,7 @@ fn invalid_code_action_resolve_data_returns_unchanged_action() -> Result<()> { let mut server = TestServerBuilder::new()?.with_workspace(".")?.build(); let action = CodeAction { - title: "Ruff: Fix all auto-fixable problems".to_string(), + title: "buff: Fix all auto-fixable problems".to_string(), kind: Some(CodeActionKind::from("source.fixAll.ruff")), data: Some(serde_json::json!("not-a-uri")), ..Default::default() diff --git a/crates/ruff_server/tests/e2e/diagnostics.rs b/crates/ruff_server/tests/e2e/diagnostics.rs index c76bae8d99..833e784636 100644 --- a/crates/ruff_server/tests/e2e/diagnostics.rs +++ b/crates/ruff_server/tests/e2e/diagnostics.rs @@ -35,7 +35,7 @@ fn uses_human_readable_names_in_preview() -> Result<()> { "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/unused-import" }, - "source": "Ruff", + "source": "buff", "message": "`os` imported but unused\n\nhelp: Remove unused import: `os`", "tags": [ 1 @@ -119,7 +119,7 @@ extend-select = ["F401"] "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/rule-codes-in-selectors" }, - "source": "Ruff", + "source": "buff", "message": "Rule code used instead of name in `lint.extend-select`\n\nhelp: Replace rule code with `unused-import`", "tags": [], "data": { @@ -180,7 +180,7 @@ fn invalid_pyproject_toml_diagnostic() -> Result<()> { "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/invalid-pyproject-toml" }, - "source": "Ruff", + "source": "buff", "message": "Failed to parse pyproject.toml: invalid type: integer `1`, expected a string", "tags": [] } diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_optimize_imports_is_answered_when_named.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_optimize_imports_is_answered_when_named.snap index ed17eac87b..b3cb0b5e77 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_optimize_imports_is_answered_when_named.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_optimize_imports_is_answered_when_named.snap @@ -4,7 +4,7 @@ expression: actions --- [ { - "title": "Ruff: Format document and optimize imports", + "title": "buff: Format document and optimize imports", "kind": "source.formatAndOptimizeImports.ruff", "edit": { "changes": { diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_organize_imports_is_answered_when_named.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_organize_imports_is_answered_when_named.snap index b99b732da8..6762ec72ed 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_organize_imports_is_answered_when_named.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__format_and_organize_imports_is_answered_when_named.snap @@ -4,7 +4,7 @@ expression: actions --- [ { - "title": "Ruff: Format document and organize imports", + "title": "buff: Format document and organize imports", "kind": "source.formatAndOrganizeImports.ruff", "edit": { "changes": { diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__optimize_imports_is_answered_when_named.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__optimize_imports_is_answered_when_named.snap index 999b068c95..786c8168bb 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__optimize_imports_is_answered_when_named.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__code_action__optimize_imports_is_answered_when_named.snap @@ -4,7 +4,7 @@ expression: actions --- [ { - "title": "Ruff: Optimize imports", + "title": "buff: Optimize imports", "kind": "source.optimizeImports.ruff", "edit": { "changes": { diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__notebook_without_ipynb_extension_open.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__notebook_without_ipynb_extension_open.snap index 3140d36d60..6749170b4c 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__notebook_without_ipynb_extension_open.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__notebook_without_ipynb_extension_open.snap @@ -22,7 +22,7 @@ expression: diagnostics "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/unsorted-imports" }, - "source": "Ruff", + "source": "buff", "message": "Import block is un-sorted or un-formatted\n\nhelp: Organize imports", "tags": [], "data": { @@ -80,7 +80,7 @@ expression: diagnostics "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/f-string-missing-placeholders" }, - "source": "Ruff", + "source": "buff", "message": "f-string without any placeholders\n\nhelp: Remove extraneous `f` prefix", "tags": [], "data": { diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_final.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_final.snap index 1d845984c3..14efdde60e 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_final.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_final.snap @@ -22,7 +22,7 @@ expression: "final_diagnostics.expect(\"at least one notebook change\")" "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/unsorted-imports" }, - "source": "Ruff", + "source": "buff", "message": "Import block is un-sorted or un-formatted\n\nhelp: Organize imports", "tags": [], "data": { @@ -80,7 +80,7 @@ expression: "final_diagnostics.expect(\"at least one notebook change\")" "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/f-string-missing-placeholders" }, - "source": "Ruff", + "source": "buff", "message": "f-string without any placeholders\n\nhelp: Remove extraneous `f` prefix", "tags": [], "data": { diff --git a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_open.snap b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_open.snap index 3140d36d60..6749170b4c 100644 --- a/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_open.snap +++ b/crates/ruff_server/tests/e2e/snapshots/e2e__notebook__super_resolution_overview_open.snap @@ -22,7 +22,7 @@ expression: diagnostics "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/unsorted-imports" }, - "source": "Ruff", + "source": "buff", "message": "Import block is un-sorted or un-formatted\n\nhelp: Organize imports", "tags": [], "data": { @@ -80,7 +80,7 @@ expression: diagnostics "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/f-string-missing-placeholders" }, - "source": "Ruff", + "source": "buff", "message": "f-string without any placeholders\n\nhelp: Remove extraneous `f` prefix", "tags": [], "data": { diff --git a/crates/ruff_server/tests/e2e/workspace.rs b/crates/ruff_server/tests/e2e/workspace.rs index f118be5842..e1aab2e951 100644 --- a/crates/ruff_server/tests/e2e/workspace.rs +++ b/crates/ruff_server/tests/e2e/workspace.rs @@ -59,7 +59,7 @@ ignore = ["F401"] "codeDescription": { "href": "https://kotlinisland.github.io/basedpython/rules/unused-import" }, - "source": "Ruff", + "source": "buff", "message": "`os` imported but unused\n\nhelp: Remove unused import: `os`", "tags": [ 1 From 5abe9f825b562cceb7fd6d932868404bc4dcda00 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:32:03 +1000 Subject: [PATCH 020/110] offer to ignore a diagnostic's rule in the whole file, with a ty: ignore comment at the top of the file or the rule added to the one already there, beside ignoring it for the line --- .../src/encoding_declaration.rs | 185 +++++++++ crates/ruff_python_trivia/src/lib.rs | 2 + crates/ty/Cargo.toml | 2 +- crates/ty/src/by_source_encoding.rs | 142 ++----- crates/ty_ide/src/code_action.rs | 379 +++++++++++++++++- ...tests__add_ignore_trailing_whitespace.snap | 10 + crates/ty_python_semantic/src/lib.rs | 2 +- crates/ty_python_semantic/src/suppression.rs | 2 +- .../src/suppression/add_ignore.rs | 164 ++++++++ ..._is_offered_the_command_that_installs.snap | 45 +++ ..._none_is_fixed_with_optional_chaining.snap | 45 +++ .../e2e__code_actions__code_action.snap | 48 +++ ...action_attribute_access_on_unimported.snap | 45 +++ ...n_existing_import_undefined_decorator.snap | 45 +++ ...ode_action_invalid_string_annotations.snap | 45 +++ ..._possible_missing_submodule_attribute.snap | 45 +++ ...ions__code_action_undefined_decorator.snap | 45 +++ ...code_action_undefined_reference_multi.snap | 45 +++ ...actions__code_action_undefined_reveal.snap | 45 +++ ..._code_actions__code_action_unsafe_fix.snap | 48 +++ ...tion_with_full_diagnostic_output_link.snap | 45 +++ docs/basedpython/features/editor.md | 10 + 22 files changed, 1331 insertions(+), 113 deletions(-) create mode 100644 crates/ruff_python_trivia/src/encoding_declaration.rs diff --git a/crates/ruff_python_trivia/src/encoding_declaration.rs b/crates/ruff_python_trivia/src/encoding_declaration.rs new file mode 100644 index 0000000000..f6cf6a02eb --- /dev/null +++ b/crates/ruff_python_trivia/src/encoding_declaration.rs @@ -0,0 +1,185 @@ +//! [PEP 263](https://peps.python.org/pep-0263/) encoding declarations +//! +//! read over bytes rather than text, since a source that declares its encoding may not be utf-8 +//! yet. the syntax is ascii, so a byte scan reads it exactly + +use ruff_text_size::{TextRange, TextSize}; + +/// the encoding a python source declares, as python's tokenizer reads it: a comment that is the +/// whole of its line and holds `coding[:=][ \t]*[-\w.]+`, on the first line or on the second when +/// the first could not be code, which is to say it is blank or a comment. a utf-8 byte order mark +/// ahead of the first line is not part of it +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodingDeclaration<'a> { + /// the encoding the source names, as written + pub name: &'a str, + /// where `name` is written + pub name_range: TextRange, + /// the line the declaration is on, its line ending included + pub line: TextRange, +} + +/// the encoding declaration of `source`, if it has one +pub fn encoding_declaration(source: &[u8]) -> Option> { + let start = if source.starts_with(b"\xef\xbb\xbf") { + 3 + } else { + 0 + }; + let first = line_at(source, start); + if let Some(declaration) = declaration_in(source, first) { + return Some(declaration); + } + let could_be_code = source[first.start..first.content_end] + .iter() + .find(|byte| !is_blank(**byte)) + .is_some_and(|byte| *byte != b'#'); + if could_be_code || first.end == first.content_end { + return None; + } + declaration_in(source, line_at(source, first.end)) +} + +#[derive(Clone, Copy)] +struct Line { + start: usize, + /// where the line's ending starts + content_end: usize, + /// past the line's ending + end: usize, +} + +/// the line of `source` starting at `start`, ended by `\n`, `\r\n` or `\r`, as python reads a +/// source +fn line_at(source: &[u8], start: usize) -> Line { + let rest = &source[start..]; + let Some(at) = rest.iter().position(|byte| matches!(byte, b'\n' | b'\r')) else { + return Line { + start, + content_end: source.len(), + end: source.len(), + }; + }; + let content_end = start + at; + let ending = if rest[at..].starts_with(b"\r\n") { + 2 + } else { + 1 + }; + Line { + start, + content_end, + end: content_end + ending, + } +} + +fn declaration_in(source: &[u8], line: Line) -> Option> { + let text = &source[line.start..line.content_end]; + let hash = text.iter().position(|byte| !is_blank(*byte))?; + if text[hash] != b'#' { + return None; + } + let mut from = hash + 1; + while let Some(found) = find(&text[from..], b"coding") { + let after = from + found + b"coding".len(); + from += found + 1; + if !matches!(text.get(after), Some(b':' | b'=')) { + continue; + } + let name_start = after + + 1 + + text[after + 1..] + .iter() + .take_while(|byte| matches!(byte, b' ' | b'\t')) + .count(); + let name_len = text[name_start..] + .iter() + .take_while(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + .count(); + if name_len == 0 { + continue; + } + let name = std::str::from_utf8(&text[name_start..name_start + name_len]).ok()?; + let name_start = line.start + name_start; + return Some(EncodingDeclaration { + name, + name_range: TextRange::new(offset(name_start)?, offset(name_start + name_len)?), + line: TextRange::new(offset(line.start)?, offset(line.end)?), + }); + } + None +} + +/// the whitespace python skips ahead of a comment +fn is_blank(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\x0c') +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn offset(at: usize) -> Option { + TextSize::try_from(at).ok() +} + +#[cfg(test)] +mod tests { + use super::encoding_declaration; + + fn name(source: &str) -> Option<&str> { + encoding_declaration(source.as_bytes()).map(|declaration| declaration.name) + } + + #[test] + fn on_the_first_line() { + assert_eq!(name("# -*- coding: latin-1 -*-\nx = 1\n"), Some("latin-1")); + assert_eq!(name("# vim: set fileencoding=utf-8 :\n"), Some("utf-8")); + assert_eq!(name("\x0c # coding:\tascii"), Some("ascii")); + } + + #[test] + fn on_the_second_line_only_below_a_blank_line_or_a_comment() { + assert_eq!( + name("#!/usr/bin/env python\n# coding=cp1252\n"), + Some("cp1252") + ); + assert_eq!(name("\r\n# coding=cp1252\n"), Some("cp1252")); + assert_eq!(name("x = 1\n# coding: latin-1\n"), None); + assert_eq!(name("#\n#\n# coding: latin-1\n"), None); + } + + #[test] + fn only_in_a_comment_that_is_the_whole_line() { + assert_eq!(name("x = 1 # coding: latin-1\n"), None); + assert_eq!(name("'# coding: latin-1'\n"), None); + } + + #[test] + fn any_coding_the_comment_spells_out() { + assert_eq!(name("# the coding, coding: latin-1\n"), Some("latin-1")); + assert_eq!(name("# coding: \n"), None); + assert_eq!(name("# coding latin-1\n"), None); + } + + #[test] + fn after_a_byte_order_mark() { + let declaration = encoding_declaration("\u{feff}# coding: utf-8\r\nx\n".as_bytes()); + assert_eq!( + declaration.map(|declaration| (declaration.name, declaration.line)), + Some(("utf-8", ruff_text_size::TextRange::new(3.into(), 20.into()))) + ); + } + + #[test] + fn where_the_name_is() { + let source = "#!python\n# coding=latin-1\n"; + let declaration = encoding_declaration(source.as_bytes()); + assert_eq!( + declaration.map(|declaration| &source[declaration.name_range]), + Some("latin-1") + ); + } +} diff --git a/crates/ruff_python_trivia/src/lib.rs b/crates/ruff_python_trivia/src/lib.rs index 40972eecb3..cfd66caa3f 100644 --- a/crates/ruff_python_trivia/src/lib.rs +++ b/crates/ruff_python_trivia/src/lib.rs @@ -2,6 +2,7 @@ pub mod basedpython; mod comment_ranges; mod comments; mod cursor; +mod encoding_declaration; mod name_matcher; mod pragmas; pub mod textwrap; @@ -11,6 +12,7 @@ mod whitespace; pub use comment_ranges::{CommentRanges, ParenthesizedExpressions, TriviaRanges}; pub use comments::*; pub use cursor::*; +pub use encoding_declaration::{EncodingDeclaration, encoding_declaration}; pub use name_matcher::NameMatcher; pub use pragmas::*; pub use tokenizer::*; diff --git a/crates/ty/Cargo.toml b/crates/ty/Cargo.toml index d58ca7819a..f33fbafc80 100644 --- a/crates/ty/Cargo.toml +++ b/crates/ty/Cargo.toml @@ -33,6 +33,7 @@ ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } ruff_diagnostics = { workspace = true } ruff_python_parser = { workspace = true, features = ["stacker"] } +ruff_python_trivia = { workspace = true } by_build = { workspace = true } by_ir = { workspace = true } by_irbuild = { workspace = true } @@ -73,7 +74,6 @@ tikv-jemallocator = { workspace = true } [dev-dependencies] ruff_db = { workspace = true, features = ["testing"] } -ruff_python_trivia = { workspace = true } ty_module_resolver = { workspace = true } ty_python_core = { workspace = true } diff --git a/crates/ty/src/by_source_encoding.rs b/crates/ty/src/by_source_encoding.rs index de720c91ed..006ac1ee3a 100644 --- a/crates/ty/src/by_source_encoding.rs +++ b/crates/ty/src/by_source_encoding.rs @@ -21,6 +21,8 @@ //! //! [PEP 263]: https://peps.python.org/pep-0263/ +use ruff_python_trivia::encoding_declaration; + /// a python source decoded to utf-8, and whether decoding had to rewrite it. #[derive(Debug)] pub(crate) struct DecodedSource { @@ -44,83 +46,26 @@ pub(crate) fn decode(bytes: &[u8]) -> Result { }); } - let Some(cookie) = coding_cookie(bytes) else { + let Some(declaration) = encoding_declaration(bytes) else { return Err( "not valid utf-8, and no PEP 263 encoding declaration on its first two lines" .to_owned(), ); }; - let Some(text) = decode_single_byte(bytes, &cookie.encoding) else { + let Some(text) = decode_single_byte(bytes, declaration.name) else { return Err(format!( "declares `{}`, which this build cannot decode (only utf-8 and the latin family)", - cookie.encoding + declaration.name )); }; Ok(DecodedSource { - text: rewrite_cookie(&text), - recoded_from: Some(cookie.encoding), - }) -} - -/// the encoding a source declares, and where it says so. -struct Cookie { - encoding: String, -} - -/// PEP 263: `coding[:=]\s*([-\w.]+)` in a comment on line 1 or line 2, and only -/// on line 2 when line 1 is blank or a comment. -/// -/// scanned over bytes rather than text, since the whole point is that the text -/// cannot be decoded yet. the syntax is ascii, so a byte scan reads it exactly. -fn coding_cookie(bytes: &[u8]) -> Option { - let mut lines = bytes.split(|byte| *byte == b'\n'); - let first = lines.next()?; - if let Some(cookie) = cookie_in_line(first) { - return Some(cookie); - } - // a declaration on line 2 only counts when line 1 could not have been code, - // which is to say line 1 is blank or a comment - let first_is_code = first - .iter() - .find(|byte| !byte.is_ascii_whitespace()) - .is_some_and(|byte| *byte != b'#'); - if first_is_code { - return None; - } - cookie_in_line(lines.next()?) -} - -fn cookie_in_line(line: &[u8]) -> Option { - let hash = line.iter().position(|byte| *byte == b'#')?; - let comment = &line[hash..]; - let at = find(comment, b"coding")?; - let rest = &comment[at + b"coding".len()..]; - let mut rest = rest.iter().copied(); - // `coding:` or `coding=`, then optional spaces - if !matches!(rest.next(), Some(b':' | b'=')) { - return None; - } - let rest: Vec = rest.skip_while(u8::is_ascii_whitespace).collect(); - let name: Vec = rest - .into_iter() - .take_while(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) - .collect(); - if name.is_empty() { - return None; - } - Some(Cookie { - encoding: String::from_utf8(name).ok()?, + text: rewrite_declaration(text), + recoded_from: Some(declaration.name.to_owned()), }) } -fn find(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - /// decode a single-byte encoding whose code points are its byte values, which is /// what latin-1 is by construction. /// @@ -146,54 +91,12 @@ fn decode_single_byte(bytes: &[u8], encoding: &str) -> Option { /// /// only the name is replaced, so the shape the file chose — `# -*- coding: X -*-`, /// `# coding=X` — survives. -fn rewrite_cookie(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut lines = text.split_inclusive('\n'); - let mut rewritten = false; - for _ in 0..2 { - let Some(line) = lines.next() else { break }; - if rewritten { - out.push_str(line); - continue; - } - match rewrite_cookie_in_line(line) { - Some(replaced) => { - out.push_str(&replaced); - rewritten = true; - } - None => out.push_str(line), - } +fn rewrite_declaration(mut text: String) -> String { + if let Some(declaration) = encoding_declaration(text.as_bytes()) { + let range = declaration.name_range.to_std_range(); + text.replace_range(range, "utf-8"); } - for line in lines { - out.push_str(line); - } - out -} - -fn rewrite_cookie_in_line(line: &str) -> Option { - let hash = line.find('#')?; - let at = line[hash..].find("coding")? + hash + "coding".len(); - let after = &line[at..]; - let mut chars = after.char_indices(); - if !matches!(chars.next(), Some((_, ':' | '='))) { - return None; - } - let name_start = after - .char_indices() - .skip(1) - .find(|(_, c)| !c.is_ascii_whitespace()) - .map(|(i, _)| i)?; - let name_len = after[name_start..] - .find(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))) - .unwrap_or(after.len() - name_start); - if name_len == 0 { - return None; - } - let mut out = String::with_capacity(line.len()); - out.push_str(&line[..at + name_start]); - out.push_str("utf-8"); - out.push_str(&after[name_start + name_len..]); - Some(out) + text } #[cfg(test)] @@ -230,6 +133,27 @@ mod tests { assert!(error.contains("no PEP 263"), "{error}"); } + // python reads a declaration only from a comment that is the whole line + #[test] + fn a_declaration_after_code_on_its_line_does_not_count() { + let error = decode(b"s = '\xe9' # coding: latin-1\n").unwrap_err(); + assert!(error.contains("no PEP 263"), "{error}"); + } + + // `coding` not followed by `:` or `=` is skipped, and a later one is read + #[test] + fn the_declaration_is_any_coding_the_comment_spells_out() { + let decoded = decode(b"# the coding, coding: latin-1\ns = '\xdf'\n").unwrap(); + assert_eq!(decoded.text, "# the coding, coding: utf-8\ns = 'ß'\n"); + } + + // a non-ascii character ahead of the declaration moves it in the decoded text + #[test] + fn the_declaration_is_rewritten_where_the_decoded_text_has_it() { + let decoded = decode(b"# \xe9 coding: latin-1\ns = '\xdf'\n").unwrap(); + assert_eq!(decoded.text, "# é coding: utf-8\ns = 'ß'\n"); + } + #[test] fn an_encoding_this_build_cannot_decode_is_refused_by_name() { let error = decode(b"# coding: shift_jis\ns = '\x82\xa0'\n").unwrap_err(); diff --git a/crates/ty_ide/src/code_action.rs b/crates/ty_ide/src/code_action.rs index b9c54ec226..6042a04b09 100644 --- a/crates/ty_ide/src/code_action.rs +++ b/crates/ty_ide/src/code_action.rs @@ -11,10 +11,10 @@ use ruff_text_size::TextRange; use ty_project::Db; use ty_python_core::ProgramFile; use ty_python_semantic::lint::LintId; -use ty_python_semantic::suppress_single; use ty_python_semantic::types::{ MISPLACED_DEPENDENCY, UNDECLARED_DEPENDENCY, UNRESOLVED_IMPORT, UNRESOLVED_REFERENCE, }; +use ty_python_semantic::{suppress_in_file, suppress_single}; /// A `QuickFix` Code Action #[derive(Debug, Clone)] @@ -111,6 +111,12 @@ pub fn code_actions( suppress_single(db, file.python_file(db), lint_id, diagnostic_range).into_edits(), false, )); + actions.push(QuickFix::new( + format!("Ignore '{}' in this file", lint_id.name()), + file.file(db), + suppress_in_file(db, file.python_file(db), lint_id).into_edits(), + false, + )); actions } @@ -163,6 +169,69 @@ mod tests { types::{UNDEFINED_REVEAL, UNRESOLVED_REFERENCE}, }; + /// a file that already starts with a suppression of its own has the code added to it + #[test] + fn add_ignore_in_file_to_an_existing_file_suppression() { + let test = CodeActionTest::with_source( + "#!/usr/bin/env python\n# ty: ignore[division-by-zero]\nb = a / 10", + ); + + assert_snapshot!(test.code_actions(&UNRESOLVED_REFERENCE), @" + info[code-action]: Ignore 'unresolved-reference' for this line + --> main.py:3:5 + | + 3 | b = a / 10 + | ^ + | + 2 | # ty: ignore[division-by-zero] + - b = a / 10 + 3 + b = a / 10 # ty: ignore[unresolved-reference] + | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:5 + | + 3 | b = a / 10 + | ^ + | + 1 | #!/usr/bin/env python + - # ty: ignore[division-by-zero] + 2 + # ty: ignore[division-by-zero, unresolved-reference] + 3 | b = a / 10 + | + "); + } + + /// a new file suppression goes after a `#!` line, which has to stay first + #[test] + fn add_ignore_in_file_after_a_shebang() { + let test = CodeActionTest::with_source("#!/usr/bin/env python\nb = a / 10"); + + assert_snapshot!(test.code_actions(&UNRESOLVED_REFERENCE), @" + info[code-action]: Ignore 'unresolved-reference' for this line + --> main.py:2:5 + | + 2 | b = a / 10 + | ^ + | + 1 | #!/usr/bin/env python + - b = a / 10 + 2 + b = a / 10 # ty: ignore[unresolved-reference] + | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 10 + | ^ + | + 1 | #!/usr/bin/env python + 2 + # ty: ignore[unresolved-reference] + 3 | b = a / 10 + | + "); + } + #[test] fn add_ignore() { let test = CodeActionTest::with_source(r#"b = a / 10"#); @@ -177,6 +246,16 @@ mod tests { - b = a / 10 1 + b = a / 10 # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:1:5 + | + 1 | b = a / 10 + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | b = a / 10 + | "); } @@ -194,6 +273,16 @@ mod tests { - b = a / 10 # fmt: off 1 + b = a / 10 # fmt: off # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:1:5 + | + 1 | b = a / 10 # fmt: off + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | b = a / 10 # fmt: off + | "); } @@ -224,6 +313,16 @@ mod tests { - b = a / 0 # ty:ignore[division-by-zero] 2 + b = a / 0 # ty:ignore[division-by-zero, unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # ty:ignore[division-by-zero] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -246,6 +345,16 @@ mod tests { - b = a / 10 # ty:ignore[] 2 + b = a / 10 # ty:ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 10 # ty:ignore[] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -271,6 +380,16 @@ mod tests { 3 + # ty:ignore[unresolved-reference] 4 | b = a / 10 | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:4:5 + | + 4 | b = a / 10 + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -301,6 +420,16 @@ mod tests { 3 + # ty:ignore[ignore-comment-unknown-rule] # ty:ignore[not-a-rule] # ty:ignore[division-by-zero] 4 | value = 1 / 0 | + + info[code-action]: Ignore 'ignore-comment-unknown-rule' in this file + --> main.py:3:27 + | + 3 | # ty:ignore[] # ty:ignore[not-a-rule] # ty:ignore[division-by-zero] + | ^^^^^^^^^^ + | + 1 + # ty: ignore[ignore-comment-unknown-rule] + 2 | + | "); } @@ -330,6 +459,16 @@ mod tests { 3 + # ty:ignore[unresolved-reference] 4 | values = [ | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:7:5 + | + 7 | absent, + | ^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -361,6 +500,16 @@ mod tests { 5 + # ty:ignore[invalid-assignment, unresolved-reference] 6 | values: tuple[int] = [ | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:9:5 + | + 9 | absent, + | ^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -383,6 +532,16 @@ mod tests { - b = a / 0 # type:ignore[ty:division-by-zero] 2 + b = a / 0 # type:ignore[ty:division-by-zero, ty:unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # type:ignore[ty:division-by-zero] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -405,6 +564,16 @@ mod tests { - b = a / 0 # type:ignore[mypy-code] 2 + b = a / 0 # type:ignore[mypy-code] # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # type:ignore[mypy-code] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -429,6 +598,18 @@ mod tests { - b = a / 0 4 + b = a / 0 # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:4:5 + | + 4 | b = a / 0 + | ^ + | + 1 | + - # ty:ignore[division-by-zero] + 2 + # ty:ignore[division-by-zero, unresolved-reference] + 3 | + | "); } @@ -451,6 +632,16 @@ mod tests { - b = a / 0 # ty:ignore[division-by-zero,] 2 + b = a / 0 # ty:ignore[division-by-zero, unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # ty:ignore[division-by-zero,] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -473,6 +664,16 @@ mod tests { - b = a / 0 # ty:ignore[division-by-zero ] 2 + b = a / 0 # ty:ignore[division-by-zero, unresolved-reference ] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # ty:ignore[division-by-zero ] + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -495,6 +696,16 @@ mod tests { - b = a / 0 # ty:ignore[division-by-zero] some explanation 2 + b = a / 0 # ty:ignore[division-by-zero] some explanation # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a / 0 # ty:ignore[division-by-zero] some explanation + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -524,6 +735,18 @@ mod tests { 3 + a # ty:ignore[division-by-zero, unresolved-reference] 4 | / | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:9 + | + 3 | / a # ty:ignore[division-by-zero] + 4 | | / + 5 | | 0 + | |_________^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -553,6 +776,18 @@ mod tests { 5 + 0 # ty:ignore[division-by-zero, unresolved-reference] 6 | ) | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:9 + | + 3 | / a + 4 | | / + 5 | | 0 # ty:ignore[division-by-zero] + | |_________^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -582,6 +817,18 @@ mod tests { 3 + a # ty:ignore[division-by-zero, unresolved-reference] 4 | / | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:9 + | + 3 | / a # ty:ignore[division-by-zero] + 4 | | / + 5 | | 0 # ty:ignore[division-by-zero] + | |_________^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -607,6 +854,16 @@ mod tests { - """ 5 + """ # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:6 + | + 3 | {a} + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "#); } @@ -635,6 +892,16 @@ mod tests { 4 + a # ty: ignore[unresolved-reference] 5 | } | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:4:5 + | + 4 | a + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -659,6 +926,16 @@ mod tests { - """ 4 + """ # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a + """ + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "#); } @@ -682,6 +959,16 @@ mod tests { - + "test" 3 + + "test" # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:5 + | + 2 | b = a \ + | ^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "#); } @@ -709,6 +996,16 @@ mod tests { 5 + # ty: ignore[unresolved-reference] 6 | ] # test | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:4:11 + | + 4 | + ddd \ + | ^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | b = bbbbb \ + | "); } @@ -731,6 +1028,16 @@ mod tests { - reveal_type(1) 2 + reveal_type(1) # ty: ignore[undefined-reveal] | + + info[code-action]: Ignore 'undefined-reveal' in this file + --> main.py:2:1 + | + 2 | reveal_type(1) + | ^^^^^^^^^^^ + | + 1 + # ty: ignore[undefined-reveal] + 2 | + | "); } @@ -777,6 +1084,16 @@ mod tests { 2 + @deprecated("do not use") # ty: ignore[unresolved-reference] 3 | def my_func(): ... | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:2 + | + 2 | @deprecated("do not use") + | ^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "#); } @@ -812,6 +1129,16 @@ mod tests { - dt.timedelta(days=1) 2 + dt.timedelta(days=1) # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:1 + | + 2 | dt.timedelta(days=1) + | ^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -873,6 +1200,16 @@ mod tests { 4 + @deprecated("do not use") # ty: ignore[unresolved-reference] 5 | def my_func(): ... | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:4:2 + | + 4 | @deprecated("do not use") + | ^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "#); } @@ -908,6 +1245,16 @@ mod tests { - ExecutionLoader 2 + ExecutionLoader # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:2:1 + | + 2 | ExecutionLoader + | ^^^^^^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -947,6 +1294,16 @@ mod tests { - ExecutionLoader 3 + ExecutionLoader # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:1 + | + 3 | ExecutionLoader + | ^^^^^^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -995,6 +1352,16 @@ mod tests { - ExecutionLoader 3 + ExecutionLoader # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:3:1 + | + 3 | ExecutionLoader + | ^^^^^^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | + | "); } @@ -1026,6 +1393,16 @@ mod tests { - NamespaceReport() 1 + NamespaceReport() # ty: ignore[unresolved-reference] | + + info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:1:1 + | + 1 | NamespaceReport() + | ^^^^^^^^^^^^^^^ + | + 1 + # ty: ignore[unresolved-reference] + 2 | NamespaceReport() + | "); } diff --git a/crates/ty_ide/src/snapshots/ty_ide__code_action__tests__add_ignore_trailing_whitespace.snap b/crates/ty_ide/src/snapshots/ty_ide__code_action__tests__add_ignore_trailing_whitespace.snap index 40cf35facd..ac5f03e4c5 100644 --- a/crates/ty_ide/src/snapshots/ty_ide__code_action__tests__add_ignore_trailing_whitespace.snap +++ b/crates/ty_ide/src/snapshots/ty_ide__code_action__tests__add_ignore_trailing_whitespace.snap @@ -11,3 +11,13 @@ info[code-action]: Ignore 'unresolved-reference' for this line - b = a / 10 1 + b = a / 10 # ty: ignore[unresolved-reference] | + +info[code-action]: Ignore 'unresolved-reference' in this file + --> main.py:1:5 + | +1 | b = a / 10 + | ^ + | +1 + # ty: ignore[unresolved-reference] +2 | b = a / 10 + | diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 82181e8d4e..b8db5a4851 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -35,8 +35,8 @@ pub use semantic_model::{ }; use std::hash::BuildHasherDefault; pub use suppression::UNUSED_IGNORE_COMMENT; -pub use suppression::suppress_single; pub(crate) use suppression::{SuppressFix, is_unused_ignore_comment_lint, suppress_all}; +pub use suppression::{suppress_in_file, suppress_single}; use ty_module_resolver::ModuleGlobSet; pub use ty_python_core::Program; use ty_python_core::ProgramFile; diff --git a/crates/ty_python_semantic/src/suppression.rs b/crates/ty_python_semantic/src/suppression.rs index 835c8556d8..4e1cad5ae6 100644 --- a/crates/ty_python_semantic/src/suppression.rs +++ b/crates/ty_python_semantic/src/suppression.rs @@ -17,8 +17,8 @@ use rustc_hash::FxHasher; use crate::diagnostic::DiagnosticGuard; use crate::lint::{GetLintError, Level, LintMetadata, LintRegistry, LintStatus, TyCompat}; -pub use crate::suppression::add_ignore::suppress_single; pub(crate) use crate::suppression::add_ignore::{SuppressFix, suppress_all}; +pub use crate::suppression::add_ignore::{suppress_in_file, suppress_single}; use crate::suppression::parser::{ ParseError, ParseErrorKind, SuppressionComment, SuppressionParser, }; diff --git a/crates/ty_python_semantic/src/suppression/add_ignore.rs b/crates/ty_python_semantic/src/suppression/add_ignore.rs index 2bd716e9a7..21515ab237 100644 --- a/crates/ty_python_semantic/src/suppression/add_ignore.rs +++ b/crates/ty_python_semantic/src/suppression/add_ignore.rs @@ -16,6 +16,8 @@ use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::token::TokenKind; +use ruff_python_trivia::encoding_declaration; +use ruff_source_file::UniversalNewlineIterator; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use smallvec::SmallVec; @@ -180,6 +182,67 @@ pub fn suppress_single(db: &dyn Db, file: PythonFile<'_>, id: LintId, range: Tex add_end_of_line_suppression(&source, codes, suppression_range.end()) } +/// creates a fix to suppress a lint in the whole of `file` +/// +/// that is a `ty: ignore` comment on a line of its own at the top of the file, before any code, +/// which the typing spec reads as covering the whole file. an editable one there already takes +/// the code instead of a second comment being written; a new one goes on the first line, or +/// after a `#!` line or an encoding declaration, which have to stay where they are +pub fn suppress_in_file(db: &dyn Db, file: PythonFile<'_>, id: LintId) -> Fix { + let suppressions = suppressions(db, file); + let source = source_text(db, file.file(db)); + let codes = &[id.name()]; + + let existing = suppressions.file.iter().find_map(|suppression| { + let prefix = editable_suppression_prefix(&source[suppression.comment_range])?; + let separator = if prefix.ends_with('[') { "" } else { ", " }; + Some(ExistingSuppression { + insertion_offset: suppression.comment_range.start() + prefix.text_len(), + kind: suppression.kind, + separator, + }) + }); + if let Some(existing) = existing { + return add_to_existing_suppression(existing, codes); + } + + let offset = file_suppression_offset(&source); + // the line the comment goes after needs a newline ending it, which the last line of a file + // may not have + let before = source[..offset.to_usize()].trim_start_matches('\u{feff}'); + let leading = if before.is_empty() || before.ends_with(['\n', '\r']) { + "" + } else { + "\n" + }; + Fix::safe_edit(Edit::insertion( + format!( + "{leading}# ty: ignore[{codes}]\n", + codes = Codes(SuppressionKind::Ty, codes) + ), + offset, + )) +} + +/// where a new file-level suppression goes: after the lines python reads before any code, which +/// have to stay where they are. that is a `#!` line, which has to be the first, and a PEP 263 +/// encoding declaration, which python reads only on the first two lines. a byte order mark +/// stays ahead of everything +fn file_suppression_offset(source: &str) -> TextSize { + let start = if source.starts_with('\u{feff}') { + '\u{feff}'.text_len() + } else { + TextSize::ZERO + }; + if let Some(declaration) = encoding_declaration(source.as_bytes()) { + return declaration.line.end(); + } + match UniversalNewlineIterator::with_offset(&source[start.to_usize()..], start).next() { + Some(first) if first.starts_with("#!") => first.full_end(), + _ => start, + } +} + /// Returns the suppression range for the given `range`. /// /// The suppression range is defined as: @@ -339,3 +402,104 @@ impl std::fmt::Display for Codes<'_> { joiner.finish() } } + +#[cfg(test)] +mod tests { + use ruff_db::files::system_path_to_file; + use ruff_db::source::source_text; + use ruff_db::{PythonFile, system::DbWithWritableSystem as _}; + use ruff_text_size::Ranged as _; + + use super::suppress_in_file; + use crate::Db as _; + use crate::db::tests::TestDbBuilder; + use crate::suppression::suppressions; + + /// `source` with a file-level suppression of `unresolved-reference` added, checking that the + /// comment it adds is one that covers the whole file + fn suppressed_in_file(source: &str) -> String { + let mut db = TestDbBuilder::new() + .with_file("test.py", source) + .build() + .unwrap(); + let file = system_path_to_file(&db, "test.py").unwrap(); + let lint = db.lint_registry().get("unresolved-reference").unwrap(); + let fix = suppress_in_file(&db, PythonFile::new(&db, file, db.python_version()), lint); + let mut suppressed = source.to_owned(); + for edit in fix.edits().iter().rev() { + suppressed.replace_range( + edit.start().to_usize()..edit.end().to_usize(), + edit.content().unwrap_or_default(), + ); + } + db.write_file("test.py", &suppressed).unwrap(); + let file = system_path_to_file(&db, "test.py").unwrap(); + assert_eq!(source_text(&db, file).as_str(), suppressed); + let python_file = PythonFile::new(&db, file, db.python_version()); + assert_eq!( + suppressions(&db, python_file).file.len(), + 1, + "{suppressed:?}" + ); + suppressed + } + + #[test] + fn file_suppression_goes_first() { + assert_eq!( + suppressed_in_file("a = b\n"), + "# ty: ignore[unresolved-reference]\na = b\n" + ); + } + + #[test] + fn file_suppression_goes_after_an_encoding_declaration() { + assert_eq!( + suppressed_in_file("# -*- coding: latin-1 -*-\na = b\n"), + "# -*- coding: latin-1 -*-\n# ty: ignore[unresolved-reference]\na = b\n" + ); + assert_eq!( + suppressed_in_file("#!/usr/bin/env python\n# vim: set fileencoding=utf-8 :\na = b\n"), + "#!/usr/bin/env python\n# vim: set fileencoding=utf-8 :\n\ + # ty: ignore[unresolved-reference]\na = b\n" + ); + assert_eq!( + suppressed_in_file("# a comment\n# coding=utf-8\na = b\n"), + "# a comment\n# coding=utf-8\n# ty: ignore[unresolved-reference]\na = b\n" + ); + } + + /// python reads a declaration on the second line only when the first could not be code, and + /// a comment that only mentions an encoding is not one + #[test] + fn file_suppression_goes_first_without_an_encoding_declaration() { + assert_eq!( + suppressed_in_file("# no coding here\na = b\n"), + "# ty: ignore[unresolved-reference]\n# no coding here\na = b\n" + ); + assert_eq!( + suppressed_in_file("a = b\n# coding: utf-8\n"), + "# ty: ignore[unresolved-reference]\na = b\n# coding: utf-8\n" + ); + } + + #[test] + fn file_suppression_goes_after_a_byte_order_mark() { + assert_eq!( + suppressed_in_file("\u{feff}a = b\n"), + "\u{feff}# ty: ignore[unresolved-reference]\na = b\n" + ); + assert_eq!( + suppressed_in_file("\u{feff}# coding: utf-8\r\na = b\r\n"), + "\u{feff}# coding: utf-8\r\n# ty: ignore[unresolved-reference]\na = b\r\n" + ); + } + + #[test] + fn file_suppression_after_a_last_line_without_a_newline() { + assert_eq!( + suppressed_in_file("#!/usr/bin/env python"), + "#!/usr/bin/env python\n# ty: ignore[unresolved-reference]\n" + ); + } +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__an_unresolved_import_in_a_uv_project_is_offered_the_command_that_installs.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__an_unresolved_import_in_a_uv_project_is_offered_the_command_that_installs.snap index 77615e97b3..6c6cabf407 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__an_unresolved_import_in_a_uv_project_is_offered_the_command_that_installs.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__an_unresolved_import_in_a_uv_project_is_offered_the_command_that_installs.snap @@ -83,5 +83,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-import' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 7 + }, + "end": { + "line": 0, + "character": 12 + } + }, + "severity": 1, + "code": "unresolved-import", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-import" + }, + "source": "ty", + "message": "Cannot resolve imported module `numpy`\n\ninfo: Searched in the following paths during module resolution:\ninfo: 1. /src (first-party code)\ninfo: 2. vendored://stdlib (stdlib typeshed stubs vendored by ty)\ninfo: make sure your Python environment is properly configured: https://docs.astral.sh/ty/modules/#python-environment" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-import]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__attribute_missing_only_on_none_is_fixed_with_optional_chaining.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__attribute_missing_only_on_none_is_fixed_with_optional_chaining.snap index 9114b4ddb5..59034d4887 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__attribute_missing_only_on_none_is_fixed_with_optional_chaining.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__attribute_missing_only_on_none_is_fixed_with_optional_chaining.snap @@ -92,5 +92,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-attribute' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 4, + "character": 11 + }, + "end": { + "line": 4, + "character": 22 + } + }, + "severity": 1, + "code": "unresolved-attribute", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-attribute" + }, + "source": "ty", + "message": "Attribute `name` is not defined on `None` in union `User | None`\n\nhelp: Use `?.` to read `name` only when the value is not `None`" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.by": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-attribute]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action.snap index 282da9eed1..45de175c07 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action.snap @@ -98,5 +98,53 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unused-ignore-comment' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 12 + }, + "end": { + "line": 0, + "character": 42 + } + }, + "severity": 2, + "code": "unused-ignore-comment", + "codeDescription": { + "href": "https://ty.dev/rules#unused-ignore-comment" + }, + "source": "ty", + "message": "Unused `ty: ignore` directive\n\nhelp: Remove the unused suppression comment", + "tags": [ + 1 + ] + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unused-ignore-comment]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap index 67f77cb3ea..12bf198438 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap @@ -95,5 +95,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 3 + }, + "end": { + "line": 0, + "character": 9 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `typing` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap index 105a8654bc..2d2beb7110 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap @@ -182,5 +182,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 3, + "character": 1 + }, + "end": { + "line": 3, + "character": 11 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `deprecated` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap index d2acbb63dd..d3d588d230 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap @@ -47,5 +47,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 1, + "character": 5 + }, + "end": { + "line": 1, + "character": 11 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `foobar` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_possible_missing_submodule_attribute.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_possible_missing_submodule_attribute.snap index 7310c18cff..bfe41f3e62 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_possible_missing_submodule_attribute.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_possible_missing_submodule_attribute.snap @@ -47,5 +47,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'possibly-missing-submodule' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 1, + "character": 0 + }, + "end": { + "line": 1, + "character": 11 + } + }, + "severity": 2, + "code": "possibly-missing-submodule", + "codeDescription": { + "href": "https://ty.dev/rules#possibly-missing-submodule" + }, + "source": "ty", + "message": "Submodule `parser` might not have been imported\n\nhelp: Consider explicitly importing `html.parser`" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[possibly-missing-submodule]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap index 2111a88479..f0aa232746 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap @@ -137,5 +137,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 1, + "character": 1 + }, + "end": { + "line": 1, + "character": 11 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `deprecated` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap index ff9357c46e..885f683dd6 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap @@ -137,5 +137,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 3 + }, + "end": { + "line": 0, + "character": 10 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `Literal` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reveal.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reveal.snap index b3560995e7..8c6aafc00d 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reveal.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reveal.snap @@ -92,5 +92,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'undefined-reveal' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 11 + } + }, + "severity": 2, + "code": "undefined-reveal", + "codeDescription": { + "href": "https://ty.dev/rules#undefined-reveal" + }, + "source": "ty", + "message": "`reveal_type` used without importing it\n\ninfo: This is allowed for debugging convenience but will fail at runtime\nhelp: Import `reveal_type` from `typing`" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[undefined-reveal]\n" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_unsafe_fix.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_unsafe_fix.snap index dd6e2c7a06..09c84c4d96 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_unsafe_fix.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_unsafe_fix.snap @@ -98,5 +98,53 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unused-ignore-comment' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 31 + } + }, + "severity": 2, + "code": "unused-ignore-comment", + "codeDescription": { + "href": "https://ty.dev/rules#unused-ignore-comment" + }, + "source": "ty", + "message": "Unused `ty: ignore` directive\n\nhelp: Remove the unused suppression comment", + "tags": [ + 1 + ] + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 29 + }, + "end": { + "line": 0, + "character": 29 + } + }, + "newText": ", unused-ignore-comment" + } + ] + } + } } ] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap index 11cea2b4d8..9ada88e52e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap @@ -137,5 +137,50 @@ expression: code_actions ] } } + }, + { + "title": "Ignore 'unresolved-reference' in this file", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 3 + }, + "end": { + "line": 0, + "character": 10 + } + }, + "severity": 1, + "code": "Click for full diagnostic", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `Literal` used when not defined" + } + ], + "isPreferred": false, + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "# ty: ignore[unresolved-reference]\n" + } + ] + } + } } ] diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 014e524ba0..0167e83c3a 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -634,6 +634,16 @@ the caret goes in the document as the edits leave it: like the other document requests it takes `textHash`, and it is `null` when language services are off +## suppressing a diagnostic + +every diagnostic has two quick fixes that silence it rather than fix it. +*ignore for this line* writes `# ty: ignore[]` at the end of its line, +and *ignore in this file* writes it on a line of its own at the top of the +file, before any code, which suppresses the rule throughout the file. a +suppression already there takes the rule instead of a second comment being +written, and a `#!` line stays first. ty has no suppression that covers one +function or one class, so neither does the editor + ## hover a hover opens with what the name is: its kind and the dotted name it is From 79fed1936d4b94bea46d098dd8727babcb13097c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:34:28 +1000 Subject: [PATCH 021/110] register buff's formatting and range formatting for basedpython documents too, for a client that registers them dynamically, and test that a .by document's range is formatted on its own --- crates/ruff_server/src/server/main_loop.rs | 7 +++ crates/ruff_server/tests/e2e/capabilities.rs | 57 +++++++++++++++++++- docs/basedpython/features/linter.md | 11 ++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/crates/ruff_server/src/server/main_loop.rs b/crates/ruff_server/src/server/main_loop.rs index 34357d22f5..b8dd0f0aed 100644 --- a/crates/ruff_server/src/server/main_loop.rs +++ b/crates/ruff_server/src/server/main_loop.rs @@ -195,6 +195,13 @@ impl Server { pattern: None, }) .into(), + // a `.by` file, which buff formats as it does python + types::TextDocumentFilter::Language(types::TextDocumentFilterLanguage { + language: "basedpython".to_string(), + scheme: None, + pattern: None, + }) + .into(), types::TextDocumentFilter::Language(types::TextDocumentFilterLanguage { language: "markdown".to_string(), scheme: None, diff --git a/crates/ruff_server/tests/e2e/capabilities.rs b/crates/ruff_server/tests/e2e/capabilities.rs index b84179db6e..abe12d71b7 100644 --- a/crates/ruff_server/tests/e2e/capabilities.rs +++ b/crates/ruff_server/tests/e2e/capabilities.rs @@ -1,7 +1,9 @@ use anyhow::Result; use insta::assert_json_snapshot; use lsp_types::Request as _; -use lsp_types::{DocumentFormattingRequest, DocumentRangeFormattingRequest, RegistrationRequest}; +use lsp_types::{ + DocumentFormattingRequest, DocumentRangeFormattingRequest, Position, Range, RegistrationRequest, +}; use crate::TestServerBuilder; @@ -39,7 +41,8 @@ fn statically_registers_formatting_when_dynamic_registration_is_unsupported() -> } #[test] -fn dynamically_registers_formatting_and_range_formatting_for_python_and_markdown() -> Result<()> { +fn dynamically_registers_formatting_and_range_formatting_for_python_basedpython_and_markdown() +-> Result<()> { let mut server = TestServerBuilder::new()? .enable_formatting_dynamic_registration(true) .enable_range_formatting_dynamic_registration(true) @@ -73,6 +76,9 @@ fn dynamically_registers_formatting_and_range_formatting_for_python_and_markdown { "language": "python" }, + { + "language": "basedpython" + }, { "language": "markdown" }, @@ -91,3 +97,50 @@ fn dynamically_registers_formatting_and_range_formatting_for_python_and_markdown Ok(()) } + +/// a `.by` document is range-formatted as python is: only the statements the range covers are laid +/// out again +#[test] +fn range_formats_a_basedpython_document() -> Result<()> { + let mut server = TestServerBuilder::new()?.with_workspace(".")?.build(); + server.open_text_document_with_language_id( + "test.by", + "basedpython", + "x=[1,2]\ny = {'a':1}\n", + 1, + ); + + let edits = server.format_range_request( + "test.by", + Range { + start: Position { + line: 1, + character: 0, + }, + end: Position { + line: 1, + character: 15, + }, + }, + ); + + assert_json_snapshot!(edits, @r#" + [ + { + "range": { + "start": { + "line": 1, + "character": 0 + }, + "end": { + "line": 1, + "character": 15 + } + }, + "newText": "y = {\"a\": 1}" + } + ] + "#); + + Ok(()) +} diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 8e17f34eb5..4049106605 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -56,6 +56,17 @@ into result = a ?? b ``` +## formatting in an editor + +`buff server` formats a `.by` document as it formats python, the whole of it +with `textDocument/formatting` and a selection with +`textDocument/rangeFormatting`, which lays out again only the statements the +range covers. to a client that registers formatting dynamically it registers +both for the `basedpython` language as well as `python`, so a client that names +`.by` documents `basedpython` is offered them. the server reports its name as +`buff` in the `serverInfo` of its `initialize` answer, and `buff` is the source of +its diagnostics + ## what the linter does not check the linter reads one file at a time and has no types. anything that takes a type From 5373e1c72a2c1ae6092bbcea82b9479b42cdc9c0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:39:09 +1000 Subject: [PATCH 022/110] mark the name of a def or class nested in a function functionScope where it is declared, and colour a name as a method only when the def it names is made in a class body, so field called in a dataclass's body is a function --- crates/ty_ide/src/semantic_tokens.rs | 103 +++++++++++++++++++++++---- docs/basedpython/features/editor.md | 9 ++- 2 files changed, 95 insertions(+), 17 deletions(-) diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 761a46a176..2845bb939c 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -57,8 +57,8 @@ use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; use ty_python_core::scope::ScopeKind; use ty_python_semantic::{ - HasType, ImportAliasResolution, ResolvedDefinition, SemanticModel, definitions_for_attribute, - definitions_for_imported_symbol, + HasDefinition, HasType, ImportAliasResolution, ResolvedDefinition, SemanticModel, + definitions_for_attribute, definitions_for_imported_symbol, types::format::{SpecLanguage, spec_language}, types::ide_support::{ CallArgumentForm, call_argument_forms, definition_for_name, @@ -968,8 +968,10 @@ impl<'db> SemanticTokenVisitor<'db> { match definition.kind(db) { DefinitionKind::Function(_) => { - // Check if this is a method based on current scope - if self.in_class_scope { + // a method is a `def` made in a class body, wherever it is used + // from: `field` called in a dataclass's body is the function + // `dataclasses` defines + if definition.scope(db).scope(db).kind() == ScopeKind::Class { Some((SemanticTokenType::Method, modifiers)) } else { Some((SemanticTokenType::Function, modifiers)) @@ -1776,14 +1778,19 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { // token by the synthetic `__init_method__` decorator above. if !is_init_method(func) { // Function name + let mut modifiers = SemanticTokenModifier::DEFINITION; + if func.is_async { + modifiers |= SemanticTokenModifier::ASYNC; + } + // a `def` nested in a function binds a local, as an + // assignment there would + if self.is_function_scoped(func.definition(self.model)) { + modifiers |= SemanticTokenModifier::FUNCTION_SCOPE; + } self.add_token( func.name.range(), self.classify_function_definition(func), - if func.is_async { - SemanticTokenModifier::DEFINITION | SemanticTokenModifier::ASYNC - } else { - SemanticTokenModifier::DEFINITION - }, + modifiers, ); } @@ -1854,12 +1861,12 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.visit_decorator(decorator); } - // Class name - self.add_token( - class.name.range(), - SemanticTokenType::Class, - SemanticTokenModifier::DEFINITION, - ); + // class name, a local when the class is declared in a function + let mut modifiers = SemanticTokenModifier::DEFINITION; + if self.is_function_scoped(class.definition(self.model)) { + modifiers |= SemanticTokenModifier::FUNCTION_SCOPE; + } + self.add_token(class.name.range(), SemanticTokenType::Class, modifiers); // Type parameters (Python 3.12+ syntax) if let Some(type_params) = &class.type_params { @@ -6370,6 +6377,71 @@ def clamp(value: int) -> int: ); } + /// a `def` or a class nested in a function binds a local, so its name is marked where + /// it is declared as well as where it is used; a module's are not + #[test] + fn function_scope_marks_nested_definitions() { + let test = SemanticTokenTest::new( + r#" +def outer(): + def inner(): + pass + + class Local: + pass + + return inner, Local +"#, + ); + + let tokens = test.highlight_file(); + + assert_eq!( + test.function_scoped_names(&tokens), + ["inner@22", "Local@55", "inner@87", "Local@94"] + ); + } + + /// a function is a method where it is defined in a class body, not where it is used: + /// `field` called in a dataclass's body is the function `dataclasses` defines + #[test] + fn a_function_used_in_a_class_body_is_a_function() { + let test = SemanticTokenTest::new( + r#" +from dataclasses import dataclass, field + +@dataclass +class P: + x: list[int] = field(default_factory=list) + + def m(self) -> None: ... + + alias = m +"#, + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "dataclasses" @ 6..17: Namespace + "dataclass" @ 25..34: Function + "field" @ 36..41: Function + "dataclass" @ 44..53: Decorator + "P" @ 60..61: Class [definition] + "x" @ 67..68: Variable [definition] + "list" @ 70..74: Class + "int" @ 75..78: Class + "field" @ 82..87: Function + "default_factory" @ 88..103: Parameter + "list" @ 104..108: Class + "m" @ 119..120: Method [definition] + "self" @ 121..125: SelfParameter [definition] + "None" @ 130..134: BuiltinConstant + "alias" @ 145..150: Variable [definition] + "m" @ 153..154: Method + "#); + } + /// A lambda's parameters and a comprehension's variables are the lambda's and the /// comprehension's own, wherever they are written. #[test] @@ -6455,6 +6527,7 @@ def outer(): "x@28", "x@34", "count@67", + "inner@85", "count@111", "count@146", "count@176", diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 0167e83c3a..8fc4f14974 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -543,8 +543,13 @@ where one fits; the others are basedpython's own class attribute. it is the binding that decides, not where the name is written: a global read inside a function is not marked, a `nonlocal` name is, and an assignment expression in a comprehension at the top of a module binds a global. -an editor that colours each local by its name ("rainbow" highlighting) reads it -off this modifier +a `def` or a class nested in a function is a local too, and its name is marked +where it is declared as well as where it is used. an editor that colours each +local by its name ("rainbow" highlighting) reads it off this modifier + +a name is a `method` where the `def` it names is made in a class body, wherever +the name is used: `field` called in a dataclass's body is a `function`, the one +`dataclasses` defines `annotation` marks every token of a type expression: a parameter's or a variable's annotation, what a function returns or raises, a type alias's value, From b4975ef25d38162bbed7fb3867cff16d96bfea07 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:41:33 +1000 Subject: [PATCH 023/110] answer buff/formatSettings, the line length, indentation, quote style and line ending buff formats a document with and the configuration file they were read from, for an editor to show the project's code style and draw its right margin where buff wraps --- crates/ruff_server/src/server/api.rs | 3 + crates/ruff_server/src/server/api/requests.rs | 2 + .../server/api/requests/format_settings.rs | 103 ++++++++++++++++++ .../src/session/index/ruff_settings.rs | 5 +- .../ruff_server/tests/e2e/format_settings.rs | 81 ++++++++++++++ crates/ruff_server/tests/e2e/main.rs | 1 + crates/ruff_workspace/src/lib.rs | 2 +- docs/basedpython/features/linter.md | 28 +++++ 8 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 crates/ruff_server/src/server/api/requests/format_settings.rs create mode 100644 crates/ruff_server/tests/e2e/format_settings.rs diff --git a/crates/ruff_server/src/server/api.rs b/crates/ruff_server/src/server/api.rs index 6c02e6b19a..5019b151a4 100644 --- a/crates/ruff_server/src/server/api.rs +++ b/crates/ruff_server/src/server/api.rs @@ -64,6 +64,9 @@ pub(super) fn request(req: server::Request) -> Task { request::Format::METHOD => { background_request_task::(req, BackgroundSchedule::Fmt) } + request::FormatSettingsHandler::METHOD => background_request_task::< + request::FormatSettingsHandler, + >(req, BackgroundSchedule::Worker), request::FormatRange::METHOD => { background_request_task::(req, BackgroundSchedule::Fmt) } diff --git a/crates/ruff_server/src/server/api/requests.rs b/crates/ruff_server/src/server/api/requests.rs index c9a9439f10..ca14be4896 100644 --- a/crates/ruff_server/src/server/api/requests.rs +++ b/crates/ruff_server/src/server/api/requests.rs @@ -5,6 +5,7 @@ mod execute_command; mod explain_rule; mod format; mod format_range; +mod format_settings; mod hover; mod shutdown; @@ -22,6 +23,7 @@ pub(super) use execute_command::ExecuteCommand; pub(super) use explain_rule::ExplainRule; pub(super) use format::Format; pub(super) use format_range::FormatRange; +pub(super) use format_settings::FormatSettingsHandler; pub(super) use hover::Hover; pub(super) use shutdown::ShutdownHandler; diff --git a/crates/ruff_server/src/server/api/requests/format_settings.rs b/crates/ruff_server/src/server/api/requests/format_settings.rs new file mode 100644 index 0000000000..d3b550fb05 --- /dev/null +++ b/crates/ruff_server/src/server/api/requests/format_settings.rs @@ -0,0 +1,103 @@ +//! `buff/formatSettings` — the formatter settings `buff` lays a document out with +//! +//! what an editor shows as the document's code style, and where it draws its right margin, has to +//! be what the formatter will do: `buff` resolves them from the project's configuration, its +//! `pyproject.toml` or `ruff.toml` and whatever the editor's settings override in it, and that +//! resolution is the server's alone. an editor reading `line-length` out of a file itself would +//! pick the wrong file in a nested project, and miss the defaults and the editor's own overrides + +use lsp_types::{LspRequestMethod, MessageDirection, Request, TextDocumentIdentifier, Uri}; +use ruff_formatter::IndentStyle; +use ruff_python_formatter::QuoteStyle; +use ruff_workspace::LineEnding; + +use crate::server::Result; +use crate::session::{Client, DocumentSnapshot}; + +pub(crate) enum FormatSettingsRequest {} + +impl Request for FormatSettingsRequest { + type Params = FormatSettingsParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("buff/formatSettings"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the document to answer for. a field it does not name, a `workDoneToken` among them, is +/// ignored, as LSP's own requests ignore one +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct FormatSettingsParams { + text_document: TextDocumentIdentifier, +} + +/// the settings, as the configuration spells them +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct FormatSettings { + /// `line-length`: the column a line is wrapped before + line_length: u16, + /// `indent-width`: the columns one level of indentation takes + indent_width: u32, + /// `indent-style`: `space` or `tab` + indent_style: String, + /// `quote-style`: `double`, `single` or `preserve` + quote_style: String, + /// `line-ending`: `auto`, `lf`, `cr-lf` or `native` + line_ending: String, + /// the configuration file the settings were read from, `null` when no file configures the + /// document and the settings are the defaults with the editor's overrides + configuration_file: Option, +} + +pub(crate) struct FormatSettingsHandler; + +impl super::RequestHandler for FormatSettingsHandler { + type RequestType = FormatSettingsRequest; +} + +impl super::BackgroundDocumentRequestHandler for FormatSettingsHandler { + super::define_document_uri!(params: &FormatSettingsParams); + + /// `null` for a document the editor has not opened + fn run_with_snapshot( + snapshot: Self::Snapshot, + _client: &Client, + _params: FormatSettingsParams, + ) -> Result> { + let snapshot: DocumentSnapshot = match snapshot { + Ok(snapshot) => snapshot, + Err(uri) => { + tracing::warn!("Returning no format settings because document `{uri}` isn't open."); + return Ok(None); + } + }; + let settings = snapshot.query().settings(); + let formatter = &settings.formatter; + Ok(Some(FormatSettings { + line_length: formatter.line_width.value(), + indent_width: formatter.indent_width.value(), + indent_style: match formatter.indent_style { + IndentStyle::Space => "space", + IndentStyle::Tab => "tab", + } + .to_string(), + quote_style: match formatter.quote_style { + QuoteStyle::Double => "double", + QuoteStyle::Single => "single", + QuoteStyle::Preserve => "preserve", + } + .to_string(), + line_ending: match formatter.line_ending { + LineEnding::Auto => "auto", + LineEnding::Lf => "lf", + LineEnding::CrLf => "cr-lf", + LineEnding::Native => "native", + } + .to_string(), + configuration_file: settings + .path() + .and_then(|path| Uri::from_file_path(path).ok()), + })) + } +} diff --git a/crates/ruff_server/src/session/index/ruff_settings.rs b/crates/ruff_server/src/session/index/ruff_settings.rs index d1ab2c317c..d5566f0aa7 100644 --- a/crates/ruff_server/src/session/index/ruff_settings.rs +++ b/crates/ruff_server/src/session/index/ruff_settings.rs @@ -24,8 +24,9 @@ use crate::session::settings::{EditorSettings, ResolvedConfiguration}; #[derive(Debug)] pub(crate) struct RuffSettings { - /// The path to this configuration file, used for debugging. - /// The default fallback configuration does not have a file path. + /// the path to this configuration file, which `buff/formatSettings` answers as the file a + /// document's settings were read from. the default fallback configuration does not have a + /// file path path: Option, /// The resolved settings. settings: Settings, diff --git a/crates/ruff_server/tests/e2e/format_settings.rs b/crates/ruff_server/tests/e2e/format_settings.rs new file mode 100644 index 0000000000..6c046b7b77 --- /dev/null +++ b/crates/ruff_server/tests/e2e/format_settings.rs @@ -0,0 +1,81 @@ +//! `buff/formatSettings` over the wire: the settings `buff` formats a document with, resolved from +//! the configuration that governs it + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; + +use crate::TestServerBuilder; + +/// the request as a client sends it, in json throughout +enum FormatSettings {} + +impl Request for FormatSettings { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("buff/formatSettings"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[test] +fn a_document_is_answered_with_the_configuration_that_governs_it() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_workspace(".")? + .with_file( + "pyproject.toml", + "[tool.ruff]\nline-length = 100\n\n[tool.ruff.format]\nquote-style = \"single\"\n", + )? + .build(); + server.open_text_document_with_language_id("test.by", "basedpython", "x = 1\n", 1); + + let uri = server.file_uri("test.by"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": uri }, + })); + let answer = server + .await_response::(&id) + .expect("settings for an open document"); + + assert_eq!(answer["lineLength"], 100); + assert_eq!(answer["indentWidth"], 4); + assert_eq!(answer["indentStyle"], "space"); + assert_eq!(answer["quoteStyle"], "single"); + assert_eq!(answer["lineEnding"], "auto"); + assert_eq!( + answer["configurationFile"], + serde_json::json!(server.file_uri("pyproject.toml")) + ); + Ok(()) +} + +#[test] +fn a_document_that_is_not_open_is_answered_null() -> Result<()> { + let mut server = TestServerBuilder::new()?.with_workspace(".")?.build(); + + let uri = server.file_uri("missing.by"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": uri }, + })); + + assert_eq!(server.await_response::(&id), None); + Ok(()) +} + +/// a field the request does not name, such as the `workDoneToken` LSP lets any request carry, is +/// left alone, so a client that sends more than this server reads is still answered +#[test] +fn a_request_with_a_field_it_does_not_name_is_answered() -> Result<()> { + let mut server = TestServerBuilder::new()?.with_workspace(".")?.build(); + server.open_text_document_with_language_id("test.by", "basedpython", "x = 1\n", 1); + + let uri = server.file_uri("test.by"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": uri }, + "workDoneToken": "progress", + })); + let answer = server + .await_response::(&id) + .expect("settings for an open document"); + + assert_eq!(answer["lineLength"], 88); + Ok(()) +} diff --git a/crates/ruff_server/tests/e2e/main.rs b/crates/ruff_server/tests/e2e/main.rs index bcc0ee6d6b..f94ed16ac7 100644 --- a/crates/ruff_server/tests/e2e/main.rs +++ b/crates/ruff_server/tests/e2e/main.rs @@ -30,6 +30,7 @@ mod code_action; mod custom_extension; mod diagnostics; mod execute_command; +mod format_settings; mod hover; mod notebook; mod workspace; diff --git a/crates/ruff_workspace/src/lib.rs b/crates/ruff_workspace/src/lib.rs index bc8505c346..5ac23d8deb 100644 --- a/crates/ruff_workspace/src/lib.rs +++ b/crates/ruff_workspace/src/lib.rs @@ -5,7 +5,7 @@ pub mod resolver; mod settings; -pub use settings::{FileResolverSettings, FormatterSettings, Settings}; +pub use settings::{FileResolverSettings, FormatterSettings, LineEnding, Settings}; #[cfg(test)] mod tests { diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 4049106605..6638cefc74 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -67,6 +67,34 @@ both for the `basedpython` language as well as `python`, so a client that names `buff` in the `serverInfo` of its `initialize` answer, and `buff` is the source of its diagnostics +an editor that shows the project's code style, or draws its right margin +where `buff` wraps, asks `buff/formatSettings` for a document it has opened: + +```json +{ "textDocument": { "uri": "file:///src/main.by" } } +``` + +and is answered with the settings `buff` formats that document with, resolved +from the configuration that governs it and whatever the editor's own settings +override in it, spelled as the configuration spells them: + +```json +{ + "lineLength": 100, + "indentWidth": 4, + "indentStyle": "space", + "quoteStyle": "single", + "lineEnding": "auto", + "configurationFile": "file:///pyproject.toml" +} +``` + +`indentStyle` is `space` or `tab`, `quoteStyle` is `double`, `single` or +`preserve`, and `lineEnding` is `auto`, `lf`, `cr-lf` or `native`. +`configurationFile` is the file the settings were read from, and `null` when no +file configures the document. the answer is `null` for a document the editor has +not opened + ## what the linter does not check the linter reads one file at a time and has no types. anything that takes a type From 5e4cc4ae7e3f6568167910db734b1b2c82dbd9b9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:04:07 +1000 Subject: [PATCH 024/110] extract a run of operands an associative operator chains, b + c in a + b + c over integers or strings, ask the checker whether what is evaluated before an expression could observe it moving, so radius ** 2 is extracted from scale * math.pi * radius ** 2, and offer an extraction for each expression around a caret when one is asked for explicitly, each naming its expression and carrying its range --- crates/ty_ide/src/refactor.rs | 68 ++- crates/ty_ide/src/refactor/evaluation.rs | 180 ++++++- .../ty_ide/src/refactor/extract_variable.rs | 457 ++++++++++++++++-- crates/ty_ide/src/refactor/inline_variable.rs | 7 +- .../src/types/ide_support.rs | 191 ++++++++ .../src/server/api/requests/code_action.rs | 77 ++- crates/ty_server/tests/e2e/refactors.rs | 46 ++ ...aret_offers_the_expressions_around_it.snap | 36 ++ docs/basedpython/features/editor.md | 32 ++ 9 files changed, 1017 insertions(+), 77 deletions(-) create mode 100644 crates/ty_server/tests/e2e/snapshots/e2e__refactors__extraction_at_a_caret_offers_the_expressions_around_it.snap diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 950dfe4fe0..4f6ec5a33c 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -107,6 +107,9 @@ impl RefactorKind { pub struct RefactorOffer { pub kind: RefactorKind, pub title: String, + /// The range to ask about to have this rewrite again: the range the offer was asked for, + /// or, for one of the expressions offered around a caret, that expression. + pub range: TextRange, /// `Ok` holds the rewrite and `Err` why there is none. /// /// The edits come with the offer because working them out is how the question was answered: @@ -221,11 +224,18 @@ impl<'db> RefactorContext<'db> { /// `kinds` is asked for rather than assumed because answering for one costs working its rewrite /// out, so a caller that will not offer a refactoring should not name it — a client narrowing a /// `textDocument/codeAction` with `only` then pays for nothing it asked no question about. +/// +/// With `around_caret`, an empty `range` is answered for extracting a variable or a constant +/// with one offer for each expression around the caret, innermost first, each carrying the +/// expression's range: what an editor offers to choose from when extract is asked for with +/// nothing selected. Without it a caret extracts nothing, which is what a client asking while +/// the caret moves wants, rather than an offer for every expression the caret is in. pub fn refactors( db: &dyn Db, file: ProgramFile<'_>, range: TextRange, kinds: &[RefactorKind], + around_caret: bool, ) -> Vec { if kinds.is_empty() || !is_refactorable(db, file) { return Vec::new(); @@ -235,23 +245,36 @@ pub fn refactors( return Vec::new(); } - kinds - .iter() - .copied() - .filter_map(|kind| match context.plan(kind, range) { + let mut offers = Vec::new(); + for &kind in kinds { + let extracted = match kind { + RefactorKind::ExtractVariable => Some(extract_variable::Target::Variable), + RefactorKind::IntroduceConstant => Some(extract_variable::Target::Constant), + _ => None, + }; + let plans = match extracted { + Some(target) if around_caret && range.is_empty() => { + extract_variable::around_caret(&context, range.start(), target) + } + _ => vec![(range, context.plan(kind, range))], + }; + offers.extend(plans.into_iter().filter_map(|(range, plan)| match plan { Ok(plan) => Some(RefactorOffer { kind, title: plan.title, + range, availability: Ok(context.file_edits(plan.edits)), }), Err(Refusal::NotApplicable) => None, Err(Refusal::Refused { title, reason }) => Some(RefactorOffer { kind, title, + range, availability: Err(reason), }), - }) - .collect() + })); + } + offers } /// The edits of the refactoring `kind` at `range`, or why it cannot be applied. @@ -341,10 +364,41 @@ pub(crate) mod test_support { Self { db, file, range } } + /// What `kind` offers around the test's caret, one line for each offer: the text it + /// is about, its title, and why it is refused when it is. Each offer that applies is + /// checked to be the rewrite asking about its own range gives. + pub(crate) fn around_caret(&self, kind: RefactorKind) -> String { + let program_file = self.db.program_file(self.file); + let source = ruff_db::source::source_text(&self.db, self.file); + refactors(&self.db, program_file, self.range, &[kind], true) + .into_iter() + .map(|offer| { + let text = &source.as_str()[offer.range]; + match offer.availability { + Ok(edits) => { + let again = refactor(&self.db, program_file, kind, offer.range) + .expect("an offer's range to give its rewrite again"); + assert_eq!( + edits.iter().map(|edit| &edit.edit).collect::>(), + again + .edits + .iter() + .map(|edit| &edit.edit) + .collect::>(), + ); + format!("{text}: {}", offer.title) + } + Err(reason) => format!("{text}: refused: {} ({reason})", offer.title), + } + }) + .collect::>() + .join("\n") + } + /// The file after applying `kind`, or why it was not offered or refused. pub(crate) fn apply(&self, kind: RefactorKind) -> String { let program_file = self.db.program_file(self.file); - let offers = refactors(&self.db, program_file, self.range, RefactorKind::ALL); + let offers = refactors(&self.db, program_file, self.range, RefactorKind::ALL, false); let Some(offer) = offers.iter().find(|offer| offer.kind == kind) else { return "not offered".to_string(); }; diff --git a/crates/ty_ide/src/refactor/evaluation.rs b/crates/ty_ide/src/refactor/evaluation.rs index 6375bc6c00..156fc19c66 100644 --- a/crates/ty_ide/src/refactor/evaluation.rs +++ b/crates/ty_ide/src/refactor/evaluation.rs @@ -6,8 +6,12 @@ //! its statement runs, and when nothing evaluated before it could observe the //! move. This module answers both, following python's evaluation order. -use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt, UnaryOp}; +use ruff_python_ast::{self as ast, AnyNodeRef, CmpOp, Expr, Operator, Stmt, UnaryOp}; use ruff_text_size::Ranged; +use ty_python_semantic::SemanticModel; +use ty_python_semantic::types::ide_support::{ + BuiltinScalar, builtin_scalar_kinds, is_module_value, +}; /// Where a target expression sits in the evaluation of its statement. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -85,8 +89,13 @@ pub(crate) fn is_pure(expr: &Expr) -> bool { /// Where the expression `target` sits in the evaluation of `statement`. `target` /// must be a node of `statement`'s tree, since it is found by identity. -pub(crate) fn placement_in_statement(statement: &Stmt, target: AnyNodeRef<'_>) -> Placement { +pub(crate) fn placement_in_statement( + model: &SemanticModel<'_>, + statement: &Stmt, + target: AnyNodeRef<'_>, +) -> Placement { let mut walk = Walk { + model, target, effects: false, }; @@ -94,14 +103,157 @@ pub(crate) fn placement_in_statement(statement: &Stmt, target: AnyNodeRef<'_>) - .unwrap_or(Placement::NotOnce(NotOnce::Opaque)) } -struct Walk<'a> { +/// Whether evaluating `expr` has no effect at all: it runs no code of the program's, +/// binds nothing and raises nothing, so evaluating something else before it or after it +/// cannot tell the difference. +/// +/// Wider than [`is_pure`], because it asks the checker what the operands are: `a + b` +/// over two `int`s is python's own addition, and `math.pi` looks a name up in a +/// module, where the same spelling over values of a class the program wrote could +/// run any `__add__` or property. +pub(crate) fn is_effect_free(model: &SemanticModel<'_>, expr: &Expr) -> bool { + if is_pure(expr) { + return true; + } + let effect_free = |expr: &Expr| is_effect_free(model, expr); + match expr { + Expr::BinOp(bin_op) => { + effect_free(&bin_op.left) + && effect_free(&bin_op.right) + && operation_is_quiet(model, expr) + } + Expr::UnaryOp(unary) => effect_free(&unary.operand) && operation_is_quiet(model, expr), + Expr::Compare(compare) => { + compare.operands.iter().all(effect_free) && operation_is_quiet(model, expr) + } + Expr::BoolOp(bool_op) => { + // `and` and `or` ask each operand's truth, which a builtin scalar answers itself + bool_op + .values + .iter() + .all(|value| effect_free(value) && builtin_scalar_kinds(model, value).is_some()) + } + Expr::If(if_expr) => { + effect_free(&if_expr.test) + && builtin_scalar_kinds(model, &if_expr.test).is_some() + && effect_free(&if_expr.body) + && effect_free(&if_expr.orelse) + } + Expr::Attribute(attribute) => { + effect_free(&attribute.value) && operation_is_quiet(model, expr) + } + Expr::Tuple(tuple) => { + tuple.ctx.is_load() + && !tuple.is_anon_named_tuple + && !tuple.is_anon_named_tuple_value + && tuple.callable_shape.is_none() + && !tuple.is_parameter_shape + && tuple.elts.iter().all(effect_free) + } + _ => false, + } +} + +/// Whether the operation `expr` performs on its operands — not the evaluation of the +/// operands themselves — runs no code of the program's and cannot raise. +/// +/// That is an operator of python's own over builtin scalars that is defined for every +/// pair of values it is given: `+`, `-` and `*` over numbers, `+` joining two strings +/// or two byte strings and `*` repeating one, the bitwise operators over integers, +/// equality, and ordering among numbers that are not complex or among strings. `/`, +/// `//`, `%`, `**` and the shifts are left out because each raises for some operands, +/// and something that may raise before an expression decides whether it runs. Reading +/// an attribute of a module is quiet too. +fn operation_is_quiet(model: &SemanticModel<'_>, expr: &Expr) -> bool { + use BuiltinScalar::{Bool, Bytes, Complex, Float, Int, Str}; + const NUMBERS: &[BuiltinScalar] = &[Bool, Int, Float, Complex]; + const INTEGERS: &[BuiltinScalar] = &[Bool, Int]; + const ORDERED_NUMBERS: &[BuiltinScalar] = &[Bool, Int, Float]; + + let kinds = |expr: &Expr| builtin_scalar_kinds(model, expr); + let all_in = |kinds: &[BuiltinScalar], allowed: &[BuiltinScalar]| { + kinds.iter().all(|kind| allowed.contains(kind)) + }; + + match expr { + Expr::BinOp(bin_op) => { + let (Some(left), Some(right)) = (kinds(&bin_op.left), kinds(&bin_op.right)) else { + return false; + }; + match bin_op.op { + Operator::Add => { + (all_in(&left, NUMBERS) && all_in(&right, NUMBERS)) + || (left == [Str] && right == [Str]) + || (left == [Bytes] && right == [Bytes]) + } + Operator::Sub => all_in(&left, NUMBERS) && all_in(&right, NUMBERS), + Operator::Mult => { + (all_in(&left, NUMBERS) && all_in(&right, NUMBERS)) + || ((left == [Str] || left == [Bytes]) && all_in(&right, INTEGERS)) + || (all_in(&left, INTEGERS) && (right == [Str] || right == [Bytes])) + } + Operator::BitAnd | Operator::BitOr | Operator::BitXor => { + all_in(&left, INTEGERS) && all_in(&right, INTEGERS) + } + _ => false, + } + } + Expr::UnaryOp(unary) => { + let Some(operand) = kinds(&unary.operand) else { + return false; + }; + match unary.op { + UnaryOp::Not => true, + UnaryOp::USub | UnaryOp::UAdd => all_in(&operand, NUMBERS), + UnaryOp::Invert => all_in(&operand, INTEGERS), + UnaryOp::Optional | UnaryOp::Propagate | UnaryOp::Force => false, + } + } + Expr::Compare(compare) => { + let [first, comparators @ ..] = &*compare.operands else { + return false; + }; + let Some(mut left) = kinds(first) else { + return false; + }; + for (op, comparator) in compare.ops.iter().zip(comparators) { + let Some(right) = kinds(comparator) else { + return false; + }; + let quiet = match op { + CmpOp::Eq | CmpOp::NotEq | CmpOp::Is | CmpOp::IsNot => true, + CmpOp::Lt | CmpOp::LtE | CmpOp::Gt | CmpOp::GtE => { + (all_in(&left, ORDERED_NUMBERS) && all_in(&right, ORDERED_NUMBERS)) + || (left == [Str] && right == [Str]) + || (left == [Bytes] && right == [Bytes]) + } + CmpOp::In | CmpOp::NotIn => { + (left == [Str] && right == [Str]) || (left == [Bytes] && right == [Bytes]) + } + }; + if !quiet { + return false; + } + left = right; + } + true + } + Expr::Attribute(attribute) => { + !attribute.optional && is_module_value(model, &attribute.value) + } + _ => false, + } +} + +struct Walk<'a, 'm> { + model: &'m SemanticModel<'m>, target: AnyNodeRef<'a>, effects: bool, } type Found = Option; -impl Walk<'_> { +impl Walk<'_, '_> { fn contains(&self, node: impl Ranged) -> bool { node.range().contains_range(self.target.range()) } @@ -122,7 +274,7 @@ impl Walk<'_> { if self.contains(expr) { return Some(Placement::NotOnce(reason)); } - if !is_pure(expr) { + if !is_effect_free(self.model, expr) { self.effects = true; } None @@ -154,6 +306,14 @@ impl Walk<'_> { self.effects = true; } + /// The operation `expr` performs once its operands are evaluated, which is an + /// effect unless the checker shows it to be one of python's own quiet ones. + fn operation(&mut self, expr: &Expr) { + if !operation_is_quiet(self.model, expr) { + self.effect(); + } + } + fn statement(&mut self, statement: &Stmt) -> Found { match statement { Stmt::Expr(stmt) => self.expr(&stmt.value), @@ -341,7 +501,7 @@ impl Walk<'_> { }); } if !self.contains(expr) { - if !is_pure(expr) { + if !is_effect_free(self.model, expr) { self.effect(); } return None; @@ -372,7 +532,7 @@ impl Walk<'_> { if let Some(found) = self.exprs([&*bin_op.left, &*bin_op.right]) { return Some(found); } - self.effect(); + self.operation(expr); None } }, @@ -382,7 +542,7 @@ impl Walk<'_> { } UnaryOp::Not | UnaryOp::Invert | UnaryOp::UAdd | UnaryOp::USub => { let found = self.expr(&unary.operand); - self.effect(); + self.operation(expr); found } }, @@ -448,7 +608,7 @@ impl Walk<'_> { if let Some(found) = self.expr(first) { return Some(found); } - self.effect(); + self.operation(expr); // a chained comparison stops at the first comparison that fails self.not_once_all(rest, NotOnce::Conditional) } @@ -475,7 +635,7 @@ impl Walk<'_> { return self.opaque(expr, NotOnce::OptionalChain); } let found = self.expr(&attribute.value); - self.effect(); + self.operation(expr); found } Expr::Subscript(subscript) => { diff --git a/crates/ty_ide/src/refactor/extract_variable.rs b/crates/ty_ide/src/refactor/extract_variable.rs index 361597f002..cb0134bae2 100644 --- a/crates/ty_ide/src/refactor/extract_variable.rs +++ b/crates/ty_ide/src/refactor/extract_variable.rs @@ -15,12 +15,15 @@ //! anything mutable, and cannot raise; see [`is_constant`]. use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal, walk_node}; use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt, UnaryOp}; use ruff_python_trivia::PythonWhitespace; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; -use ty_python_semantic::types::ide_support::folds_to_a_literal; +use ty_python_semantic::types::ide_support::{ + BuiltinScalar, builtin_scalar_kinds, folds_to_a_literal, +}; use super::evaluation::{NotOnce, Placement, placement_in_statement}; use super::hazards::{contains_named_expression, context_dependent_construct}; @@ -42,46 +45,126 @@ pub(super) fn plan( let Some(selection) = selected_expression(context, range) else { return Err(Refusal::NotApplicable); }; - let expr = selection.expr; - if target == Target::Constant && !is_constant(context, expr) { + plan_selection(context, &selection, target, Title::Plain) +} + +/// What extracting does at a caret rather than a selection: each expression of the statement +/// around `offset`, innermost first, with the range a client selects to extract it and the +/// rewrite — the expressions `by/expressionTypes` answers at the same caret, less a bare name +/// and those that cannot be bound to a name. When none of them can be extracted, the innermost one's refusal is +/// the answer, so an editor asking about a caret is told why rather than nothing. +pub(super) fn around_caret( + context: &RefactorContext<'_>, + offset: TextSize, + target: Target, +) -> Vec<(TextRange, Result)> { + let covering = covering_node(context.parsed.syntax().into(), TextRange::empty(offset)); + let mut ranges: Vec = Vec::new(); + for node in covering.ancestors() { + if node.is_statement() { + break; + } + if node.as_expr_ref().is_some() && ranges.last() != Some(&node.range()) { + ranges.push(node.range()); + } + } + + let mut available = Vec::new(); + let mut first_refusal = None; + for range in ranges { + let Some(selection) = selected_expression(context, range) else { + continue; + }; + // a name is already a name + if matches!(selection.extracted, Extracted::Expr(Expr::Name(_))) { + continue; + } + match plan_selection(context, &selection, target, Title::Naming) { + Ok(plan) => available.push((range, Ok(plan))), + Err(Refusal::NotApplicable) => {} + Err(refusal) => { + if first_refusal.is_none() { + first_refusal = Some((range, Err(refusal))); + } + } + } + } + if available.is_empty() { + available.extend(first_refusal); + } + available +} + +/// Whether a title names the expression it extracts, which it must when several are offered at +/// once. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Title { + Plain, + Naming, +} + +fn plan_selection( + context: &RefactorContext<'_>, + selection: &Selection<'_>, + target: Target, + title_kind: Title, +) -> Result { + if target == Target::Constant + && !selection + .operands() + .iter() + .all(|operand| is_constant(context, operand)) + { return Err(Refusal::NotApplicable); } let module = context.parsed.suite(); - let ancestors = statement_ancestors(module, expr.range()); + let ancestors = statement_ancestors(module, selection.replaced); let Some(&statement) = ancestors.last() else { return Err(Refusal::NotApplicable); }; // the statement already is the expression on its own - if matches!(statement, Stmt::Expr(stmt) if stmt.value.range() == expr.range()) { + if let Extracted::Expr(expr) = selection.extracted + && matches!(statement, Stmt::Expr(stmt) if stmt.value.range() == expr.range()) + { return Err(Refusal::NotApplicable); } let names = names_in_file(context); let version = context.file.python_version(context.db); - let base = suggest_for_expression(expr, version.minor); - let (name, title) = match target { - Target::Variable => { - let name = fresh_name(&base, &names); - let title = format!("Extract variable `{name}`"); - (name, title) - } - Target::Constant => { - let name = fresh_name(&base.to_uppercase(), &names); - let title = format!("Introduce constant `{name}`"); - (name, title) - } + let base = selection.suggested_name(version.minor); + let name = match target { + Target::Variable => fresh_name(&base, &names), + Target::Constant => fresh_name(&base.to_uppercase(), &names), + }; + let mut title = match target { + Target::Variable => format!("Extract variable `{name}`"), + Target::Constant => format!("Introduce constant `{name}`"), }; + if title_kind == Title::Naming { + title = format!( + "{title} from `{}`", + abbreviated(&context.source()[selection.replaced]) + ); + } let refuse = |reason: String| Refusal::refused(title.clone(), reason); - if let Some(reason) = context_dependent_construct(context, AnyNodeRef::from(expr)) { - return Err(refuse(format!("the expression can't be moved: {reason}"))); - } - if contains_named_expression(expr) { - return Err(refuse("the expression binds a name with `:=`".to_string())); + for operand in selection.operands() { + if let Some(reason) = context_dependent_construct(context, AnyNodeRef::from(operand)) { + return Err(refuse(format!("the expression can't be moved: {reason}"))); + } + if contains_named_expression(operand) { + return Err(refuse("the expression binds a name with `:=`".to_string())); + } } - let placement = placement_in_statement(statement, AnyNodeRef::from(expr)); - let expression_text = expression_source(context, expr); + // a run of operands is evaluated where its first operand was, so it is that operand's place + // in the statement that decides whether the run can be evaluated before the statement + let placement = placement_in_statement( + &context.model, + statement, + AnyNodeRef::from(selection.first()), + ); + let expression_text = expression_source(context, selection); let line_ending = context.line_ending(); let insertion = match target { @@ -159,15 +242,63 @@ pub(super) fn plan( }) } +/// `text` on one line and at most a few dozen characters, for a title to name it by. +fn abbreviated(text: &str) -> String { + const MOST: usize = 40; + let one_line = text.split_whitespace().collect::>().join(" "); + if one_line.chars().count() <= MOST { + return one_line; + } + let mut shortened: String = one_line.chars().take(MOST - 1).collect(); + shortened.push('…'); + shortened +} + +/// What a selection extracts. +enum Extracted<'a> { + /// An expression of the tree. + Expr(&'a Expr), + /// Consecutive operands of a chain of one associative operator, `b + c` in `a + b + c`, + /// which python groups as `(a + b) + c` and so has no node of its own. + Run { operands: Vec<&'a Expr> }, +} + struct Selection<'a> { - expr: &'a Expr, + extracted: Extracted<'a>, /// The range the name replaces: the expression, with the parentheses the /// selection included. replaced: TextRange, } +impl<'a> Selection<'a> { + /// The expressions the selection is made of, evaluated in this order. + fn operands(&self) -> Vec<&'a Expr> { + match &self.extracted { + Extracted::Expr(expr) => vec![*expr], + Extracted::Run { operands, .. } => operands.clone(), + } + } + + /// The first expression of the selection to be evaluated. + fn first(&self) -> &'a Expr { + match &self.extracted { + Extracted::Expr(expr) => expr, + Extracted::Run { operands, .. } => operands[0], + } + } + + /// A name that says what the selection evaluates to. + fn suggested_name(&self, python_version_minor: u8) -> String { + match &self.extracted { + Extracted::Expr(expr) => suggest_for_expression(expr, python_version_minor), + Extracted::Run { .. } => "value".to_string(), + } + } +} + /// The expression the selection covers exactly, ignoring surrounding -/// whitespace and allowing parentheses around it. +/// whitespace and allowing parentheses around it — or, when the selection +/// covers consecutive operands of an associative chain, those operands. fn selected_expression<'a>( context: &'a RefactorContext<'_>, range: TextRange, @@ -176,7 +307,7 @@ fn selected_expression<'a>( context: &'c RefactorContext<'db>, range: TextRange, parents: Vec>, - found: Option>, + found: Option<&'a Expr>, } impl<'a> SourceOrderVisitor<'a> for Finder<'a, '_, '_> { fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { @@ -205,10 +336,7 @@ fn selected_expression<'a>( && (expr.range() == self.range || self.context.parenthesized_range(expr, parent) == self.range) { - self.found = Some(Selection { - expr, - replaced: self.range, - }); + self.found = Some(expr); return; } ruff_python_ast::visitor::source_order::walk_expr(self, expr); @@ -232,10 +360,101 @@ fn selected_expression<'a>( found: None, }; walk_node(&mut finder, context.parsed.syntax().into()); - let selection = finder.found?; - is_extractable(selection.expr).then_some(selection) + if let Some(expr) = finder.found { + return is_extractable(expr).then_some(Selection { + extracted: Extracted::Expr(expr), + replaced: trimmed, + }); + } + selected_run(context, trimmed) +} + +/// Consecutive operands of an associative chain that `range` covers exactly, from the first +/// one's opening parenthesis, if it has one, to the last one's closing parenthesis. +/// +/// `a + b + c` is `(a + b) + c`, so `b + c` is no expression of it. Binding `b + c` to a name +/// and reading `a + value` instead evaluates `a + (b + c)`, which is the same value exactly when +/// the operator is associative over the operands' types — which the checker is asked, not the +/// spelling: `+` over integers or over strings is, `+` over floats rounds differently grouped +/// the other way, and an operator a class defines is anything it likes. +fn selected_run<'a>(context: &'a RefactorContext<'_>, range: TextRange) -> Option> { + let covering = covering_node(context.parsed.syntax().into(), range); + let chain = covering.ancestors().find_map(|node| match node { + AnyNodeRef::ExprBinOp(bin_op) => Some(bin_op), + _ => None, + })?; + // the operands of the chain, each with the range it has in it, parentheses included + let mut operands: Vec<(&Expr, TextRange)> = Vec::new(); + let mut node = chain; + loop { + let parent = AnyNodeRef::from(node); + operands.push(( + &node.right, + context.parenthesized_range(&node.right, parent), + )); + let left_range = context.parenthesized_range(&node.left, parent); + match &*node.left { + Expr::BinOp(left) if left.op == chain.op && left_range == left.range() => node = left, + left => { + operands.push((left, left_range)); + break; + } + } + } + operands.reverse(); + + let first = operands + .iter() + .position(|(_, operand_range)| operand_range.start() == range.start())?; + let last = operands + .iter() + .position(|(_, operand_range)| operand_range.end() == range.end())?; + // a run starting at the first operand is an expression of the tree, found already, and a + // single operand is not a run + if first == 0 || last <= first { + return None; + } + let all: Vec<&Expr> = operands.iter().map(|(operand, _)| *operand).collect(); + if !is_associative(context, chain.op, &all[..=last]) { + return None; + } + Some(Selection { + extracted: Extracted::Run { + operands: all[first..=last].to_vec(), + }, + replaced: range, + }) } +/// Whether `op` is associative over the values `operands` evaluate to, as the checker reads +/// them: integers under `+`, `*` and the bitwise operators, and strings or byte strings — all of +/// one kind — under `+`. +fn is_associative(context: &RefactorContext<'_>, op: ast::Operator, operands: &[&Expr]) -> bool { + let Some(kinds) = operands + .iter() + .map(|operand| builtin_scalar_kinds(&context.model, operand)) + .collect::>>() + else { + return false; + }; + let integers = || { + kinds + .iter() + .flatten() + .all(|kind| matches!(kind, BuiltinScalar::Int | BuiltinScalar::Bool)) + }; + let all_one = |wanted: BuiltinScalar| kinds.iter().all(|kinds| kinds.as_slice() == [wanted]); + match op { + ast::Operator::Add => { + integers() || all_one(BuiltinScalar::Str) || all_one(BuiltinScalar::Bytes) + } + ast::Operator::Mult + | ast::Operator::BitAnd + | ast::Operator::BitOr + | ast::Operator::BitXor => integers(), + _ => false, + } +} /// Whether `expr` is a value that can be bound to a name: it is read rather /// than assigned, and it means something outside its position. fn is_extractable(expr: &Expr) -> bool { @@ -256,9 +475,17 @@ fn is_extractable(expr: &Expr) -> bool { } } -/// The source of `expr` as the value of an assignment, parenthesized where its -/// text would not parse there on its own. -fn expression_source(context: &RefactorContext<'_>, expr: &Expr) -> String { +/// The source of the selection as the value of an assignment, parenthesized where its text +/// would not parse there on its own. +fn expression_source(context: &RefactorContext<'_>, selection: &Selection<'_>) -> String { + let Extracted::Expr(expr) = selection.extracted else { + let text = &context.source()[selection.replaced]; + return if context.is_multiline(selection.replaced) { + format!("({text})") + } else { + text.to_string() + }; + }; let text = context.expression_text(expr); let bracketed = matches!( expr, @@ -511,6 +738,164 @@ mod tests { ), @"not offered"); } + /// `a + b + c` is `(a + b) + c`; extracting `b + c` evaluates `a + (b + c)`, which is + /// the same value because adding integers is associative. + #[test] + fn operands_of_an_associative_chain() { + assert_snapshot!(extract( + " + def f(a: int, b: int, c: int, d: int): + return a + b + c + d + ", + ), @" + Extract variable `value` + --- + + def f(a: int, b: int, c: int, d: int): + value = b + c + return a + value + d + "); + } + + #[test] + fn operands_of_a_string_concatenation() { + assert_snapshot!(extract( + " + def f(name: str): + return 'hello ' + name + '!' + ", + ), @" + Extract variable `value` + --- + + def f(name: str): + value = name + '!' + return 'hello ' + value + "); + } + + /// floats round, so `(a + b) + c` and `a + (b + c)` can differ + #[test] + fn operands_of_a_float_chain_are_not_offered() { + assert_snapshot!(extract( + " + def f(a: float, b: float, c: float): + return a + b + c + ", + ), @"not offered"); + } + + #[test] + fn operands_under_another_operator_are_not_offered() { + assert_snapshot!(extract( + " + def f(a: int, b: int, c: int): + return a * b + c + ", + ), @"not offered"); + } + + #[test] + fn operands_after_a_call_are_refused() { + assert_snapshot!(extract( + " + def f(b: int, c: int): + return g() + b + c + + def g() -> int: ... + ", + ), @"refused: Extract variable `value` (something with effects is evaluated before the expression, which would now run after it)"); + } + + /// reading a module's attribute and multiplying two numbers run nothing of the program's, + /// so evaluating the power first changes nothing + #[test] + fn after_a_module_attribute_and_arithmetic() { + assert_snapshot!(extract( + " + import math + + def f(radius: float, scale: int): + return scale * math.pi * radius ** 2 + ", + ), @" + Extract variable `value` + --- + + import math + + def f(radius: float, scale: int): + value = radius ** 2 + return scale * math.pi * value + "); + } + + /// `self.price` may be a property, which could see the program differently once the + /// expression after it is evaluated first + #[test] + fn after_an_instance_attribute_is_refused() { + assert_snapshot!(extract( + " + class Item: + price: float + quantity: int + + def total(self): + return self.price * self.quantity + ", + ), @"refused: Extract variable `quantity_1` (something with effects is evaluated before the expression, which would now run after it)"); + } + + #[test] + fn around_a_caret_every_enclosing_expression() { + assert_snapshot!( + RefactorTest::python( + " + def f(items: list[int], tax: int): + print(len(items) + tax) + ", + ) + .around_caret(RefactorKind::ExtractVariable), + @" + len(items): Extract variable `value` from `len(items)` + len(items) + tax: Extract variable `value` from `len(items) + tax` + " + ); + } + + #[test] + fn around_a_caret_the_innermost_refusal_when_nothing_applies() { + assert_snapshot!( + RefactorTest::python( + " + def f(queue): + while queue.pending(): + queue.pop() + ", + ) + .around_caret(RefactorKind::ExtractVariable), + @"queue.pending: refused: Extract variable `pending` from `queue.pending` (the expression can't be evaluated before its statement: it is evaluated again on every iteration of the loop)" + ); + } + + #[test] + fn around_a_caret_constants_are_the_literal_expressions() { + assert_snapshot!( + RefactorTest::python( + " + def f(seconds: int): + return seconds / (60 * 60 * 24) + ", + ) + .around_caret(RefactorKind::IntroduceConstant), + @" + 60: Introduce constant `NUMBER` from `60` + 60 * 60: Introduce constant `VALUE` from `60 * 60` + 60 * 60 * 24: Introduce constant `VALUE` from `60 * 60 * 24` + " + ); + } + #[test] fn assignment_target_is_not_offered() { assert_snapshot!(extract( diff --git a/crates/ty_ide/src/refactor/inline_variable.rs b/crates/ty_ide/src/refactor/inline_variable.rs index 3cb8a6aea7..857c4f1d40 100644 --- a/crates/ty_ide/src/refactor/inline_variable.rs +++ b/crates/ty_ide/src/refactor/inline_variable.rs @@ -18,6 +18,7 @@ use ruff_text_size::{Ranged, TextRange}; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_core::scope::{FileScopeId, NodeWithScopeKind, ScopeKind}; use ty_python_core::{SemanticIndex, semantic_index}; +use ty_python_semantic::SemanticModel; use super::evaluation::{NotOnce, Placement, is_pure, placement_in_statement}; use super::flow::{ @@ -156,7 +157,8 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result(expr: &'a Expr, names: &mut Vec<&'a ast::ExprName>) { /// single read, in the statement right after the assignment, with nothing that /// could observe the difference evaluated before the read. fn check_single_immediate_read( + model: &SemanticModel<'_>, reads: &[Read<'_>], scope: FileScopeId, in_suite: &super::text::InSuite<'_>, @@ -588,7 +591,7 @@ fn check_single_immediate_read( .to_string(), ); } - match placement_in_statement(next, AnyNodeRef::from(read.name)) { + match placement_in_statement(model, next, AnyNodeRef::from(read.name)) { Placement::Once { effects_before: false, } => Ok(()), diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index de7af87fef..b085fc977c 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -2820,6 +2820,128 @@ pub fn folds_to_a_literal(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { ) } +/// One of the builtin types whose operators are python's own: none of them runs code of the +/// program's. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuiltinScalar { + Bool, + Int, + Float, + Complex, + Str, + Bytes, +} + +/// The builtin scalar types `expr` evaluates to, one for each kind a union of them holds, or +/// `None` when it can evaluate to anything else. +/// +/// A literal is of its own type exactly, and so is an instance of `final int`, or of a class no +/// class can subclass, as `bool`. A plain `int` may be an instance of a subclass, whose operators +/// are the program's own, so it is not one. An operator python applies to values of exactly +/// builtin scalar types is python's own, and makes a value of exactly the builtin type it is +/// declared to give, as `a + b` of two `final int`s is exactly an `int`, and `not x` is always +/// exactly a `bool`. +pub fn builtin_scalar_kinds( + model: &SemanticModel<'_>, + expr: &ast::Expr, +) -> Option> { + /// the scalar an instance of exactly `instance`'s class is + fn scalar( + db: &dyn Db, + instance: crate::types::NominalInstanceType<'_>, + ) -> Option { + match instance.known_class(db)? { + KnownClass::Bool => Some(BuiltinScalar::Bool), + KnownClass::Int => Some(BuiltinScalar::Int), + KnownClass::Float => Some(BuiltinScalar::Float), + KnownClass::Complex => Some(BuiltinScalar::Complex), + KnownClass::Str => Some(BuiltinScalar::Str), + KnownClass::Bytes => Some(BuiltinScalar::Bytes), + _ => None, + } + } + + /// whether `expr` is an operation python carries out itself on values of exactly builtin + /// scalar types, whose value is then exactly of the builtin type the operation declares + fn is_builtin_operation(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { + let exact = |operand: &ast::Expr| builtin_scalar_kinds(model, operand).is_some(); + match expr { + ast::Expr::BinOp(bin_op) => exact(&bin_op.left) && exact(&bin_op.right), + ast::Expr::UnaryOp(unary) => unary.op == ast::UnaryOp::Not || exact(&unary.operand), + ast::Expr::Compare(compare) => compare.operands.iter().all(exact), + ast::Expr::BoolOp(bool_op) => bool_op.values.iter().all(exact), + ast::Expr::If(if_expr) => exact(&if_expr.body) && exact(&if_expr.orelse), + ast::Expr::Named(named) => exact(&named.value), + _ => false, + } + } + + /// `exact` says whether an instance of a builtin scalar class is known to be of exactly that + /// class, rather than of one that may be a subclass + fn collect( + db: &dyn Db, + env: &ProgramEnvironment<'_>, + ty: Type<'_>, + exact: bool, + kinds: &mut Vec, + ) -> bool { + let kind = match ty.resolve_type_alias(db) { + Type::Union(union) => { + return union + .elements(db) + .iter() + .all(|element| collect(db, env, *element, exact, kinds)); + } + Type::NominalInstance(instance) + if exact || instance.class(db, env).class_literal(db).is_final(db) => + { + scalar(db, instance) + } + Type::Restricted(restricted) + if restricted.modifier(db) == ast::helpers::TypeModifier::Final => + { + match restricted.type_argument(db) { + Type::NominalInstance(instance) => scalar(db, instance), + _ => None, + } + } + Type::NominalInstance(_) => None, + ty => match ty.as_literal_value_kind() { + Some(LiteralValueTypeKind::Bool(_)) => Some(BuiltinScalar::Bool), + Some(LiteralValueTypeKind::Int(_)) => Some(BuiltinScalar::Int), + Some(LiteralValueTypeKind::Float(_)) => Some(BuiltinScalar::Float), + Some(LiteralValueTypeKind::Complex(_)) => Some(BuiltinScalar::Complex), + Some( + LiteralValueTypeKind::String(_) + | LiteralValueTypeKind::LiteralString + | LiteralValueTypeKind::Template(_), + ) => Some(BuiltinScalar::Str), + Some(LiteralValueTypeKind::Bytes(_)) => Some(BuiltinScalar::Bytes), + Some(LiteralValueTypeKind::Enum(_)) | None => None, + }, + }; + let Some(kind) = kind else { + return false; + }; + if !kinds.contains(&kind) { + kinds.push(kind); + } + true + } + + let ty = expr.inferred_type(model)?; + let env = model.program_environment(); + let exact = is_builtin_operation(model, expr); + let mut kinds = Vec::new(); + (collect(model.db(), &env, ty, exact, &mut kinds) && !kinds.is_empty()).then_some(kinds) +} + +/// Whether `expr` evaluates to a module, so that reading an attribute of it looks the name up +/// in the module's namespace and runs nothing. +pub fn is_module_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { + matches!(expr.inferred_type(model), Some(Type::ModuleLiteral(_))) +} + pub fn inferred_raises<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -4725,8 +4847,77 @@ mod tests { use ruff_db::parsed::parsed_module; use ruff_db::testing::assert_function_query_was_not_run_by_name; use ruff_python_ast as ast; + use ruff_text_size::Ranged; use ty_python_core::ProgramFile; + #[test] + fn only_a_value_of_exactly_a_builtin_scalar_type_has_a_scalar_kind() -> anyhow::Result<()> { + let source = r#" +class Number(int): ... + +def f(declared: int, text: str, flag: bool, subclass: Number, either: int | str): + 1 + "a" + flag + declared + text + subclass + either + int("1") + declared + 1 + final_int: final int = declared + final_int + 1 + final_int / 2 + not declared + declared < 1 + final_int < 1 +"#; + let db = TestDbBuilder::new() + .with_file("/src/foo.by", source) + .build()?; + let file = system_path_to_file(&db, "/src/foo.by")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + let parsed = parsed_module(&db, file.python_file(&db)).load(&db); + let model = SemanticModel::new(&db, file); + let function = parsed + .suite() + .iter() + .find_map(ast::Stmt::as_function_def_stmt) + .context("expected a function")?; + let kinds: Vec = function + .body + .iter() + .filter_map(ast::Stmt::as_expr_stmt) + .map(|statement| { + format!( + "{}: {:?}", + &source[statement.value.range()], + super::builtin_scalar_kinds(&model, &statement.value) + ) + }) + .collect(); + assert_eq!( + kinds, + [ + "1: Some([Int])", + "\"a\": Some([Str])", + "flag: Some([Bool])", + "declared: None", + "text: None", + "subclass: None", + "either: None", + "int(\"1\"): Some([Int])", + "declared + 1: None", + "final_int + 1: Some([Int])", + "final_int / 2: Some([Float])", + "not declared: Some([Bool])", + "declared < 1: Some([Bool])", + "final_int < 1: Some([Bool])", + ] + ); + Ok(()) + } + #[test] fn builtin_definition_lookup_does_not_infer_scope() -> anyhow::Result<()> { let mut db = TestDbBuilder::new() diff --git a/crates/ty_server/src/server/api/requests/code_action.rs b/crates/ty_server/src/server/api/requests/code_action.rs index 815914675e..31e91084f4 100644 --- a/crates/ty_server/src/server/api/requests/code_action.rs +++ b/crates/ty_server/src/server/api/requests/code_action.rs @@ -190,8 +190,25 @@ impl BackgroundDocumentRequestHandler for CodeActionRequestHandler { .range .to_text_range(db, file, snapshot.document().uri(), snapshot.encoding()) { - for offer in refactors(db, program_file, range, &wanted) { + // a client asking for refactorings by kind, on purpose, is asking what it can extract + // around a caret too; one asking while the caret moves would be offered every + // expression the caret is in + let around_caret = invoked && only.is_some(); + for offer in refactors(db, program_file, range, &wanted, around_caret) { let kind = CodeActionKind::new(offer.kind.code_action_kind()); + let offered_for = if offer.range == range { + Some(params.range) + } else { + offer + .range + .to_lsp_range(db, program_file.file(db), snapshot.encoding()) + .map(|range| range.local_range()) + }; + let Some(offered_for) = offered_for else { + continue; + }; + // an action about another range than the one asked about says which + let elsewhere = offered_for != params.range; let action = match offer.availability { Ok(edits) => Some(refactor_action( db, @@ -199,7 +216,8 @@ impl BackgroundDocumentRequestHandler for CodeActionRequestHandler { offer.kind, offer.title, edits, - params.range, + offered_for, + elsewhere, )), // a refactoring that cannot be applied is only worth showing to a // user who asked for refactorings, who is owed the reason @@ -207,6 +225,16 @@ impl BackgroundDocumentRequestHandler for CodeActionRequestHandler { title: offer.title, kind: Some(kind), disabled: Some(types::CodeActionDisabled { reason }), + data: elsewhere + .then(|| { + serde_json::to_value(RefactorData::new( + snapshot, + offer.kind, + offered_for, + )) + .ok() + }) + .flatten(), ..lsp_types::CodeAction::default() }), Err(_) => None, @@ -258,7 +286,9 @@ pub(crate) fn to_lsp_edits( /// A client that can resolve an action's edit is sent the action without one, carrying what /// `codeAction/resolve` needs to spell it again; any other client is sent `edits` up front. That /// saves serialising a rewrite nobody chose, not working it out — the rewrite in hand is how this -/// refactoring was known to apply at all. +/// refactoring was known to apply at all. An action about another range than the one asked +/// about — an extraction offered around a caret — carries its [`RefactorData`] either way, for +/// its `range` to tell a client which expression it extracts. fn refactor_action( db: &ProjectDatabase, snapshot: &DocumentSnapshot, @@ -266,38 +296,30 @@ fn refactor_action( title: String, edits: Vec, lsp_range: types::Range, + elsewhere: bool, ) -> lsp_types::CodeAction { - let action_kind = CodeActionKind::new(kind.code_action_kind()); - if snapshot + let resolvable = snapshot .resolved_client_capabilities() - .supports_code_action_edit_resolve() - { - let data = RefactorData { - uri: snapshot.document().uri().clone(), - version: snapshot.document().version(), - refactor: kind.id().to_string(), - range: lsp_range, - }; - return lsp_types::CodeAction { - title, - kind: Some(action_kind), - data: serde_json::to_value(data).ok(), - ..lsp_types::CodeAction::default() - }; - } + .supports_code_action_edit_resolve(); lsp_types::CodeAction { title, - kind: Some(action_kind), - edit: Some(lsp_types::WorkspaceEdit { + kind: Some(CodeActionKind::new(kind.code_action_kind())), + edit: (!resolvable).then(|| lsp_types::WorkspaceEdit { changes: to_lsp_edits(db, snapshot.encoding(), edits), ..lsp_types::WorkspaceEdit::default() }), + data: (resolvable || elsewhere) + .then(|| serde_json::to_value(RefactorData::new(snapshot, kind, lsp_range)).ok()) + .flatten(), ..lsp_types::CodeAction::default() } } /// What a refactoring's code action carries for `codeAction/resolve` to compute /// its edit: the refactoring, and the document and range it was offered for. +/// +/// `range` is the range the refactoring is about: the range the client asked about, or, for +/// an extraction offered around a caret, the expression it extracts. #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct RefactorData { pub(crate) uri: Uri, @@ -308,6 +330,17 @@ pub(crate) struct RefactorData { pub(crate) range: types::Range, } +impl RefactorData { + fn new(snapshot: &DocumentSnapshot, kind: ty_ide::RefactorKind, range: types::Range) -> Self { + Self { + uri: snapshot.document().uri().clone(), + version: snapshot.document().version(), + refactor: kind.id().to_string(), + range, + } + } +} + /// What a quick fix's `data` tells a client about it, beyond what the protocol's own /// fields can say. /// diff --git a/crates/ty_server/tests/e2e/refactors.rs b/crates/ty_server/tests/e2e/refactors.rs index c7d1b36cd7..10da5b2885 100644 --- a/crates/ty_server/tests/e2e/refactors.rs +++ b/crates/ty_server/tests/e2e/refactors.rs @@ -197,3 +197,49 @@ fn resolve_after_an_edit_is_content_modified() -> Result<()> { Ok(()) } + +/// Asked explicitly for extractions with nothing selected, the server offers one for each +/// expression around the caret, each naming the expression and carrying its range; asked as +/// the caret moves, it offers none. +#[test] +fn extraction_at_a_caret_offers_the_expressions_around_it() -> Result<()> { + let (mut server, file) = server(true)?; + let only = Some(vec![CodeActionKind::new("refactor.extract")]); + + let moving = request( + &mut server, + file, + caret(1, 10), + only.clone(), + Some(CodeActionTriggerKind::Automatic), + ); + assert_eq!(moving, Vec::new()); + + let actions = request( + &mut server, + file, + caret(1, 10), + only, + Some(CodeActionTriggerKind::Invoked), + ); + let [action] = actions.as_slice() else { + panic!("expected the call to be offered, got {actions:#?}"); + }; + assert_eq!( + action.title, + "Extract variable `compute_1` from `compute()`" + ); + let data = action.data.as_ref().expect("the action to carry its range"); + assert_eq!( + data["range"], + serde_json::json!({ + "start": { "line": 1, "character": 8 }, + "end": { "line": 1, "character": 17 }, + }) + ); + + let resolved = server.send_request_await::(action.clone()); + insta::assert_json_snapshot!(resolved.edit); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__refactors__extraction_at_a_caret_offers_the_expressions_around_it.snap b/crates/ty_server/tests/e2e/snapshots/e2e__refactors__extraction_at_a_caret_offers_the_expressions_around_it.snap new file mode 100644 index 0000000000..77b0a8da32 --- /dev/null +++ b/crates/ty_server/tests/e2e/snapshots/e2e__refactors__extraction_at_a_caret_offers_the_expressions_around_it.snap @@ -0,0 +1,36 @@ +--- +source: crates/ty_server/tests/e2e/refactors.rs +expression: resolved.edit +--- +{ + "changes": { + "file:///src/main.py": [ + { + "range": { + "start": { + "line": 1, + "character": 0 + }, + "end": { + "line": 1, + "character": 0 + } + }, + "newText": " compute_1 = compute()\n" + }, + { + "range": { + "start": { + "line": 1, + "character": 8 + }, + "end": { + "line": 1, + "character": 17 + } + }, + "newText": "compute_1" + } + ] + } +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 8fc4f14974..d017639105 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -445,6 +445,38 @@ while queue.pending(): # extracting `queue.pending()` is refused: it runs every queue.pop() ``` +what could observe the move is asked of the checker rather than read off the +spelling. an operator of python's own over builtin numbers and strings — `+`, `-` +and `*` over numbers python does not convert to compute them, `+` joining strings, +the bitwise operators over integers, comparisons — runs no code of the program's +and cannot raise, and nor does reading a module's attribute. an operand counts as +a builtin number or string only where the checker knows it is exactly one: a +literal, a `final` type such as `final int`, or a `bool`. a plain `int` or `str` +may be an instance of a subclass, whose operators are the program's own, so a +declared `int` is not enough. so in `scale * math.pi * radius ** 2` the power can +be extracted where `scale` is a `final float` in a `.by` file. in python a `float` +also admits an `int`, and an `int` meeting a `float` or a `complex` is not taken +to be quiet, since one too large for it raises `OverflowError` (`10**400 + 0.5`); +nor is repeating a string, which raises it for a count too large for an index. +reading an attribute of an instance is not taken to be quiet, since it may be a +property; nor are `/`, `//`, `%` and `**`, which raise for some operands + +a selection may be a run of operands that python groups otherwise: `a + b + c` is +`(a + b) + c`, so `b + c` is no expression of it, and extracting it evaluates +`a + (b + c)`. that is offered when the operator is associative over what the +operands are — `+`, `*` and the bitwise operators over integers, `+` over strings +or over byte strings — and not over floats, which round differently grouped the +other way + +with nothing selected, a client that asks for `refactor.extract` kinds explicitly +(`triggerKind: Invoked` and `only` given) is offered an extraction for each +expression around the caret, innermost first, the same expressions +[type info](#type-info) lists less a bare name. each action's title names the +expression — `` Extract variable `value` from `len(items) + 1` `` — and its +`data.range` is the expression's range, for an editor that lets its user pick one +and highlights it; when none can be extracted, the innermost one's refusal is the +answer. a client asking as the caret moves is offered no extraction at a caret + adding a return annotation writes the return type the checker infers, importing what it names. it is refused when that type is partly `Unknown` — `x + y` of two unannotated parameters — since `Unknown` is no type a program can name, and the From f078b893fdc014a2704d416d346fca99e12bc155 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:05:41 +1000 Subject: [PATCH 025/110] inline a variable whose value is python's own arithmetic or concatenation over builtin numbers and strings into every read, as it does a name, rather than refusing it for having two reads, and write a number inlined as an operand without parentheses, keeping them only before an attribute --- crates/ty_ide/src/refactor/evaluation.rs | 27 +++++++++ crates/ty_ide/src/refactor/inline_variable.rs | 60 ++++++++++++++++++- docs/basedpython/features/editor.md | 8 ++- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/crates/ty_ide/src/refactor/evaluation.rs b/crates/ty_ide/src/refactor/evaluation.rs index 156fc19c66..043e1742c8 100644 --- a/crates/ty_ide/src/refactor/evaluation.rs +++ b/crates/ty_ide/src/refactor/evaluation.rs @@ -6,6 +6,7 @@ //! its statement runs, and when nothing evaluated before it could observe the //! move. This module answers both, following python's evaluation order. +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal, walk_node}; use ruff_python_ast::{self as ast, AnyNodeRef, CmpOp, Expr, Operator, Stmt, UnaryOp}; use ruff_text_size::Ranged; use ty_python_semantic::SemanticModel; @@ -154,6 +155,32 @@ pub(crate) fn is_effect_free(model: &SemanticModel<'_>, expr: &Expr) -> bool { } } +/// whether `expr` is effect free and evaluates to the same value wherever it is +/// evaluated, as long as the names it reads are not rebound: [`is_effect_free`] without +/// reading an attribute, since a module's attribute can be assigned between the two +/// places — the value of `config.debug` is not the value of the names it reads +pub(crate) fn is_stable_value(model: &SemanticModel<'_>, expr: &Expr) -> bool { + struct Attributes(bool); + impl<'a> SourceOrderVisitor<'a> for Attributes { + fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { + if node.is_expr_attribute() { + self.0 = true; + } + if self.0 { + TraversalSignal::Skip + } else { + TraversalSignal::Traverse + } + } + } + if !is_effect_free(model, expr) { + return false; + } + let mut attributes = Attributes(false); + walk_node(&mut attributes, AnyNodeRef::from(expr)); + !attributes.0 +} + /// Whether the operation `expr` performs on its operands — not the evaluation of the /// operands themselves — runs no code of the program's and cannot raise. /// diff --git a/crates/ty_ide/src/refactor/inline_variable.rs b/crates/ty_ide/src/refactor/inline_variable.rs index 857c4f1d40..8536179aab 100644 --- a/crates/ty_ide/src/refactor/inline_variable.rs +++ b/crates/ty_ide/src/refactor/inline_variable.rs @@ -20,7 +20,7 @@ use ty_python_core::scope::{FileScopeId, NodeWithScopeKind, ScopeKind}; use ty_python_core::{SemanticIndex, semantic_index}; use ty_python_semantic::SemanticModel; -use super::evaluation::{NotOnce, Placement, is_pure, placement_in_statement}; +use super::evaluation::{NotOnce, Placement, is_stable_value, placement_in_statement}; use super::flow::{ bindings_of_symbol, reaching_definitions, rebound_from_nested_scope, resolving_scope, }; @@ -152,7 +152,7 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result !string.value.is_implicit_concatenated(), Expr::BytesLiteral(bytes) => !bytes.value.is_implicit_concatenated(), Expr::FString(fstring) => !fstring.value.is_implicit_concatenated(), + // `2.real` reads as a float literal and a name, so a number keeps its own parentheses + // only before an attribute + Expr::NumberLiteral(_) => !matches!(parent, AnyNodeRef::ExprAttribute(_)), _ => false, }; if atomic { @@ -774,6 +777,59 @@ mod tests { "); } + /// adding a string to a string runs nothing of the program's and makes the same value + /// wherever it is evaluated, so it is inlined into both reads + #[test] + fn value_over_builtin_scalars_inlines_into_every_read() { + assert_snapshot!(inline( + " + def f(name: str, count: int): + greeting = 'hello ' + name + print(greeting * count) + return greeting + ", + ), @" + Inline variable `greeting` + --- + + def f(name: str, count: int): + print(('hello ' + name) * count) + return 'hello ' + name + "); + } + + /// a module's attribute can be assigned between the assignment and a read + #[test] + fn value_reading_a_module_attribute_is_not_inlined_into_every_read() { + assert_snapshot!(inline( + " + import math + + def f(): + tau = math.pi * 2 + print(tau) + return tau + ", + ), @"refused: Inline variable `tau` (its value would be evaluated once for each read, and evaluating it may have effects)"); + } + + #[test] + fn a_number_needs_no_parentheses_as_an_operand() { + assert_snapshot!(inline( + " + def f(total: int): + scale = 2 + return scale * total, scale.bit_length() + ", + ), @" + Inline variable `scale` + --- + + def f(total: int): + return 2 * total, (2).bit_length() + "); + } + #[test] fn operator_value_is_parenthesized_under_a_tighter_operator() { assert_snapshot!(inline( diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index d017639105..0750af63b3 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -433,8 +433,14 @@ one that asks while the caret moves is only offered what applies | put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | | put on one line | `refactor.rewrite.joinList` | a list spread over several lines | +inlining is offered on the variable's assignment and on any of its reads alike, and +replaces every read either way + moving an expression can change when it runs, so that is what most refusals are -about. a value with an effect is only inlined into a single read in the very next +about. a value made of names and literals, and of python's own operators over +builtin numbers and strings — `'hello ' + name` of a `str` — is the same value +wherever it is evaluated, and is inlined into every read. any other value is only +inlined into a single read in the very next statement, with nothing that could observe the move evaluated before it; an expression is only extracted above its statement when it runs exactly once each time the statement does — not in a loop condition, a conditional branch, a lambda From 222fb31d6f0c4cdba6135afa204c17ce40d39c58 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:14:20 +1000 Subject: [PATCH 026/110] introduce a parameter, refactor.extract.parameter, whose default is a selected expression of literals, added last to the def's parameters or before its ** parameter, and an attribute, refactor.extract.field, assigned on the method's receiver above the statement under a name the class does not already have, refused for a class whose instances take no new attribute --- crates/ty_ide/src/refactor.rs | 16 + .../ty_ide/src/refactor/extract_variable.rs | 365 ++++++++++++++++-- crates/ty_python_semantic/src/types/class.rs | 1 + .../src/types/class/slots.rs | 15 +- .../src/types/ide_support.rs | 151 +++++++- crates/ty_server/tests/e2e/refactors.rs | 2 +- .../e2e__initialize__initialization.snap | 2 + ...ialize__initialization_with_workspace.snap | 2 + docs/basedpython/features/editor.md | 39 +- 9 files changed, 540 insertions(+), 53 deletions(-) diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 4f6ec5a33c..8996ee1b2a 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -45,6 +45,8 @@ pub enum RefactorKind { InlineVariable, ExtractVariable, IntroduceConstant, + IntroduceParameter, + IntroduceField, ExtractFunction, AddReturnAnnotation, ConvertToDataClass, @@ -58,6 +60,8 @@ impl RefactorKind { RefactorKind::InlineVariable, RefactorKind::ExtractVariable, RefactorKind::IntroduceConstant, + RefactorKind::IntroduceParameter, + RefactorKind::IntroduceField, RefactorKind::ExtractFunction, RefactorKind::AddReturnAnnotation, RefactorKind::ConvertToDataClass, @@ -72,6 +76,8 @@ impl RefactorKind { RefactorKind::InlineVariable => "refactor.inline.variable", RefactorKind::ExtractVariable => "refactor.extract.variable", RefactorKind::IntroduceConstant => "refactor.extract.constant", + RefactorKind::IntroduceParameter => "refactor.extract.parameter", + RefactorKind::IntroduceField => "refactor.extract.field", RefactorKind::ExtractFunction => "refactor.extract.function", RefactorKind::AddReturnAnnotation => "refactor.rewrite.returnAnnotation", RefactorKind::ConvertToDataClass => "refactor.rewrite.toDataClass", @@ -88,6 +94,8 @@ impl RefactorKind { RefactorKind::InlineVariable => "inline-variable", RefactorKind::ExtractVariable => "extract-variable", RefactorKind::IntroduceConstant => "introduce-constant", + RefactorKind::IntroduceParameter => "introduce-parameter", + RefactorKind::IntroduceField => "introduce-field", RefactorKind::ExtractFunction => "extract-function", RefactorKind::AddReturnAnnotation => "add-return-annotation", RefactorKind::ConvertToDataClass => "convert-to-data-class", @@ -204,6 +212,12 @@ impl<'db> RefactorContext<'db> { RefactorKind::IntroduceConstant => { extract_variable::plan(self, range, extract_variable::Target::Constant) } + RefactorKind::IntroduceParameter => { + extract_variable::plan(self, range, extract_variable::Target::Parameter) + } + RefactorKind::IntroduceField => { + extract_variable::plan(self, range, extract_variable::Target::Field) + } RefactorKind::ExtractFunction => extract_function::plan(self, range), RefactorKind::AddReturnAnnotation => return_annotation::plan(self, range), RefactorKind::ConvertToDataClass => { @@ -250,6 +264,8 @@ pub fn refactors( let extracted = match kind { RefactorKind::ExtractVariable => Some(extract_variable::Target::Variable), RefactorKind::IntroduceConstant => Some(extract_variable::Target::Constant), + RefactorKind::IntroduceParameter => Some(extract_variable::Target::Parameter), + RefactorKind::IntroduceField => Some(extract_variable::Target::Field), _ => None, }; let plans = match extracted { diff --git a/crates/ty_ide/src/refactor/extract_variable.rs b/crates/ty_ide/src/refactor/extract_variable.rs index cb0134bae2..64d253f86d 100644 --- a/crates/ty_ide/src/refactor/extract_variable.rs +++ b/crates/ty_ide/src/refactor/extract_variable.rs @@ -22,7 +22,8 @@ use ruff_python_trivia::PythonWhitespace; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_semantic::types::ide_support::{ - BuiltinScalar, builtin_scalar_kinds, folds_to_a_literal, + BuiltinScalar, builtin_scalar_kinds, class_instance_has_member, folds_to_a_literal, + new_attribute_obstacle, }; use super::evaluation::{NotOnce, Placement, placement_in_statement}; @@ -35,6 +36,8 @@ use super::{Plan, RefactorContext, Refusal}; pub(super) enum Target { Variable, Constant, + Parameter, + Field, } pub(super) fn plan( @@ -129,16 +132,57 @@ fn plan_selection( return Err(Refusal::NotApplicable); } + // what the expression becomes a parameter or an attribute of: the innermost function + // whose body the statement is in, and for an attribute the class it is a method of + let function = ancestors + .iter() + .enumerate() + .rev() + .skip(1) + .find(|(_, stmt)| matches!(stmt, Stmt::FunctionDef(_) | Stmt::ClassDef(_))) + .and_then(|(at, stmt)| Some((at, stmt.as_function_def_stmt()?))); + let method = function.and_then(|(at, function)| { + let class = ancestors.get(at.checked_sub(1)?)?.as_class_def_stmt()?; + Some((class, function)) + }); + match target { + Target::Parameter if function.is_none() => return Err(Refusal::NotApplicable), + Target::Field if method.is_none() => return Err(Refusal::NotApplicable), + _ => {} + } + let names = names_in_file(context); let version = context.file.python_version(context.db); let base = selection.suggested_name(version.minor); + let receiver = method + .filter(|(_, function)| function.decorator_list.is_empty()) + .and_then(|(_, function)| function.parameters.iter_non_variadic_params().next()) + .map(|parameter| parameter.name().as_str()); let name = match target { - Target::Variable => fresh_name(&base, &names), + Target::Variable | Target::Parameter => fresh_name(&base, &names), Target::Constant => fresh_name(&base.to_uppercase(), &names), + Target::Field => { + // a name the class or a base already has would read or replace that member + let mut taken = names; + loop { + let name = fresh_name(&base, &taken); + match method { + Some((class, _)) if class_instance_has_member(&context.model, class, &name) => { + taken.insert(name); + } + _ => break name, + } + } + } }; let mut title = match target { Target::Variable => format!("Extract variable `{name}`"), Target::Constant => format!("Introduce constant `{name}`"), + Target::Parameter => format!("Introduce parameter `{name}`"), + Target::Field => format!( + "Introduce attribute `{}.{name}`", + receiver.unwrap_or("self") + ), }; if title_kind == Title::Naming { title = format!( @@ -167,27 +211,65 @@ fn plan_selection( let expression_text = expression_source(context, selection); let line_ending = context.line_ending(); + // a binding made directly above the statement, where the expression is then evaluated + let above_statement = |binding: &str| -> Result { + match placement { + Placement::Once { + effects_before: false, + } => {} + Placement::Once { + effects_before: true, + } => { + return Err(refuse( + "something with effects is evaluated before the expression, which would now run after it" + .to_string(), + )); + } + Placement::NotOnce(reason) => { + return Err(refuse(format!( + "the expression can't be evaluated before its statement: {}", + reason.describe() + ))); + } + } + if suite_of(module, statement).is_none() + || ruff_python_trivia::indentation_at_offset(statement.start(), context.source()) + .is_none() + { + return Err(refuse( + "the statement shares its line with the one it belongs to".to_string(), + )); + } + let indentation = context.indentation(statement); + Ok(Edit::insertion( + format!("{indentation}{binding} = {expression_text}{line_ending}"), + context.source().line_start(statement.start()), + )) + }; + // a constant and a parameter's default are evaluated once, before the code they are made + // from runs, so only an expression of literals that cannot raise may become one + let hoisted = |what: &str| -> Result<(), Refusal> { + if !selection + .operands() + .iter() + .all(|operand| is_constant(context, operand)) + { + return Err(refuse(format!( + "only an expression of literals can become {what}, which is evaluated once, before the code it is read in runs" + ))); + } + if let Placement::NotOnce(reason @ (NotOnce::Annotation | NotOnce::Opaque)) = placement { + return Err(refuse(format!( + "the expression can't be moved: {}", + reason.describe() + ))); + } + Ok(()) + }; + + let mut replacement = name.clone(); let insertion = match target { Target::Variable => { - match placement { - Placement::Once { - effects_before: false, - } => {} - Placement::Once { - effects_before: true, - } => { - return Err(refuse( - "something with effects is evaluated before the expression, which would now run after it" - .to_string(), - )); - } - Placement::NotOnce(reason) => { - return Err(refuse(format!( - "the expression can't be evaluated before its statement: {}", - reason.describe() - ))); - } - } if ancestors .iter() .rev() @@ -200,19 +282,7 @@ fn plan_selection( .to_string(), )); } - if suite_of(module, statement).is_none() - || ruff_python_trivia::indentation_at_offset(statement.start(), context.source()) - .is_none() - { - return Err(refuse( - "the statement shares its line with the one it belongs to".to_string(), - )); - } - let indentation = context.indentation(statement); - Edit::insertion( - format!("{indentation}{name} = {expression_text}{line_ending}"), - context.source().line_start(statement.start()), - ) + above_statement(&name)? } Target::Constant => { if let Placement::NotOnce(reason @ (NotOnce::Annotation | NotOnce::Opaque)) = placement @@ -234,14 +304,67 @@ fn plan_selection( at, ) } + Target::Parameter => { + hoisted("a parameter's default")?; + let Some((_, function)) = function else { + return Err(Refusal::NotApplicable); + }; + parameter_insertion(&function.parameters, &name, &expression_text) + } + Target::Field => { + let Some((class, function)) = method else { + return Err(Refusal::NotApplicable); + }; + if !function.decorator_list.is_empty() { + return Err(refuse( + "the method is decorated, so its first parameter need not be the instance" + .to_string(), + )); + } + let Some(receiver) = receiver else { + return Err(refuse( + "the method takes no parameter the instance is passed as".to_string(), + )); + }; + if let Some(reason) = new_attribute_obstacle(&context.model, class) { + return Err(refuse(reason.to_string())); + } + replacement = format!("{receiver}.{name}"); + above_statement(&replacement)? + } }; Ok(Plan { title, - edits: vec![insertion, Edit::range_replacement(name, selection.replaced)], + edits: vec![ + insertion, + Edit::range_replacement(replacement, selection.replaced), + ], }) } +/// the edit adding `name=value` to `parameters`: last, or before a `**` parameter, which has to +/// stay last. after a `*args` or a bare `*` it is keyword-only, which a parameter with a default +/// can be +fn parameter_insertion(parameters: &ast::Parameters, name: &str, value: &str) -> Edit { + let parameter = format!("{name}={value}"); + let kwarg = parameters.kwarg.as_deref().map(Ranged::range); + let previous = parameters + .iter() + .filter(|parameter| Some(parameter.range()) != kwarg) + .map(|parameter| parameter.end()) + .max(); + match (previous, kwarg) { + (Some(previous), _) => Edit::insertion(format!(", {parameter}"), previous), + // `(**kwargs)`: the parameter goes first, after the parenthesis + (None, Some(_)) => Edit::insertion( + format!("{parameter}, "), + parameters.start() + TextSize::from(1), + ), + (None, None) => Edit::insertion(parameter, parameters.end() - TextSize::from(1)), + } +} + /// `text` on one line and at most a few dozen characters, for a title to name it by. fn abbreviated(text: &str) -> String { const MOST: usize = 40; @@ -1102,4 +1225,176 @@ mod tests { @"refused: Extract variable `name_1` (the expression can't be evaluated before its statement: it is a link of an optional chain, which is evaluated as a whole)" ); } + + fn parameter(source: &str) -> String { + RefactorTest::python(source).apply(RefactorKind::IntroduceParameter) + } + + fn field(source: &str) -> String { + RefactorTest::python(source).apply(RefactorKind::IntroduceField) + } + + #[test] + fn parameter_takes_the_expression_as_its_default() { + assert_snapshot!(parameter( + " + def wrap(text: str, *, margin: int = 2): + return text.center(80 - margin) + ", + ), @" + Introduce parameter `number` + --- + + def wrap(text: str, *, margin: int = 2, number=80): + return text.center(number - margin) + "); + } + + #[test] + fn parameter_goes_before_keyword_variadics() { + assert_snapshot!(parameter( + " + def show(*args, **kwargs): + print(*args, sep=', ', **kwargs) + ", + ), @" + Introduce parameter `text` + --- + + def show(*args, text=', ', **kwargs): + print(*args, sep=text, **kwargs) + "); + } + + #[test] + fn parameter_of_a_function_without_parameters() { + assert_snapshot!(parameter( + " + def timeout(): + return 60 * 60 + ", + ), @" + Introduce parameter `value` + --- + + def timeout(value=60 * 60): + return value + "); + } + + /// a default is evaluated once, when the function is defined + #[test] + fn parameter_from_a_value_read_at_each_call_is_refused() { + assert_snapshot!(parameter( + " + def count(items: list[int]): + return len(items) + 1 + ", + ), @"refused: Introduce parameter `value` (only an expression of literals can become a parameter's default, which is evaluated once, before the code it is read in runs)"); + } + + #[test] + fn parameter_outside_a_function_is_not_offered() { + assert_snapshot!(parameter( + " + limit = 10 * 2 + ", + ), @"not offered"); + } + + #[test] + fn field_is_assigned_on_the_instance_above_the_statement() { + assert_snapshot!(field( + " + class Cart: + def __init__(self, prices: list[float]): + self.prices = prices + + def total(self, tax: float): + return sum(self.prices) * tax + ", + ), @" + Introduce attribute `self.value` + --- + + class Cart: + def __init__(self, prices: list[float]): + self.prices = prices + + def total(self, tax: float): + self.value = sum(self.prices) + return self.value * tax + "); + } + + /// `copy` is a method the class inherits from `list`, so the attribute is named apart from it + #[test] + fn field_name_avoids_the_class_members() { + assert_snapshot!(field( + " + class Cart(list[int]): + def backup(this, items: list[int]): + return items.copy() + ", + ), @" + Introduce attribute `this.copy_1` + --- + + class Cart(list[int]): + def backup(this, items: list[int]): + this.copy_1 = items.copy() + return this.copy_1 + "); + } + + #[test] + fn field_of_a_class_with_slots_is_refused() { + assert_snapshot!(field( + " + class Point: + __slots__ = ('x',) + + def scaled(self, factor: int): + return self.x * factor + ", + ), @"refused: Introduce attribute `self.value` (the class has `__slots__`, which leaves no room for another attribute)"); + } + + #[test] + fn field_of_a_frozen_data_class_is_refused() { + assert_snapshot!(field( + " + from dataclasses import dataclass + + @dataclass(frozen=True) + class Point: + x: int + + def scaled(self, factor: int): + return self.x * factor + ", + ), @"refused: Introduce attribute `self.value` (a frozen data class refuses every assignment to an attribute)"); + } + + #[test] + fn field_in_a_static_method_is_refused() { + assert_snapshot!(field( + " + class Math: + @staticmethod + def double(x: int): + return x * 2 + ", + ), @"refused: Introduce attribute `self.value` (the method is decorated, so its first parameter need not be the instance)"); + } + + #[test] + fn field_outside_a_method_is_not_offered() { + assert_snapshot!(field( + " + def double(x: int): + return x * 2 + ", + ), @"not offered"); + } } diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 349ff54b7e..d7f68e853b 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -11,6 +11,7 @@ use self::named_tuple::synthesize_namedtuple_class_member; pub(super) use self::named_tuple::{ DynamicNamedTupleAnchor, DynamicNamedTupleLiteral, NamedTupleField, NamedTupleSpec, }; +pub(crate) use self::slots::InstanceDictionary; pub use self::slots::SlotDescriptorType; pub(crate) use self::static_literal::{ ClassLiteralFlags, ExpandedClassBaseEntry, FrozenDataclassDispatch, StaticClassLiteral, diff --git a/crates/ty_python_semantic/src/types/class/slots.rs b/crates/ty_python_semantic/src/types/class/slots.rs index 2729b8c250..ee9cd82dbe 100644 --- a/crates/ty_python_semantic/src/types/class/slots.rs +++ b/crates/ty_python_semantic/src/types/class/slots.rs @@ -52,7 +52,7 @@ impl get_size2::GetSize for SlotDescriptorType<'_> {} /// This describes `instance.__dict__`, not `Class.__dict__`: the class's own namespace remains /// available regardless of its instance layout. #[derive(Clone, Copy, Debug, PartialEq, Eq, get_size2::GetSize, salsa::SalsaValue)] -enum InstanceDictionary { +pub(crate) enum InstanceDictionary { /// Instances definitely have dictionary-backed attribute storage. Present, /// Instances definitely lack dictionary-backed attribute storage. @@ -524,14 +524,19 @@ impl<'db> StaticClassLiteral<'db> { } } - /// Returns whether instance dictionary storage exists or cannot be ruled out. - fn has_instance_dictionary(self, db: &'db dyn Db) -> bool { + /// basedpython: whether instances of this class have dictionary-backed attribute storage + pub(crate) fn instance_dictionary(self, db: &'db dyn Db) -> InstanceDictionary { if !self.has_explicit_slots(db) && !self.has_generated_slots(db) && self.known(db).is_none() { - return true; + return InstanceDictionary::Present; } - self.instance_layout(db).dictionary != InstanceDictionary::Absent + self.instance_layout(db).dictionary + } + + /// returns whether instance dictionary storage exists or cannot be ruled out + fn has_instance_dictionary(self, db: &'db dyn Db) -> bool { + self.instance_dictionary(db) != InstanceDictionary::Absent } /// Returns whether this class or any base defines a slot with the given name. diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index b085fc977c..0ba1526374 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -17,7 +17,8 @@ use crate::types::abstract_methods::type_as_abstract_method; use crate::types::call::bind::CheckTypesMode; use crate::types::call::{CallArguments, CallError, MatchedArgument}; use crate::types::class::{ - DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor, expanded_class_base_entries, + DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor, InstanceDictionary, + expanded_class_base_entries, }; use crate::types::constraints::ConstraintSetBuilder; use crate::types::context_sensitive::for_each_candidate; @@ -4457,6 +4458,74 @@ pub fn data_class_conversion_obstacle( None } +/// why an instance of `class` cannot be given a new attribute by assigning it in a method, or +/// `None` when it can +/// +/// an enum's members, a named tuple's fields and a `TypedDict`'s keys are fixed, a protocol has +/// no instances of its own, a frozen data class refuses every assignment, and an instance has a +/// `__dict__` to hold an attribute no slot lists only when some class in its MRO gives it one: a +/// class without `__slots__`, or one whose `__slots__` lists `__dict__` +pub fn new_attribute_obstacle( + model: &SemanticModel<'_>, + class: &ast::StmtClassDef, +) -> Option<&'static str> { + let db = model.db(); + let index = semantic_index(db, model.program_file()); + let Some(literal) = index + .try_definition(class) + .and_then(|definition| crate::types::infer::original_class_type(db, definition)) + else { + return Some("the class is not one ty can read"); + }; + let Some(static_class) = literal.as_static() else { + return Some("the class is not one ty can read statically"); + }; + if crate::types::enums::is_enum_class(db, Type::ClassLiteral(literal)) { + return Some("an enum's members are fixed"); + } + if static_class.is_protocol(db) { + return Some("a protocol has no instances of its own"); + } + if static_class.is_typed_dict(db) { + return Some("a `TypedDict` is a dict, whose keys are not attributes"); + } + if static_class.has_named_tuple_class_in_mro(db) { + return Some("a named tuple's fields are fixed"); + } + if static_class.is_frozen_dataclass(db) == Some(true) { + return Some("a frozen data class refuses every assignment to an attribute"); + } + match static_class.instance_dictionary(db) { + InstanceDictionary::Present => None, + InstanceDictionary::Absent => { + Some("the class has `__slots__`, which leaves no room for another attribute") + } + InstanceDictionary::Unknown => { + Some("ty cannot tell whether an instance has a `__dict__` to hold another attribute") + } + } +} + +/// whether an instance of `class` has a member called `name`, its own or inherited, declared +/// in the class body or assigned in a method +pub fn class_instance_has_member( + model: &SemanticModel<'_>, + class: &ast::StmtClassDef, + name: &str, +) -> bool { + let db = model.db(); + let env = model.program_environment(); + let index = semantic_index(db, model.program_file()); + let Some(literal) = index + .try_definition(class) + .and_then(|definition| crate::types::infer::original_class_type(db, definition)) + else { + return true; + }; + let instance = Type::instance(db, &env, literal.default_specialization(db)); + !instance.member(db, &env, name).place.is_undefined() +} + /// The type worth showing for a parameter the source leaves unannotated. /// /// A receiver's `Self` type says nothing the enclosing class does not already, @@ -4850,6 +4919,86 @@ mod tests { use ruff_text_size::Ranged; use ty_python_core::ProgramFile; + /// what [`super::new_attribute_obstacle`] says of each class in `source`, by name + fn new_attribute_obstacles(source: &str) -> anyhow::Result> { + let db = TestDbBuilder::new() + .with_file("/src/foo.py", source) + .build()?; + let file = system_path_to_file(&db, "/src/foo.py")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + let parsed = parsed_module(&db, file.python_file(&db)).load(&db); + let model = SemanticModel::new(&db, file); + Ok(parsed + .suite() + .iter() + .filter_map(ast::Stmt::as_class_def_stmt) + .map(|class| { + format!( + "{}: {}", + class.name, + super::new_attribute_obstacle(&model, class).unwrap_or("none") + ) + }) + .collect()) + } + + #[test] + fn a_class_without_slots_throughout_its_bases_takes_a_new_attribute() -> anyhow::Result<()> { + let obstacles = new_attribute_obstacles( + r#" +from typing import Any + +class Slotted: + __slots__ = ("a",) + +class Plain: + pass + +class Unslotted(Slotted): + pass + +class SlottedOverPlain(Plain): + __slots__ = ("b",) + +class SlottedOverSlotted(Slotted): + __slots__ = ("b",) + +class WithDictionary(Slotted): + __slots__ = ("b", "__dict__") + +class SlottedOverInt(int): + __slots__ = () + +class SlottedOverException(Exception): + __slots__ = () + +def slots() -> tuple[str, ...]: ... + +class DynamicSlots: + __slots__ = slots() + +class SlottedOverAny(Any): + __slots__ = () +"#, + )?; + assert_eq!( + obstacles, + [ + "Slotted: the class has `__slots__`, which leaves no room for another attribute", + "Plain: none", + "Unslotted: none", + "SlottedOverPlain: none", + "SlottedOverSlotted: the class has `__slots__`, which leaves no room for another attribute", + "WithDictionary: none", + "SlottedOverInt: the class has `__slots__`, which leaves no room for another attribute", + "SlottedOverException: none", + "DynamicSlots: ty cannot tell whether an instance has a `__dict__` to hold another attribute", + "SlottedOverAny: ty cannot tell whether an instance has a `__dict__` to hold another attribute", + ] + ); + Ok(()) + } + #[test] fn only_a_value_of_exactly_a_builtin_scalar_type_has_a_scalar_kind() -> anyhow::Result<()> { let source = r#" diff --git a/crates/ty_server/tests/e2e/refactors.rs b/crates/ty_server/tests/e2e/refactors.rs index 10da5b2885..8fc6d7e605 100644 --- a/crates/ty_server/tests/e2e/refactors.rs +++ b/crates/ty_server/tests/e2e/refactors.rs @@ -204,7 +204,7 @@ fn resolve_after_an_edit_is_content_modified() -> Result<()> { #[test] fn extraction_at_a_caret_offers_the_expressions_around_it() -> Result<()> { let (mut server, file) = server(true)?; - let only = Some(vec![CodeActionKind::new("refactor.extract")]); + let only = Some(vec![CodeActionKind::new("refactor.extract.variable")]); let moving = request( &mut server, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 6e273961df..d900370732 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -58,6 +58,8 @@ expression: initialization_result "refactor.inline.variable", "refactor.extract.variable", "refactor.extract.constant", + "refactor.extract.parameter", + "refactor.extract.field", "refactor.extract.function", "refactor.rewrite.returnAnnotation", "refactor.rewrite.toDataClass", diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 6e273961df..d900370732 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -58,6 +58,8 @@ expression: initialization_result "refactor.inline.variable", "refactor.extract.variable", "refactor.extract.constant", + "refactor.extract.parameter", + "refactor.extract.field", "refactor.extract.function", "refactor.rewrite.returnAnnotation", "refactor.rewrite.toDataClass", diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 0750af63b3..116daca5b3 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -421,17 +421,19 @@ a refactoring either keeps the program meaning what it meant or is refused, and refusal says why. an editor that asks for refactorings explicitly shows the reason; one that asks while the caret moves is only offered what applies -| refactoring | kind | offered on | -| ----------------------- | ----------------------------------- | --------------------------------- | -| inline variable | `refactor.inline.variable` | a variable assigned once | -| extract variable | `refactor.extract.variable` | a selected expression | -| introduce constant | `refactor.extract.constant` | a selected expression of literals | -| extract function | `refactor.extract.function` | selected statements | -| add return annotation | `refactor.rewrite.returnAnnotation` | a `def` header with no `->` | -| convert to data class | `refactor.rewrite.toDataClass` | a class header | -| convert from data class | `refactor.rewrite.fromDataClass` | a `data class` header | -| put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | -| put on one line | `refactor.rewrite.joinList` | a list spread over several lines | +| refactoring | kind | offered on | +| ----------------------- | ----------------------------------- | ------------------------------------ | +| inline variable | `refactor.inline.variable` | a variable assigned once | +| extract variable | `refactor.extract.variable` | a selected expression | +| introduce constant | `refactor.extract.constant` | a selected expression of literals | +| introduce parameter | `refactor.extract.parameter` | an expression of literals in a `def` | +| introduce attribute | `refactor.extract.field` | a selected expression in a method | +| extract function | `refactor.extract.function` | selected statements | +| add return annotation | `refactor.rewrite.returnAnnotation` | a `def` header with no `->` | +| convert to data class | `refactor.rewrite.toDataClass` | a class header | +| convert from data class | `refactor.rewrite.fromDataClass` | a `data class` header | +| put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | +| put on one line | `refactor.rewrite.joinList` | a list spread over several lines | inlining is offered on the variable's assignment and on any of its reads alike, and replaces every read either way @@ -489,6 +491,21 @@ unannotated parameters — since `Unknown` is no type a program can name, and th rest of the union would claim more than the body says. an inlay hint's double-click edit follows the same rule +introducing a parameter adds `name=expression` to the parameters of the function +the expression is in — last, or before a `**` parameter — and reads the parameter +where the expression was, so every call goes on passing what it passed. a default +is evaluated once, when the `def` runs, so this is offered only for what a constant +can be made of + +introducing an attribute assigns `self.name = expression` above the statement in the +method and reads `self.name` where the expression was, by whatever name the method's +first parameter has. the name is one the class does not already have, its own or +inherited, and it is refused where an instance takes no new attribute: a class with +`__slots__` in its MRO, a frozen data class, an enum, a named tuple, a protocol or a +`TypedDict`, and in a class method or a static method — `__new__`, +`__init_subclass__` and `__class_getitem__` included — or a decorated method, +whose first parameter need not be the instance + extracting statements from a method makes a method of the same class, called through the receiver. what the statements read from the function becomes a parameter, and what they assign that is read afterwards is returned From 584f5da214cd98bb0c7baae1bf7e648e856aa24c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:19:18 +1000 Subject: [PATCH 027/110] read completions.importStyle, from or module, for how an auto-import brings in a name from a module the file imports nothing from yet, writing import inspect and completing inspect.walktree for module rather than from inspect import walktree --- crates/ty_ide/src/completion.rs | 23 +++++++- crates/ty_ide/src/lib.rs | 4 +- crates/ty_server/src/lib.rs | 4 +- crates/ty_server/src/session.rs | 4 +- crates/ty_server/src/session/options.rs | 30 ++++++++++- crates/ty_server/tests/e2e/completions.rs | 53 ++++++++++++++++++- .../e2e__commands__debug_command.snap | 1 + docs/basedpython/features/editor.md | 10 ++++ 8 files changed, 121 insertions(+), 8 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 30c61b846c..5eb3aaf30c 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -268,8 +268,13 @@ pub fn completion<'db>( &importer, &members, context.cursor.is_in_type_expression(&model), - |module_name: &ModuleName, symbol: &str| { - ImportRequest::import_from(module_name.as_str(), symbol) + |module_name: &ModuleName, symbol: &str| match settings.import_style { + AutoImportStyle::FromImport => { + ImportRequest::import_from(module_name.as_str(), symbol) + } + AutoImportStyle::ModuleImport => { + ImportRequest::import(module_name.as_str(), symbol) + } }, &mut completions, ); @@ -976,6 +981,19 @@ pub struct CompletionSettings { pub auto_import: bool, /// Whether callable completions should insert parentheses. pub complete_function_parentheses: bool, + /// how an auto-import brings a member of a module the file does not import yet into scope + pub import_style: AutoImportStyle, +} + +/// how an auto-import brings a module's member into a file that imports nothing from that +/// module yet. a module the file already imports is imported from the way the file does it +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AutoImportStyle { + /// `from pathlib import Path`, and `Path` where the completion was + #[default] + FromImport, + /// `import pathlib`, and `pathlib.Path` where the completion was + ModuleImport, } // N.B. It's important for the defaults here to match the defaults @@ -993,6 +1011,7 @@ impl Default for CompletionSettings { // below writes `name($0)` and asks for signature help, so accepting // one leaves the caret where the first argument goes complete_function_parentheses: true, + import_style: AutoImportStyle::FromImport, } } } diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index cd6ad21d1e..46b982ca54 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -65,8 +65,8 @@ pub use call_hierarchy::{CallHierarchyItem, prepare_call_hierarchy}; pub use code_action::{FileEdit, QuickFix, code_actions}; pub use complete_statement::{CompletedStatement, StatementEdit, complete_statement}; pub use completion::{ - Completion, CompletionCapabilities, CompletionCommand, CompletionInsertTextFormat, - CompletionKind, CompletionSettings, completion, + AutoImportStyle, Completion, CompletionCapabilities, CompletionCommand, + CompletionInsertTextFormat, CompletionKind, CompletionSettings, completion, }; pub use data_flow::{Finding, FindingKind, data_flow_at}; pub use declaration::{Declaration, DeclarationAt, DeclaredKind, declaration_at}; diff --git a/crates/ty_server/src/lib.rs b/crates/ty_server/src/lib.rs index 8f8f856d70..49a78042d1 100644 --- a/crates/ty_server/src/lib.rs +++ b/crates/ty_server/src/lib.rs @@ -7,7 +7,9 @@ use ruff_db::system::{OsSystem, SystemPathBuf}; use crate::db::Db; pub use crate::logging::{LogLevel, init_logging}; pub use crate::server::Server; -pub use crate::session::{ClientOptions, DiagnosticMode, GlobalOptions, WorkspaceOptions}; +pub use crate::session::{ + ClientOptions, DiagnosticMode, GlobalOptions, ImportStyleOption, WorkspaceOptions, +}; pub use document::{NotebookDocument, PositionEncoding, TextDocument}; pub(crate) use session::Session; diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index 234ced4400..bf6b3d4daa 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -32,7 +32,9 @@ use index::DocumentError; use ty_python_core::program::UseDefaultStrategy; pub(crate) use self::options::InitializationOptions; -pub use self::options::{ClientOptions, DiagnosticMode, GlobalOptions, WorkspaceOptions}; +pub use self::options::{ + ClientOptions, DiagnosticMode, GlobalOptions, ImportStyleOption, WorkspaceOptions, +}; pub(crate) use self::settings::{GlobalSettings, WorkspaceSettings}; use crate::capabilities::{ResolvedClientCapabilities, server_diagnostic_options}; use crate::db::Db as _; diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index 6a4e4a1f73..19f05f84ae 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use strum::IntoEnumIterator; use ty_combine::Combine; -use ty_ide::{CompletionSettings, InlayHintSettings}; +use ty_ide::{AutoImportStyle, CompletionSettings, InlayHintSettings}; use ty_project::metadata::Options as TyOptions; use ty_project::metadata::options::EnvironmentOptions; use ty_project::metadata::python_version::SupportedPythonVersion; @@ -170,6 +170,15 @@ impl ClientOptions { self } + #[must_use] + pub fn with_import_style(mut self, style: ImportStyleOption) -> Self { + self.workspace + .completions + .get_or_insert_default() + .import_style = Some(style); + self + } + #[must_use] pub fn with_complete_function_parentheses(mut self, enabled: bool) -> Self { self.workspace @@ -471,6 +480,21 @@ pub struct CompletionOptions { auto_import: Option, /// Whether callable completions should insert parentheses. complete_function_parentheses: Option, + /// how an auto-import brings in a member of a module the file imports nothing from yet + import_style: Option, +} + +/// `completions.importStyle`: `"from"` writes `from pathlib import Path`, `"module"` writes +/// `import pathlib` and reads `pathlib.Path` +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum ImportStyleOption { + From, + Module, +} + +impl Combine for ImportStyleOption { + fn combine_with(&mut self, _other: Self) {} } impl CompletionOptions { @@ -485,6 +509,10 @@ impl CompletionOptions { auto_import: self.auto_import.unwrap_or(true), // basedpython defaults this on — see `CompletionSettings::default` complete_function_parentheses: self.complete_function_parentheses.unwrap_or(true), + import_style: match self.import_style { + Some(ImportStyleOption::Module) => AutoImportStyle::ModuleImport, + Some(ImportStyleOption::From) | None => AutoImportStyle::FromImport, + }, } } } diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index b9a0462eae..a82eb77ec1 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -1,7 +1,7 @@ use anyhow::Result; use lsp_types::{Documentation, MarkupKind, Position}; use ruff_db::system::SystemPath; -use ty_server::ClientOptions; +use ty_server::{ClientOptions, ImportStyleOption}; use crate::TestServerBuilder; @@ -1138,3 +1138,54 @@ foo_ Ok(markup.kind) } + +/// `completions.importStyle: "module"` imports the module an unimported name is in and writes +/// the name through it, where the default writes `from module import name` +#[test] +fn import_style_module_imports_the_module() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +import os + +walktr +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options( + &ClientOptions::default() + .with_complete_function_parentheses(false) + .with_import_style(ImportStyleOption::Module), + ) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let completions = server.completion_request(&server.file_uri(foo), Position::new(2, 6)); + let offered: Vec = completions + .iter() + .map(|item| { + let imports: Vec<&str> = item + .additional_text_edits + .iter() + .flatten() + .map(|edit| edit.new_text.as_str()) + .collect(); + format!( + "{}: {} with {imports:?}", + item.label, + item.insert_text.as_deref().unwrap_or(&item.label) + ) + }) + .collect(); + insta::assert_debug_snapshot!(offered, @r#" + [ + "inspect.walktree (import inspect): inspect.walktree with [\"import inspect/n\"]", + ] + "#); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index 654a32a60b..b50f2df72f 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -48,6 +48,7 @@ Settings: WorkspaceSettings { completions: CompletionSettings { auto_import: true, complete_function_parentheses: true, + import_style: FromImport, }, configuration_file: None, override_options: None, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 116daca5b3..cb79ad79f2 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -171,6 +171,16 @@ nothing, because every symbol spelled `mod` anywhere would qualify like the aliases above, these are auto-imports, and turning off `ty.completions.autoImport` turns them off too +how an auto-import brings in a name from a module the file imports nothing from yet +is `ty.completions.importStyle` (`completions.importStyle` in the initialization +options): `"from"`, the default, writes `from inspect import walktree` and +completes `walktree`; `"module"` writes `import inspect` and completes +`inspect.walktree`. a module the file already imports is imported from the way the +file does it, whichever is set — a name from a module imported with `import` is +written through it, and one from a module with a `from ... import` is added to that +statement. how imports are sorted, grouped and split is not an editor setting: it +is the project's isort configuration, which `buff` reads and fixes + ### names inside a string a name written inside a plain string's braces completes as a name, and taking From 85b2c4a44a1bc79cfc675e118e7871badf51ef88 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:26:33 +1000 Subject: [PATCH 028/110] answer by/docstringStub, the docstring an editor writes into an empty one at the top of a function's body: a summary line for the caret, every parameter but the receiver, and what the function returns or yields, in the format asked for or the one the file's other docstrings are written in --- crates/ty_ide/src/docstring_stub.rs | 580 ++++++++++++++++++ crates/ty_ide/src/lib.rs | 4 + crates/ty_ide/src/method_kind.rs | 78 +++ .../src/types/ide_support.rs | 29 + crates/ty_server/src/server/api.rs | 7 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/docstring_stub.rs | 119 ++++ crates/ty_server/tests/e2e/docstring_stub.rs | 85 +++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 37 ++ 10 files changed, 942 insertions(+) create mode 100644 crates/ty_ide/src/docstring_stub.rs create mode 100644 crates/ty_ide/src/method_kind.rs create mode 100644 crates/ty_server/src/server/api/requests/docstring_stub.rs create mode 100644 crates/ty_server/tests/e2e/docstring_stub.rs diff --git a/crates/ty_ide/src/docstring_stub.rs b/crates/ty_ide/src/docstring_stub.rs new file mode 100644 index 0000000000..131f5470bd --- /dev/null +++ b/crates/ty_ide/src/docstring_stub.rs @@ -0,0 +1,580 @@ +//! the docstring an editor writes into an empty one: a line for the summary, then a section +//! for the parameters and one for what the function returns, in the docstring format the +//! project writes +//! +//! ```python +//! def area(width: float, height: float) -> float: +//! """ +//! | +//! +//! :param width: +//! :param height: +//! :return: +//! """ +//! ``` + +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor, TraversalSignal}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt, StringFlags}; +use ruff_python_trivia::indentation_at_offset; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use ty_python_core::ProgramFile; +use ty_python_semantic::SemanticModel; + +use crate::Db; +use crate::method_kind::{MethodKind, resolved_method_kind}; + +/// a docstring convention a stub can be written in +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocstringFormat { + /// reStructuredText fields, `:param width:` and `:return:`, as Sphinx reads them + Rest, + /// Google style, an `Args:` and a `Returns:` section + Google, + /// NumPy style, `Parameters` and `Returns` sections underlined with dashes + Numpy, + /// the summary line alone + Plain, +} + +/// the rewrite of an empty docstring into a stub +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DocstringStub { + /// the docstring literal, quotes included, which `text` replaces + pub range: TextRange, + pub text: String, + /// the line of the stub the summary goes on, counted from the line the literal starts on, + /// and the column it starts at — the indentation, which is ASCII whitespace and so the same + /// in every position encoding + pub caret_line: u32, + pub caret_column: u32, +} + +/// the stub for the empty docstring of the `def` whose docstring `offset` is in, in `format`, +/// or in the format the file's other docstrings are written in when `format` is `None`, and +/// reStructuredText when none of them says +/// +/// `None` unless `offset` is inside a triple-quoted string literal holding nothing but +/// whitespace that is the first statement of a function's body and starts its own line +pub fn docstring_stub( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + format: Option, +) -> Option { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let source = source.as_str(); + + let mut finder = DocstringFinder { + offset, + class_body: Vec::new(), + found: None, + formats: Vec::new(), + }; + finder.visit_body(parsed.suite()); + let found = finder.found?; + let literal = found.literal; + + let [part] = literal.value.as_slice() else { + return None; + }; + if !part.flags.is_triple_quoted() || !literal.value.to_str().trim().is_empty() { + return None; + } + let indentation = indentation_at_offset(literal.start(), source)?; + + let method = if found.in_class { + Some(resolved_method_kind( + &SemanticModel::new(db, file), + found.function, + )?) + } else { + None + }; + let skips_receiver = match method { + Some(MethodKind::Instance | MethodKind::Class) => true, + // `__new__` is passed its class explicitly, as its first argument + Some(MethodKind::Static) => found.function.name.as_str() == "__new__", + None => false, + }; + let function = found.function; + let parameters = &function.parameters; + // each parameter as the stub shows it, `*args` with its stars, and by its name alone + let named = |parameter: &ast::Parameter, stars: &str| { + ( + format!("{stars}{}", parameter.name), + parameter.name.to_string(), + ) + }; + let mut written: Vec<(String, String)> = parameters + .posonlyargs + .iter() + .chain(¶meters.args) + .skip(usize::from(skips_receiver)) + .map(|parameter| named(¶meter.parameter, "")) + .collect(); + written.extend( + parameters + .vararg + .as_deref() + .map(|vararg| named(vararg, "*")), + ); + written.extend( + parameters + .kwonlyargs + .iter() + .map(|parameter| named(¶meter.parameter, "")), + ); + written.extend(parameters.kwarg.as_deref().map(|kwarg| named(kwarg, "**"))); + written.retain(|(_, name)| !name.is_empty()); + + let result = function_result(function); + let format = format + .or_else(|| finder.formats.first().copied()) + .unwrap_or(DocstringFormat::Rest); + + let quotes = &source[TextRange::at( + literal.start() + part.flags.prefix().text_len(), + part.flags.quote_len(), + )]; + let prefix = part.flags.prefix().as_str(); + let line_ending = + ruff_source_file::find_newline(source).map_or("\n", |(_, ending)| ending.as_str()); + + let mut lines: Vec = vec![String::new()]; + let sections = format != DocstringFormat::Plain && (!written.is_empty() || result.is_some()); + if sections { + lines.push(String::new()); + match format { + DocstringFormat::Rest => { + for (_, name) in &written { + lines.push(format!(":param {name}:")); + } + match result { + Some(FunctionResult::Returns) => lines.push(":return:".to_string()), + Some(FunctionResult::Yields) => lines.push(":yield:".to_string()), + None => {} + } + } + DocstringFormat::Google => { + if !written.is_empty() { + lines.push("Args:".to_string()); + for (shown, _) in &written { + lines.push(format!(" {shown}:")); + } + } + if let Some(result) = result { + if !written.is_empty() { + lines.push(String::new()); + } + lines.push(match result { + FunctionResult::Returns => "Returns:".to_string(), + FunctionResult::Yields => "Yields:".to_string(), + }); + } + } + DocstringFormat::Numpy => { + if !written.is_empty() { + lines.push("Parameters".to_string()); + lines.push("----------".to_string()); + for (shown, _) in &written { + lines.push(shown.clone()); + } + } + if let Some(result) = result { + if !written.is_empty() { + lines.push(String::new()); + } + let heading = match result { + FunctionResult::Returns => "Returns", + FunctionResult::Yields => "Yields", + }; + lines.push(heading.to_string()); + lines.push("-".repeat(heading.len())); + } + } + DocstringFormat::Plain => {} + } + } + + let mut text = format!("{prefix}{quotes}{line_ending}"); + for (index, line) in lines.iter().enumerate() { + // the summary line is indented although empty, since the caret is put at its end + if index == 0 || !line.is_empty() { + text.push_str(indentation); + text.push_str(line); + } + text.push_str(line_ending); + } + text.push_str(indentation); + text.push_str(quotes); + + Some(DocstringStub { + range: literal.range(), + text, + caret_line: 1, + caret_column: u32::try_from(indentation.len()).ok()?, + }) +} + +/// what a function hands its caller that a docstring documents +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FunctionResult { + Returns, + Yields, +} + +/// whether `function` returns a value, yields them, or neither: a generator yields; otherwise a +/// return annotation other than `None`, or without one a `return` of something other than +/// `None`, is a value returned +fn function_result(function: &ast::StmtFunctionDef) -> Option { + #[derive(Default)] + struct Body { + yields: bool, + returns_value: bool, + } + impl<'a> SourceOrderVisitor<'a> for Body { + fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { + match node { + // what a nested function or class does is its own + AnyNodeRef::StmtFunctionDef(_) + | AnyNodeRef::StmtClassDef(_) + | AnyNodeRef::ExprLambda(_) => TraversalSignal::Skip, + AnyNodeRef::ExprYield(_) | AnyNodeRef::ExprYieldFrom(_) => { + self.yields = true; + TraversalSignal::Traverse + } + AnyNodeRef::StmtReturn(ret) => { + if ret + .value + .as_deref() + .is_some_and(|value| !value.is_none_literal_expr()) + { + self.returns_value = true; + } + TraversalSignal::Traverse + } + _ => TraversalSignal::Traverse, + } + } + } + let mut body = Body::default(); + body.visit_body(&function.body); + if body.yields { + return Some(FunctionResult::Yields); + } + match function.returns.as_deref() { + Some(Expr::NoneLiteral(_)) => None, + Some(Expr::Name(name)) if matches!(name.id.as_str(), "NoReturn" | "Never") => None, + Some(_) => Some(FunctionResult::Returns), + None => body.returns_value.then_some(FunctionResult::Returns), + } +} + +struct Found<'a> { + function: &'a ast::StmtFunctionDef, + literal: &'a ast::ExprStringLiteral, + /// whether a class body defines the function, as a method + in_class: bool, +} + +/// finds the docstring around the offset, and the formats the file's docstrings are written +/// in, in source order +struct DocstringFinder<'a> { + offset: TextSize, + /// for each suite being walked, whether it is a class body + class_body: Vec, + found: Option>, + formats: Vec, +} + +impl<'a> DocstringFinder<'a> { + fn docstring(body: &'a [Stmt]) -> Option<&'a ast::ExprStringLiteral> { + match body.first()? { + Stmt::Expr(stmt) => stmt.value.as_string_literal_expr(), + _ => None, + } + } + + fn note_format(&mut self, body: &'a [Stmt]) { + if let Some(docstring) = Self::docstring(body) + && let Some(format) = written_format(docstring.value.to_str()) + { + self.formats.push(format); + } + } +} + +impl<'a> SourceOrderVisitor<'a> for DocstringFinder<'a> { + fn visit_body(&mut self, body: &'a [Stmt]) { + self.note_format(body); + source_order::walk_body(self, body); + } + + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + if self.found.is_none() + && let Some(literal) = Self::docstring(&function.body) + && literal.start() < self.offset + && self.offset < literal.end() + { + let in_class = self.class_body.last().copied().unwrap_or(false); + self.found = Some(Found { + function, + literal, + in_class, + }); + } + self.class_body.push(false); + source_order::walk_stmt(self, stmt); + self.class_body.pop(); + } + Stmt::ClassDef(_) => { + self.class_body.push(true); + source_order::walk_stmt(self, stmt); + self.class_body.pop(); + } + _ => { + // a nested suite of an `if` in a class body is still the class's + source_order::walk_stmt(self, stmt); + } + } + } +} + +/// the convention a docstring is written in, when it says +fn written_format(docstring: &str) -> Option { + let lines: Vec<&str> = docstring.lines().map(str::trim).collect(); + for (index, line) in lines.iter().enumerate() { + if line.starts_with(":param ") + || line.starts_with(":return") + || line.starts_with(":rtype:") + || line.starts_with(":raises ") + { + return Some(DocstringFormat::Rest); + } + if matches!( + *line, + "Args:" | "Arguments:" | "Returns:" | "Yields:" | "Raises:" + ) { + return Some(DocstringFormat::Google); + } + if matches!(*line, "Parameters" | "Returns" | "Yields" | "Raises") + && lines + .get(index + 1) + .is_some_and(|next| next.len() >= 3 && next.chars().all(|c| c == '-')) + { + return Some(DocstringFormat::Numpy); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::{DocstringFormat, docstring_stub}; + use crate::tests::CursorTest; + use insta::assert_snapshot; + + fn stub(source: &str, format: Option) -> String { + let test = CursorTest::builder().source("main.py", source).build(); + let Some(stub) = docstring_stub( + &test.db, + test.program_file(test.cursor.file), + test.cursor.offset, + format, + ) else { + return "no stub".to_string(); + }; + let text = ruff_db::source::source_text(&test.db, test.cursor.file); + let mut rewritten = text.as_str().to_string(); + rewritten.replace_range( + stub.range.start().to_usize()..stub.range.end().to_usize(), + &stub.text, + ); + format!( + "caret at line {} of the stub, column {}\n{rewritten}", + stub.caret_line, stub.caret_column + ) + } + + #[test] + fn rest_by_default() { + assert_snapshot!(stub( + r#" +def area(width: float, height: float) -> float: + """ + + """ + return width * height +"#, + None, + ), @r#" + caret at line 1 of the stub, column 4 + + def area(width: float, height: float) -> float: + """ + + + :param width: + :param height: + :return: + """ + return width * height + "#); + } + + #[test] + fn google_skips_the_receiver_and_stars_the_variadics() { + assert_snapshot!(stub( + r#" +class Shape: + def scale(self, factor: int, *args, key: str, **kwargs): + """""" + return self +"#, + Some(DocstringFormat::Google), + ), @r#" + caret at line 1 of the stub, column 8 + + class Shape: + def scale(self, factor: int, *args, key: str, **kwargs): + """ + + + Args: + factor: + *args: + key: + **kwargs: + + Returns: + """ + return self + "#); + } + + #[test] + fn numpy_for_a_generator() { + assert_snapshot!(stub( + r#" +def count(limit): + """""" + yield limit +"#, + Some(DocstringFormat::Numpy), + ), @r#" + caret at line 1 of the stub, column 4 + + def count(limit): + """ + + + Parameters + ---------- + limit + + Yields + ------ + """ + yield limit + "#); + } + + /// another docstring in the file says which format the project writes + #[test] + fn the_file_s_other_docstrings_choose_the_format() { + assert_snapshot!(stub( + r#" +def first(a): + """Do it. + + Args: + a: the thing + """ + +class Builder: + @classmethod + def build(cls, name: str) -> None: + '''''' +"#, + None, + ), @r#" + caret at line 1 of the stub, column 8 + + def first(a): + """Do it. + + Args: + a: the thing + """ + + class Builder: + @classmethod + def build(cls, name: str) -> None: + ''' + + + Args: + name: + ''' + "#); + } + + /// a static method is told by what its decorator resolves to, and its first parameter is + /// the caller's, under whatever name `staticmethod` is bound to + #[test] + fn a_static_method_under_another_name_documents_its_first_parameter() { + assert_snapshot!(stub( + r#" +static = staticmethod + +class Builder: + @static + def build(name: str, size: int) -> None: + """""" +"#, + None, + ), @r#" + caret at line 1 of the stub, column 8 + + static = staticmethod + + class Builder: + @static + def build(name: str, size: int) -> None: + """ + + + :param name: + :param size: + """ + "#); + } + + #[test] + fn a_written_docstring_gets_no_stub() { + assert_snapshot!(stub( + r#" +def f(a): + """Already said.""" +"#, + None, + ), @"no stub"); + } + + #[test] + fn a_string_that_is_not_a_function_s_docstring_gets_no_stub() { + assert_snapshot!(stub( + r#" +def f(a): + print(a) + """""" +"#, + None, + ), @"no stub"); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 46b982ca54..a8af0c40ef 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -17,6 +17,7 @@ mod declaration; mod django_template; mod doc_highlights; mod docstring; +mod docstring_stub; mod document_symbols; mod evaluatable_expression; mod expression_types; @@ -34,6 +35,7 @@ mod importer; mod injection; mod inlay_hints; mod markup; +mod method_kind; mod module_rename; mod on_type_formatting; mod override_members; @@ -81,6 +83,7 @@ pub use django_template::{ django_template_semantic_tokens, django_template_signature_help, is_django_template_path, }; pub use doc_highlights::document_highlights; +pub use docstring_stub::{DocstringFormat, DocstringStub, docstring_stub}; pub use document_symbols::{document_symbols, symbol_detail}; pub use evaluatable_expression::evaluatable_expression; pub use expression_types::{ExpressionType, expression_types}; @@ -95,6 +98,7 @@ pub use inlay_hints::{ InlayHintKind, InlayHintLabel, InlayHintSettings, InlayHintTextEdit, inlay_hints, }; pub use markup::MarkupKind; +pub use method_kind::{MethodKind, method_kind}; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, }; diff --git a/crates/ty_ide/src/method_kind.rs b/crates/ty_ide/src/method_kind.rs new file mode 100644 index 0000000000..eb94ffa18a --- /dev/null +++ b/crates/ty_ide/src/method_kind.rs @@ -0,0 +1,78 @@ +//! which kind of method a `def` a class owns is: what its first parameter is passed + +use ruff_python_ast::{self as ast, Expr, ExprContext}; +use ty_python_semantic::types::ide_support::{InheritedMethodKind, method_kind_of}; +use ty_python_semantic::{HasDefinition, SemanticModel}; + +/// what a method's first parameter is passed when it is called through an instance +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MethodKind { + /// the instance, conventionally `self` + Instance, + /// the class, conventionally `cls` + Class, + /// nothing: every parameter is the caller's + Static, +} + +impl MethodKind { + pub const fn as_str(self) -> &'static str { + match self { + MethodKind::Instance => "instance", + MethodKind::Class => "class", + MethodKind::Static => "static", + } + } +} + +/// the kind of method `function`, a `def` written in a class body of `model`'s file, is: its +/// decorators as the checker resolves them, under whatever name they are bound to, and the +/// methods python makes static or class methods by their name alone. `None` where the checker +/// has no function for it +pub(crate) fn resolved_method_kind( + model: &SemanticModel<'_>, + function: &ast::StmtFunctionDef, +) -> Option { + let definition = function.definition(model); + Some(match method_kind_of(model.db(), definition)? { + InheritedMethodKind::Instance => MethodKind::Instance, + InheritedMethodKind::Class => MethodKind::Class, + InheritedMethodKind::Static => MethodKind::Static, + }) +} + +/// the kind of method `function` is, given that a class body defines it, for where there is no +/// semantic model to ask: [`resolved_method_kind`] resolves the decorators instead +/// +/// read from what the parser has: `@staticmethod` and `@classmethod` (bare, or through +/// `builtins`), basedpython's `static def` and `class def`, and the methods python makes static +/// or class methods by their name alone — `__new__`, and `__init_subclass__` and +/// `__class_getitem__`. any other `def` in a class body is passed its instance, decorated or not +pub fn method_kind(function: &ast::StmtFunctionDef) -> MethodKind { + for decorator in &function.decorator_list { + let name = match &decorator.expression { + // `static def` and `class def` are parsed into decorators named for the modifier, + // which no source spells, so they are told apart from a real name by their context + Expr::Name(name) if name.ctx == ExprContext::Invalid => match name.id.as_str() { + "static" => Some("staticmethod"), + "classmethod" => Some("classmethod"), + _ => None, + }, + Expr::Name(name) => Some(name.id.as_str()), + Expr::Attribute(attribute) if matches!(&*attribute.value, Expr::Name(module) if module.id.as_str() == "builtins") => { + Some(attribute.attr.as_str()) + } + _ => None, + }; + match name { + Some("staticmethod") => return MethodKind::Static, + Some("classmethod") => return MethodKind::Class, + _ => {} + } + } + match function.name.as_str() { + "__new__" => MethodKind::Static, + "__init_subclass__" | "__class_getitem__" => MethodKind::Class, + _ => MethodKind::Instance, + } +} diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 0ba1526374..515eeff6bb 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -3671,6 +3671,35 @@ pub enum InheritedMethodKind { Static, } +/// basedpython: what the method `definition`, a `def` written in a class body, is passed as its +/// first parameter: from its decorators as the checker resolves them, under whatever name they +/// are bound to, and from the methods python makes static or class methods by their name alone, +/// `__new__`, `__init_subclass__` and `__class_getitem__`. `None` for a definition that is not a +/// `def` +pub fn method_kind_of<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> Option { + let function = infer_definition_types(db, definition).function_type(definition)?; + Some(if function.is_staticmethod(db) { + InheritedMethodKind::Static + } else if function.is_classmethod(db) { + InheritedMethodKind::Class + } else { + InheritedMethodKind::Instance + }) +} + +/// basedpython: whether `ty` is the builtin class `super`, or the bound super object +/// basedpython's `super` keyword evaluates to +pub fn is_super(db: &dyn Db, ty: Type<'_>) -> bool { + match ty { + Type::ClassLiteral(class) => class.known(db) == Some(KnownClass::Super), + Type::BoundSuper(_) => true, + _ => false, + } +} + /// basedpython: a method a class inherits and does not define itself, with everything a /// declaration overriding it has to say #[derive(Debug, Clone)] diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index d72bc349de..95b788446d 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -243,6 +243,13 @@ pub(super) fn request(req: server::Request) -> Task { requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), + // asked as Enter is typed, which waits on it + requests::DocstringStubRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } // asked while a person waits on the popup, like hover requests::ExpressionTypesRequestHandler::METHOD => { background_document_request_task::( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 14c24b3394..451504525a 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -28,6 +28,7 @@ mod declaration; mod detached_transpile; mod diagnostic; mod doc_highlights; +mod docstring_stub; mod document_super_members; mod document_symbols; mod document_usage_counts; @@ -90,6 +91,7 @@ pub(super) use data_flow::DataFlowRequestHandler; pub(super) use declaration::DeclarationRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; +pub(super) use docstring_stub::DocstringStubRequestHandler; pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use document_usage_counts::DocumentUsageCountsRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/docstring_stub.rs b/crates/ty_server/src/server/api/requests/docstring_stub.rs new file mode 100644 index 0000000000..e6408e36e7 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/docstring_stub.rs @@ -0,0 +1,119 @@ +//! `by/docstringStub`: the docstring an editor writes into an empty one, with the function's +//! parameters and what it returns, in the format the project writes docstrings in. see +//! [`ty_ide::docstring_stub`] for what is answered + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ty_ide::{DocstringFormat, docstring_stub}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToRangeExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DocstringStubRequest {} + +impl Request for DocstringStubRequest { + type Params = DocstringStubParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/docstringStub"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DocstringStubParams { + text_document: TextDocumentIdentifier, + /// a position inside the empty docstring + position: Position, + /// the format to write; the one the file's other docstrings are written in when absent + #[serde(default)] + format: Option, +} + +#[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum StubFormat { + Rest, + Google, + Numpy, + Plain, +} + +/// replace `range` with `newText`, then put the caret at `caret`, a position in the document as +/// it is after the edit +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DocstringStubAnswer { + range: Range, + new_text: String, + caret: Position, +} + +pub(crate) struct DocstringStubRequestHandler; + +impl RequestHandler for DocstringStubRequestHandler { + type RequestType = DocstringStubRequest; +} + +impl BackgroundDocumentRequestHandler for DocstringStubRequestHandler { + fn document_uri(params: &DocstringStubParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` where there is no empty docstring to fill in, when language services are off and + /// for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: DocstringStubParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + if !file.source_type(db).is_py_file() { + return Ok(None); + } + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let format = params.format.map(|format| match format { + StubFormat::Rest => DocstringFormat::Rest, + StubFormat::Google => DocstringFormat::Google, + StubFormat::Numpy => DocstringFormat::Numpy, + StubFormat::Plain => DocstringFormat::Plain, + }); + let Some(stub) = docstring_stub(db, db.program_file(file), offset, format) else { + return Ok(None); + }; + let Some(range) = stub.range.to_lsp_range(db, file, snapshot.encoding()) else { + return Ok(None); + }; + let range = range.local_range(); + Ok(Some(DocstringStubAnswer { + caret: Position::new(range.start.line + stub.caret_line, stub.caret_column), + range, + new_text: stub.text, + })) + } +} + +impl RetriableRequestHandler for DocstringStubRequestHandler {} diff --git a/crates/ty_server/tests/e2e/docstring_stub.rs b/crates/ty_server/tests/e2e/docstring_stub.rs new file mode 100644 index 0000000000..a2d809e86f --- /dev/null +++ b/crates/ty_server/tests/e2e/docstring_stub.rs @@ -0,0 +1,85 @@ +//! `by/docstringStub` over the wire +//! +//! what a stub holds is `ty_ide`'s and its unit tests cover it. what these cover is the contract +//! a client reads: a position in an empty docstring and an optional format in, and out the edit +//! that replaces the literal and where the caret goes once it is applied + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// the request as a client sends it, in json throughout +enum DocstringStub {} + +impl Request for DocstringStub { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/docstringStub"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const MAIN: &str = "\ +class Cart: + def add(self, item: int) -> bool: + \"\"\"\"\"\" + return True +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask( + server: &mut TestServer, + line: u32, + character: u32, + format: Option<&str>, +) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + let mut params = serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + }); + if let Some(format) = format { + params["format"] = serde_json::json!(format); + } + server.send_request_await::(params) +} + +#[test] +fn an_empty_docstring_is_answered_with_its_stub_and_the_caret() -> Result<()> { + let mut server = server()?; + + assert_eq!( + ask(&mut server, 2, 11, Some("google")), + Some(serde_json::json!({ + "range": { + "start": { "line": 2, "character": 8 }, + "end": { "line": 2, "character": 14 }, + }, + "newText": "\"\"\"\n \n\n Args:\n item:\n\n Returns:\n \"\"\"", + "caret": { "line": 3, "character": 8 }, + })) + ); + + Ok(()) +} + +#[test] +fn anywhere_else_is_answered_with_null() -> Result<()> { + let mut server = server()?; + + assert_eq!(ask(&mut server, 3, 10, None), None); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 59e6f0bb09..4b9dd3e936 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -42,6 +42,7 @@ mod data_flow; mod declaration; mod diagnostic_snapshots; mod django_templates; +mod docstring_stub; mod evaluatable_expression; mod expression_types; mod file_watching; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index cb79ad79f2..fd1e43aef3 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -704,6 +704,43 @@ the caret goes in the document as the edits leave it: like the other document requests it takes `textHash`, and it is `null` when language services are off +## writing a docstring + +typing `"""` and Enter at the top of a function's body leaves an empty docstring, +and an editor asks `by/docstringStub` with a position inside it for the stub to +fill it with: + +```json +{ + "textDocument": { "uri": "file:///src/shapes.by" }, + "position": { "line": 2, "character": 11 }, + "format": "google" +} +``` + +the answer replaces the whole literal, quotes and prefix kept, and says where the +caret goes in the document once `newText` is in it — the summary line, which is +written indented and empty: + +```json +{ + "range": { "start": { "line": 2, "character": 8 }, "end": { "line": 2, "character": 14 } }, + "newText": "\"\"\"\n \n\n Args:\n item:\n\n Returns:\n \"\"\"", + "caret": { "line": 3, "character": 8 } +} +``` + +the stub lists every parameter but a method's receiver — the instance or class +a method is passed, and the class `__new__` is — with `*args` and `**kwargs`, and +says what the function hands back: `Yields` for a generator, `Returns` when its +annotation is something other than `None`, `NoReturn` or `Never`, or with no +annotation when it returns a value. `format` is `rest` (`:param item:`), +`google`, `numpy` or `plain` (the summary alone); without one, the format is the +one the file's other docstrings are written in, and reStructuredText, as PyCharm +writes by default, when none of them says. the answer is `null` anywhere that is +not inside a triple-quoted docstring of a function holding nothing but whitespace +on a line of its own + ## suppressing a diagnostic every diagnostic has two quick fixes that silence it rather than fix it. From 393bcb770ebfc6fef655f55b07b6bfd267dc96fd Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:30:45 +1000 Subject: [PATCH 029/110] say on each def a class body makes in by/syntaxOutline whether it is an instance, class or static method, from its decorators, static def and class def, and the methods python makes so by name, for an editor to write self or cls when the parameter list is opened --- crates/ty_ide/src/lib.rs | 2 +- crates/ty_ide/src/method_kind.rs | 2 +- crates/ty_ide/src/syntax_outline.rs | 113 ++++++++++++++++-- .../src/server/api/requests/syntax_outline.rs | 5 + docs/basedpython/features/editor.md | 7 +- 5 files changed, 119 insertions(+), 10 deletions(-) diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index a8af0c40ef..682ddb5b0f 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -98,7 +98,7 @@ pub use inlay_hints::{ InlayHintKind, InlayHintLabel, InlayHintSettings, InlayHintTextEdit, inlay_hints, }; pub use markup::MarkupKind; -pub use method_kind::{MethodKind, method_kind}; +pub use method_kind::MethodKind; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, }; diff --git a/crates/ty_ide/src/method_kind.rs b/crates/ty_ide/src/method_kind.rs index eb94ffa18a..28cdc7799d 100644 --- a/crates/ty_ide/src/method_kind.rs +++ b/crates/ty_ide/src/method_kind.rs @@ -48,7 +48,7 @@ pub(crate) fn resolved_method_kind( /// `builtins`), basedpython's `static def` and `class def`, and the methods python makes static /// or class methods by their name alone — `__new__`, and `__init_subclass__` and /// `__class_getitem__`. any other `def` in a class body is passed its instance, decorated or not -pub fn method_kind(function: &ast::StmtFunctionDef) -> MethodKind { +pub(crate) fn method_kind(function: &ast::StmtFunctionDef) -> MethodKind { for decorator in &function.decorator_list { let name = match &decorator.expression { // `static def` and `class def` are parsed into decorators named for the modifier, diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index 100e2bb834..b4ec2d7f74 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -25,6 +25,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::Db; use crate::comma_list::comma_lists; use crate::injection::text_ranges; +use crate::method_kind::{MethodKind, method_kind}; /// The block structure and string literals of one file. #[derive(Debug, Clone, PartialEq, Eq)] @@ -64,6 +65,8 @@ pub struct OutlineStatement { pub name: Option, /// For a `def`, the names of its parameters, in source order. pub parameters: Vec, + /// for a `def` a class body makes, what its first parameter is passed + pub method: Option, } /// One clause of a compound statement: `if`, `elif`, `else`, `case`, the `def` of a function. @@ -141,7 +144,7 @@ pub fn syntax_outline(db: &dyn Db, file: PythonFile<'_>) -> SyntaxOutline { tokens: parsed.tokens(), }; - let statements = builder.statements(parsed.suite()); + let statements = builder.statements(parsed.suite(), false); let lists = comma_lists(parsed.suite(), parsed.tokens()) .into_iter() .filter(|list| list.elements.len() >= 2) @@ -240,27 +243,46 @@ impl<'a> OutlineBuilder<'a> { } impl OutlineBuilder<'_> { - fn statements(&self, suite: &[Stmt]) -> Vec { - suite.iter().map(|stmt| self.statement(stmt)).collect() + /// the statements of `suite`; `in_class` says whether it is a class's body, or a suite of + /// a statement in one, whose `def`s the class owns + fn statements(&self, suite: &[Stmt], in_class: bool) -> Vec { + suite + .iter() + .map(|stmt| self.statement(stmt, in_class)) + .collect() } - fn statement(&self, stmt: &Stmt) -> OutlineStatement { + fn statement(&self, stmt: &Stmt, in_class: bool) -> OutlineStatement { + let method = match stmt { + Stmt::FunctionDef(function) if in_class && !function.is_trailing_lambda => { + Some(method_kind(function)) + } + _ => None, + }; OutlineStatement { range: stmt.range(), - clauses: self.clauses(stmt), + clauses: self.clauses(stmt, in_class), modifiers: self.modifiers(stmt), call: self.call(stmt), name: Self::name(stmt), parameters: Self::parameters(stmt), + method, } } /// The clauses of `stmt`, each with its body. - fn clauses(&self, stmt: &Stmt) -> Vec { + fn clauses(&self, stmt: &Stmt, in_class: bool) -> Vec { + // a class's body is the class's; a function's is not, and any other compound statement + // keeps whatever the suite around it was + let body_in_class = match stmt { + Stmt::ClassDef(_) => true, + Stmt::FunctionDef(_) => false, + _ => in_class, + }; self.clause_heads(stmt) .into_iter() .map(|head| { - let body = self.statements(head.body); + let body = self.statements(head.body, body_in_class); OutlineClause { keyword: head.keyword, colon: head.colon, @@ -1553,4 +1575,81 @@ else: "C\nm(self, x, rest, kw)\nf\n" ); } + + /// every statement of the outline, depth first, with what `pick` says about it + fn each_statement( + statements: &[OutlineStatement], + text: &str, + pick: &dyn Fn(&OutlineStatement, &str) -> Option, + out: &mut Vec, + ) { + for statement in statements { + if let Some(said) = pick(statement, text) { + out.push(said); + } + for clause in &statement.clauses { + each_statement(&clause.body, text, pick, out); + } + } + } + + fn methods(source: &str) -> String { + let test = test("main.by", source); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let outline = syntax_outline(&test.db, file); + let mut out = Vec::new(); + each_statement( + &outline.statements, + test.cursor.source.as_str(), + &|statement, text| { + let name = &text[statement.name?.to_std_range()]; + let method = statement + .method + .map_or("not a method", |kind| kind.as_str()); + Some(format!("{name}: {method}")) + }, + &mut out, + ); + out.join("\n") + } + + #[test] + fn a_class_s_defs_say_what_their_first_parameter_is_passed() { + insta::assert_snapshot!(methods( + " +class Shape: + def area(self): ... + class def unit(cls): ... + static def origin(): ... + @staticmethod + def zero(): ... + @classmethod + def build(cls): ... + @property + def size(self): ... + def __new__(cls): ... + def __init_subclass__(cls): ... + if True: + def maybe(self): ... + def outer(self): + def inner(): ... + +def free(): ... +" + ), @" + Shape: not a method + area: instance + unit: class + origin: static + zero: static + build: class + size: instance + __new__: static + __init_subclass__: class + maybe: instance + outer: instance + inner: not a method + free: not a method + "); + } } diff --git a/crates/ty_server/src/server/api/requests/syntax_outline.rs b/crates/ty_server/src/server/api/requests/syntax_outline.rs index 979123667d..fde5028ee7 100644 --- a/crates/ty_server/src/server/api/requests/syntax_outline.rs +++ b/crates/ty_server/src/server/api/requests/syntax_outline.rs @@ -93,6 +93,10 @@ pub(crate) struct Statement { /// On a `def`: the names of its parameters, in source order. #[serde(default, skip_serializing_if = "Vec::is_empty")] parameters: Vec, + /// on a `def` a class body makes: `instance`, `class` or `static`, what its first parameter + /// is passed + #[serde(default, skip_serializing_if = "Option::is_none")] + method: Option, } /// One clause of a compound statement. @@ -278,6 +282,7 @@ impl Converter<'_> { .iter() .filter_map(|range| self.range(*range)) .collect(), + method: statement.method.map(|kind| kind.as_str().to_string()), }) }) .collect() diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index fd1e43aef3..a764be8027 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1169,7 +1169,12 @@ holds — one per line of a dedented string, past the indentation, and around th interpolations of an f-string. that is what an editor needs to put another language into a string it was not told about, with a place in the file for every character of the fragment. the outline also carries the name each `def` and -`class` declares and a `def`'s parameter names, and the elements of every +`class` declares, a `def`'s parameter names, and on a `def` a class body makes, +`method`: `instance`, `class` or `static`, what its first parameter is passed — +from `@staticmethod` and `@classmethod`, `static def` and `class def`, and the +methods python makes so by name, `__new__` static (it is handed its class +explicitly, as its first argument) and `__init_subclass__` and `__class_getitem__` +class methods; a plain function has none. and it carries the elements of every comma-separated list of two or more — what an editor moves an argument among, or tracks a declaration being renamed by From f33ebe3929698e6a0e7d7441831ab45e955f65ae Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:33:11 +1000 Subject: [PATCH 030/110] name in by/syntaxOutline the expression of a simple statement or a clause header that Enter may break by wrapping it in parentheses, an assignment's value or an if's condition, and leave out a statement of two expressions and basedpython's forms whose parentheses are syntax --- crates/ty_ide/src/syntax_outline.rs | 191 ++++++++++++++++++ .../src/server/api/requests/syntax_outline.rs | 8 + docs/basedpython/features/editor.md | 16 +- 3 files changed, 214 insertions(+), 1 deletion(-) diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index b4ec2d7f74..e8247ec2b0 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -67,6 +67,9 @@ pub struct OutlineStatement { pub parameters: Vec, /// for a `def` a class body makes, what its first parameter is passed pub method: Option, + /// For a simple statement, the expression Enter may break across lines by wrapping it in + /// parentheses; see [`wrappable`]. + pub wrap: Option, } /// One clause of a compound statement: `if`, `elif`, `else`, `case`, the `def` of a function. @@ -83,6 +86,9 @@ pub struct OutlineClause { pub range: TextRange, /// The statements of the suite. pub body: Vec, + /// The expression of the header Enter may break across lines by wrapping it in parentheses; + /// see [`wrappable`]. + pub wrap: Option, } /// A basedpython modifier keyword, as the parser read it. @@ -267,6 +273,9 @@ impl OutlineBuilder<'_> { name: Self::name(stmt), parameters: Self::parameters(stmt), method, + wrap: statement_expression(stmt) + .filter(|expr| wrappable(expr)) + .map(Ranged::range), } } @@ -279,15 +288,23 @@ impl OutlineBuilder<'_> { Stmt::FunctionDef(_) => false, _ => in_class, }; + let headers = header_expressions(stmt); self.clause_heads(stmt) .into_iter() .map(|head| { let body = self.statements(head.body, body_in_class); + let header_end = head.colon.map_or(head.range.end(), TextRange::start); + let header = TextRange::new(head.range.start(), header_end.max(head.range.start())); OutlineClause { keyword: head.keyword, colon: head.colon, range: head.range, body, + wrap: headers + .iter() + .find(|expr| header.contains_range(expr.range())) + .filter(|expr| wrappable(expr)) + .map(|expr| Ranged::range(*expr)), } }) .collect() @@ -618,6 +635,82 @@ impl OutlineBuilder<'_> { } /// One clause of a compound statement, as [`OutlineBuilder::clause_heads`] finds it. +/// The one expression of a simple statement that Enter could break: an assignment's value, a +/// `return`'s, an expression statement, a `raise` without `from`, an `assert` without a message +/// and a `type` alias's value. A statement with two expressions, such as `raise E from cause`, +/// has none, since one pair of parentheses cannot hold both. +fn statement_expression(stmt: &Stmt) -> Option<&Expr> { + match stmt { + Stmt::Expr(stmt) => Some(&stmt.value), + Stmt::Return(stmt) => stmt.value.as_deref(), + Stmt::Assign(stmt) if stmt.decorator_list.is_empty() => Some(&stmt.value), + Stmt::AugAssign(stmt) => Some(&stmt.value), + Stmt::AnnAssign(stmt) if stmt.decorator_list.is_empty() && !stmt.is_context => { + stmt.value.as_deref() + } + Stmt::Raise(stmt) if stmt.cause.is_none() => stmt.exc.as_deref(), + Stmt::Assert(stmt) if stmt.msg.is_none() => Some(&stmt.test), + Stmt::TypeAlias(stmt) => Some(&stmt.value), + _ => None, + } +} + +/// The expressions clause headers of `stmt` hold that Enter could break: the condition of an +/// `if`, an `elif` and a `while`, the iterable of a `for` and the subject of a `match`. +fn header_expressions(stmt: &Stmt) -> Vec<&Expr> { + match stmt { + Stmt::If(stmt) if stmt.pattern.is_none() => std::iter::once(&*stmt.test) + .chain( + stmt.elif_else_clauses + .iter() + .filter_map(|clause| clause.test.as_ref()), + ) + .collect(), + Stmt::While(stmt) => vec![&*stmt.test], + Stmt::For(stmt) => vec![&*stmt.iter], + Stmt::Match(stmt) => vec![&*stmt.subject], + _ => Vec::new(), + } +} + +/// Whether `expr` means the same wrapped in parentheses, so that Enter inside it can break it +/// over lines by wrapping it rather than with a backslash. +/// +/// Every python expression in these positions does — `x = (a + b)`, `if (a and b):`, +/// `return (1, 2)`. What does not is basedpython's: a statement used as an expression, a type +/// written in a form only an annotation reads, and a tuple whose parentheses are its syntax, +/// such as an anonymous named tuple, all of which are left alone, along with a starred +/// expression on its own. +fn wrappable(expr: &Expr) -> bool { + struct Unwrappable(bool); + impl<'a> SourceOrderVisitor<'a> for Unwrappable { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Statement(_) + | Expr::CallableType(_) + | Expr::ProtocolType(_) + | Expr::ProtocolMethod(_) + | Expr::IpyEscapeCommand(_) => self.0 = true, + Expr::Tuple(tuple) + if tuple.is_anon_named_tuple + || tuple.is_anon_named_tuple_value + || tuple.callable_shape.is_some() + || tuple.is_parameter_shape => + { + self.0 = true; + } + _ => walk_expr(self, expr), + } + } + } + if expr.is_starred_expr() { + return false; + } + let mut unwrappable = Unwrappable(false); + unwrappable.visit_expr(expr); + !unwrappable.0 +} + pub(crate) struct ClauseHead<'s> { /// The clause keyword; `None` on a trailing lambda's clause. pub(crate) keyword: Option, @@ -1652,4 +1745,102 @@ def free(): ... free: not a method "); } + + fn wraps(source: &str) -> String { + let test = test("main.by", source); + let file = test.program_file(test.cursor.file).python_file(&test.db); + let outline = syntax_outline(&test.db, file); + let text = test.cursor.source.as_str(); + let mut out = Vec::new(); + each_statement( + &outline.statements, + text, + &|statement, text| { + let mut said: Vec = statement + .clauses + .iter() + .filter_map(|clause| clause.wrap) + .map(|wrap| format!("header `{}`", &text[wrap.to_std_range()])) + .collect(); + said.extend( + statement + .wrap + .map(|wrap| format!("`{}`", &text[wrap.to_std_range()])), + ); + let first_line = text[statement.range.to_std_range()] + .lines() + .next() + .unwrap_or_default() + .trim(); + if said.is_empty() { + said.push("nothing".to_string()); + } + Some(format!("{first_line} -> {}", said.join(", "))) + }, + &mut out, + ); + out.join("\n") + } + + #[test] + fn the_expression_enter_may_wrap_in_parentheses() { + insta::assert_snapshot!(wraps( + " +total = price * count + tax +total += 1 +return total > limit +print(total) +raise Missing(name) +raise Missing(name) from error +assert total and limit +assert total, 'message' +if total and limit: + pass +elif total or limit: + pass +while total < limit: + pass +for item in items + extra: + pass +match command.split(): + case _: + pass +import os +" + ), @" + total = price * count + tax -> `price * count + tax` + total += 1 -> `1` + return total > limit -> `total > limit` + print(total) -> `print(total)` + raise Missing(name) -> `Missing(name)` + raise Missing(name) from error -> nothing + assert total and limit -> `total and limit` + assert total, 'message' -> nothing + if total and limit: -> header `total and limit`, header `total or limit` + pass -> nothing + pass -> nothing + while total < limit: -> header `total < limit` + pass -> nothing + for item in items + extra: -> header `items + extra` + pass -> nothing + match command.split(): -> header `command.split()` + pass -> nothing + import os -> nothing + "); + } + + /// basedpython writes some expressions whose parentheses are their syntax, or that are + /// statements, and wrapping them would read as something else + #[test] + fn basedpython_s_own_forms_are_not_wrapped() { + insta::assert_snapshot!(wraps( + " +point = (name='ada', age=36) +handler: (int) -> str = render +" + ), @" + point = (name='ada', age=36) -> nothing + handler: (int) -> str = render -> `render` + "); + } } diff --git a/crates/ty_server/src/server/api/requests/syntax_outline.rs b/crates/ty_server/src/server/api/requests/syntax_outline.rs index fde5028ee7..47e9832393 100644 --- a/crates/ty_server/src/server/api/requests/syntax_outline.rs +++ b/crates/ty_server/src/server/api/requests/syntax_outline.rs @@ -97,6 +97,9 @@ pub(crate) struct Statement { /// is passed #[serde(default, skip_serializing_if = "Option::is_none")] method: Option, + /// On a simple statement: the expression Enter may break by wrapping it in parentheses. + #[serde(default, skip_serializing_if = "Option::is_none")] + wrap: Option, } /// One clause of a compound statement. @@ -117,6 +120,9 @@ pub(crate) struct Clause { /// The suite's statements. #[serde(default, skip_serializing_if = "Vec::is_empty")] body: Vec, + /// The header's expression Enter may break by wrapping it in parentheses. + #[serde(default, skip_serializing_if = "Option::is_none")] + wrap: Option, } /// A basedpython modifier keyword. @@ -256,6 +262,7 @@ impl Converter<'_> { colon: clause.colon.and_then(|range| self.range(range)), range: self.range(clause.range)?, body: self.statements(&clause.body), + wrap: clause.wrap.and_then(|range| self.range(range)), }) }) .collect(), @@ -283,6 +290,7 @@ impl Converter<'_> { .filter_map(|range| self.range(*range)) .collect(), method: statement.method.map(|kind| kind.as_str().to_string()), + wrap: statement.wrap.and_then(|range| self.range(range)), }) }) .collect() diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index a764be8027..76ad6b337b 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1174,7 +1174,21 @@ character of the fragment. the outline also carries the name each `def` and from `@staticmethod` and `@classmethod`, `static def` and `class def`, and the methods python makes so by name, `__new__` static (it is handed its class explicitly, as its first argument) and `__init_subclass__` and `__class_getitem__` -class methods; a plain function has none. and it carries the elements of every +class methods; a plain function has none. + +a line broken inside an expression needs the expression in brackets, or a +backslash, to go on parsing. `wrap` names the expression Enter may break by +wrapping it in parentheses: on a simple statement, its one expression — an +assignment's value (`=`, an augmented one or an annotated one), a `return`'s, an +expression statement, a `raise` with no `from`, an `assert` with no message, a +`type` alias's value; on a clause, its header's — the condition of `if`, `elif` +and `while`, the iterable of `for`, the subject of `match`. each of these means the +same in parentheses, so an editor breaking a line inside one, outside any bracket +already there, writes `(` before the range and `)` after it. a statement with two +expressions (`raise E from cause`, `assert x, message`) has none, and nor does an +expression whose parentheses are syntax or that is not one in python's sense — a +basedpython anonymous named tuple, a callable or protocol type, a statement used as +an expression, a starred expression. and it carries the elements of every comma-separated list of two or more — what an editor moves an argument among, or tracks a declaration being renamed by From 39934608c5ba4ec765790899282e1f2dc449a013 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:38:53 +1000 Subject: [PATCH 031/110] check a program by run runs against the python --python or PYTHON names to run it on, handing that interpreter's environment to the checker, so a package only it has resolves in the check before the run rather than failing it as an unresolved import --- crates/by_stage/src/project.rs | 30 +++++++++++- crates/ty/src/by_commands.rs | 52 +++++++++++++++++---- crates/ty/tests/by_e2e.rs | 77 +++++++++++++++++++++++++++++++ docs/basedpython/cli-reference.md | 5 +- docs/basedpython/packaging.md | 5 ++ 5 files changed, 159 insertions(+), 10 deletions(-) diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 9654ad8a49..52ecfbdeea 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -22,6 +22,8 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; +use ty_project::metadata::options::{EnvironmentOptions, Options}; +use ty_project::metadata::value::RelativePathBuf; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; /// Everything needed to build a project db a second time. @@ -267,6 +269,23 @@ pub fn build_project_db( cwd: &Path, extensions: &[&str], output: Option<&Path>, +) -> anyhow::Result { + build_project_db_for(cwd, extensions, output, None) +} + +/// [`build_project_db`], checking the project against the python environment `python` — an +/// interpreter or the environment holding it, as `by check --python` takes — rather than the +/// one the project's configuration and discovery would find +/// +/// what runs a program decides which packages it imports, so a command that runs the program on +/// an interpreter it was told about has to check it against that interpreter's packages: checked +/// against another environment, an import the program will find is reported as unresolved, and +/// one it will not find passes +pub fn build_project_db_for( + cwd: &Path, + extensions: &[&str], + output: Option<&Path>, + python: Option<&SystemPath>, ) -> anyhow::Result { // the project root must be canonicalized the same way the included files // are (below) so it stays a path *prefix* of them: otherwise a file's @@ -278,8 +297,17 @@ pub fn build_project_db( let sys_cwd = SystemPath::from_std_path(&canonical_cwd) .with_context(|| format!("non-utf8 path: {}", canonical_cwd.display()))?; let system = OsSystem::new(sys_cwd); - let project_metadata = ProjectMetadata::discover(sys_cwd, &system) + let mut project_metadata = ProjectMetadata::discover(sys_cwd, &system) .with_context(|| format!("failed to discover project at {sys_cwd}"))?; + if let Some(python) = python { + project_metadata.set_override_options(Options { + environment: Some(EnvironmentOptions { + python: Some(RelativePathBuf::cli(python)), + ..EnvironmentOptions::default() + }), + ..Options::default() + }); + } // the project is the project wherever the command was run from. rooting this // at the working directory instead means `by run` inside `tests/` transpiles diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 18ef9b59c8..0c0a254dbe 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -12,7 +12,7 @@ use ruff_db::diagnostic::{ SubDiagnostic, SubDiagnosticSeverity, }; use ruff_db::files::system_path_to_file; -use ruff_db::system::{OsSystem, SystemPath}; +use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; use ruff_text_size::TextRange; use ty_project::metadata::options::OutputFormat; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; @@ -27,8 +27,8 @@ use by_stage::emit::{ CheckGate, Emit, Transpiled, is_unusable_source, transpile_failure_diagnostic, }; use by_stage::project::{ - BY_SOURCES, COMPILABLE_SOURCES, Rebuilder, build_project_db, may_contain_sources, module_roots, - source_files, + BY_SOURCES, COMPILABLE_SOURCES, Rebuilder, build_project_db, build_project_db_for, + may_contain_sources, module_roots, source_files, }; use by_stage::record::{BuildRecord, parse_soundness, stage_build_record}; use by_stage::runtime::RuntimeLayout; @@ -216,7 +216,13 @@ pub(crate) fn cmd_run( crate::by_stamps::fill_discovered(&mut config.stamps, project.root(), Some(&target)); let tmp = tempfile::TempDir::new().context("failed to create temp directory")?; - let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, None)?; + // the program is checked against the packages of the python it is about to run on + let checked_against = interpreter + .named_for_run + .then(|| environment_of(&python)) + .flatten(); + let (db, handles, rebuilder, root) = + build_project_db_for(&cwd, BY_SOURCES, None, checked_against.as_deref())?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Failure); @@ -680,6 +686,10 @@ fn latest_python_version() -> PythonVersion { struct Interpreter { path: String, origin: String, + /// whether the command was told which python to run on, by `--python` or `$PYTHON`, rather + /// than finding the project's environment: the checker finds that one itself, and an + /// interpreter named only for the run has to be handed to it + named_for_run: bool, /// whether this is the bare `python3` off `PATH` — the last resort, and the /// only origin that says nothing about what the project targets is_from_path: bool, @@ -710,8 +720,13 @@ fn discover_interpreter( let named = |path: String, origin: &str| Interpreter { path, origin: origin.to_owned(), + named_for_run: false, is_from_path: false, }; + let named_for_run = |path: String, origin: &str| Interpreter { + named_for_run: true, + ..named(path, origin) + }; if let Some(flag) = flag { // a `--python` may name the interpreter itself or the environment it @@ -719,14 +734,14 @@ fn discover_interpreter( // run as it is written, which finds a bare name like `python3.14` on // `PATH`, as `$PYTHON` and the `python3` fallback are found if flag.is_file() { - return Ok(named(flag.display().to_string(), "`--python`")); + return Ok(named_for_run(flag.display().to_string(), "`--python`")); } if let Some(interpreter) = interpreter_in_environment(flag, SysPrefixPathOrigin::PythonCliFlag) { - return Ok(named(interpreter, "`--python`")); + return Ok(named_for_run(interpreter, "`--python`")); } - return Ok(named(flag.display().to_string(), "`--python`")); + return Ok(named_for_run(flag.display().to_string(), "`--python`")); } // what the project says its environment is, which is what the checker used. @@ -756,12 +771,13 @@ fn discover_interpreter( } if let Ok(python) = std::env::var(EnvVars::PYTHON) { - return Ok(named(python, "`PYTHON`")); + return Ok(named_for_run(python, "`PYTHON`")); } Ok(discovered.unwrap_or_else(|| Interpreter { path: "python3".to_owned(), origin: "`PATH`".to_owned(), + named_for_run: false, is_from_path: true, })) } @@ -903,6 +919,7 @@ fn discovered_environment(root: &Path) -> Option { Some(Interpreter { path: interpreter.to_string(), origin: environment.origin().to_string(), + named_for_run: false, is_from_path, }) } @@ -922,6 +939,25 @@ fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Optio /// Probe `python`'s `major.minor` version (e.g. `3.9`) so `run` can target the /// interpreter it will execute on. Returns `None` if the interpreter can't be /// run or its output can't be parsed. +/// The environment `python` runs in, as the checker takes one: the interpreter's own path when +/// it names a file, and otherwise — a bare name such as `python3.14`, found on `PATH` — the +/// `sys.prefix` the interpreter reports, since the checker does not search `PATH`. +fn environment_of(python: &str) -> Option { + if Path::new(python).is_file() { + return SystemPathBuf::from_path_buf(std::path::absolute(python).ok()?).ok(); + } + let output = Command::new(python) + .arg("-c") + .arg("import sys; print(sys.prefix)") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let prefix = String::from_utf8(output.stdout).ok()?; + Some(SystemPathBuf::from(prefix.trim())) +} + fn detect_python_version(python: &str) -> Option { let output = Command::new(python) .arg("-c") diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 926c201869..0d91119d5b 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -902,6 +902,83 @@ fn run_executes_module() { ); } +/// a program is checked against the packages of the python `--python` names to run it on, not +/// against the environment the project would have found, so a package only that python has +/// resolves rather than failing the check that comes before the run +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_checks_against_the_python_it_runs_on() { + let dir = tempfile::tempdir().expect("tempdir"); + let environment = dir.path().join("env"); + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let made = Command::new(&python) + .args(["-m", "venv", "--without-pip"]) + .arg(&environment) + .status() + .is_ok_and(|status| status.success()); + if !made { + eprintln!("skipping: {python} cannot make a virtual environment"); + return; + } + // where the environment keeps its packages, which is `lib/pythonX.Y/site-packages` on unix + // and `Lib/site-packages` on windows + let purelib = Command::new(&python) + .args([ + "-c", + "import sys, sysconfig\n\ + print(sysconfig.get_path('purelib', 'venv', vars={'base': sys.argv[1], 'platbase': sys.argv[1]}))", + ]) + .arg(&environment) + .output() + .expect("failed to spawn python"); + if !purelib.status.success() { + eprintln!( + "skipping: {python} cannot say where a virtual environment keeps its packages:\n{}", + String::from_utf8_lossy(&purelib.stderr) + ); + return; + } + let site_packages = PathBuf::from(String::from_utf8_lossy(&purelib.stdout).trim()); + fs::create_dir_all(site_packages.join("fancypkg")).unwrap(); + fs::write(site_packages.join("fancypkg/__init__.py"), "VALUE = 42\n").unwrap(); + + let project = dir.path().join("project"); + fs::create_dir_all(&project).unwrap(); + fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"project\"\nversion = \"0\"\n[tool.basedpython]\n", + ) + .unwrap(); + fs::write( + project.join("main.by"), + "import fancypkg\n\nprint(fancypkg.VALUE)\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env_remove("VIRTUAL_ENV") + .env_remove("CONDA_PREFIX") + .env_remove("PYTHON") + .arg("run") + .arg("--python") + .arg(&environment) + .arg("main") + .current_dir(&project) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + /// the program runs out of a directory of transpiled copies that is deleted when /// the run ends, so every path python derives from a module's origin named a /// temporary file — leaving a tool started anywhere but the project root with diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 57f63a6aec..a7bbbe343c 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -59,7 +59,10 @@ project's `pyproject.toml`. `--python` overrides it for one run, and `PYTHON` stands in only where the project has no environment of its own, above the bare `python3` discovery falls back to. a third-party package lives in that environment's `site-packages` and nowhere else, so running anywhere else fails -on an import the checker had no complaint about. all of that resolves against +on an import the checker had no complaint about. so the check before the run is +made against the python the run uses: one `--python` or `PYTHON` names, a bare name +found on `PATH` by the `sys.prefix` it reports, has its packages checked as well as +run. all of that resolves against the project root rather than the working directory, so `by run` in a subdirectory is still this project diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index 4edb350506..c80bbcaae9 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -258,6 +258,11 @@ against, resolved the same way: 1. `$PYTHON` 1. `python3` on `PATH` +the check `by run` makes first is against the same python: a package only the +interpreter `--python` or `$PYTHON` names has installed is found by the check as it +is by the run, rather than reported as an unresolved import because the project's +own environment lacks it + all of it relative to the project, not to where the command was run: `by run` in a subdirectory is still this project, uses the project's `.venv`, and builds the project's modules From cd97c7c6b2950121844d90319388bddabd439f0d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:45:04 +1000 Subject: [PATCH 032/110] report a string handed to python's re functions as their pattern as PythonRegExp, resolved through the call and carried through parameters that pass it on, and read a # language= comment trailing a simple statement as marking that statement and no other --- crates/ty_ide/src/injection.rs | 151 +++++++++++++++++- .../src/types/ide_support.rs | 20 +++ .../features/language-injection.md | 28 ++++ 3 files changed, 197 insertions(+), 2 deletions(-) diff --git a/crates/ty_ide/src/injection.rs b/crates/ty_ide/src/injection.rs index 4e4998d09a..81c217f508 100644 --- a/crates/ty_ide/src/injection.rs +++ b/crates/ty_ide/src/injection.rs @@ -61,7 +61,7 @@ use rustc_hash::FxHashMap; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_semantic::SemanticModel; -use ty_python_semantic::types::ide_support::call_signature_details; +use ty_python_semantic::types::ide_support::{call_signature_details, is_re_pattern_parameter}; use crate::Db; @@ -77,6 +77,11 @@ const MAX_PROPAGATION_DEPTH: usize = 8; /// The text both spellings share. const MARKER: &str = "language="; +/// the language a pattern handed to python's `re` functions is written in: regular expressions +/// as python's `re` reads them, `(?P...)` groups and all. PyCharm calls its own support for +/// that language this; an editor with only a generic regular expression language maps it there +const PYTHON_REGEX: &str = "PythonRegExp"; + /// A fragment of another language inside a basedpython file. #[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] pub struct Injection { @@ -244,6 +249,15 @@ impl InjectionFinder<'_, '_> { .iter() .rev() .find(|(range, _)| range.end() <= statement.start()) + // a marker trailing the statement before is that statement's, not this one's + .filter(|(range, _)| { + let line_start = self.source[..range.start().to_usize()] + .rfind('\n') + .map_or(0, |newline| newline + 1); + self.source[line_start..range.start().to_usize()] + .chars() + .all(char::is_whitespace) + }) .filter(|(range, _)| { let between = TextRange::new(range.end(), statement.start()); self.source @@ -257,6 +271,27 @@ impl InjectionFinder<'_, '_> { .map(|(_, language)| language.clone()) } + /// the language a marker comment at the end of `statement`'s last line gives it, when + /// `statement` is a simple statement and nothing but spaces comes between the two + /// + /// a compound statement ends with the end of its body, whose own last statement the + /// comment trails, and a statement followed by another on its line is not the one the + /// comment ends + fn trailing_marker(&self, statement: &ast::Stmt) -> Option { + if is_compound(statement) { + return None; + } + self.markers + .iter() + .find(|(range, _)| range.start() >= statement.end()) + .filter(|(range, _)| { + self.source + .get(statement.end().to_usize()..range.start().to_usize()) + .is_some_and(|between| between.chars().all(|c| c == ' ' || c == '\t')) + }) + .map(|(_, language)| language.clone()) + } + /// Record every string literal in `statement`'s own expressions as a /// fragment of `language`. /// @@ -338,7 +373,10 @@ fn parameters_matching<'db>( impl<'db> SourceOrderVisitor<'db> for InjectionFinder<'_, 'db> { fn visit_stmt(&mut self, statement: &'db ast::Stmt) { - if let Some(language) = self.marker_for_statement(statement) { + if let Some(language) = self + .marker_for_statement(statement) + .or_else(|| self.trailing_marker(statement)) + { self.mark_statement(statement, &language); } walk_stmt(self, statement); @@ -471,6 +509,11 @@ fn parameter_language( { return Some((language, InjectionOrigin::Declared)); } + // what python's own `re` functions take as a pattern is a regular expression, which their + // stubs cannot say in the annotation, so it is read as declared here + if is_re_pattern_parameter(db, definition, name) { + return Some((PYTHON_REGEX.to_string(), InjectionOrigin::Declared)); + } // Nothing declared here, so follow the value: if the body does no more than // hand this parameter on, whatever the parameter it reaches expects is what @@ -492,6 +535,21 @@ fn parameter_language( }) } +/// whether `statement` has a body of statements of its own +fn is_compound(statement: &ast::Stmt) -> bool { + matches!( + statement, + ast::Stmt::FunctionDef(_) + | ast::Stmt::ClassDef(_) + | ast::Stmt::If(_) + | ast::Stmt::For(_) + | ast::Stmt::While(_) + | ast::Stmt::With(_) + | ast::Stmt::Try(_) + | ast::Stmt::Match(_) + ) || matches!(statement, ast::Stmt::Let(stmt) if !stmt.orelse.is_empty()) +} + /// The language an annotation declares, from `Annotated[T, "language="]`. /// /// [PEP 593](https://peps.python.org/pep-0593/) put `Annotated`'s metadata there @@ -1064,4 +1122,93 @@ run_later(\"const x = 1\") | ^^^^^^^^^^^ "#); } + + #[test] + fn a_marker_trailing_the_statement_marks_it() { + let test = injection_test( + "\ +query = \"select * from users\" # language=sql +other = \"not marked\" +", + ); + assert_snapshot!(test.injections(), @r#" + info[injection]: sql (comment) + --> main.by:1:10 + | + 1 | query = "select * from users" # language=sql + | ^^^^^^^^^^^^^^^^^^^ + "#); + } + + #[test] + fn a_trailing_marker_after_a_second_statement_is_that_one_s() { + let test = injection_test( + "\ +first = \"a\"; second = \"b\" # language=sql +", + ); + assert_snapshot!(test.injections(), @r#" + info[injection]: sql (comment) + --> main.by:1:24 + | + 1 | first = "a"; second = "b" # language=sql + | ^ + "#); + } + + /// what `re.compile` and the other `re` functions take as their pattern is python's regular + /// expression syntax, resolved through the call rather than read off the name `re` + #[test] + fn a_re_pattern_is_python_s_regular_expressions() { + let test = injection_test( + "\ +import re +from re import sub as replace + +word = re.compile(\"(?P\\\\w+)\") +replace(\"a+\", \"b\", \"aaa\") +re.split(\",\", \"a,b\") +", + ); + assert_snapshot!(test.injections(), @r#" + info[injection]: PythonRegExp (declared) + --> main.by:4:20 + | + 4 | word = re.compile("(?P\\w+)") + | ^^^^^^^^^^^^^^ + + info[injection]: PythonRegExp (declared) + --> main.by:5:10 + | + 5 | replace("a+", "b", "aaa") + | ^^ + + info[injection]: PythonRegExp (declared) + --> main.by:6:11 + | + 6 | re.split(",", "a,b") + | ^ + "#); + } + + #[test] + fn a_re_pattern_travels_through_a_parameter_passed_straight_on() { + let test = injection_test( + "\ +import re + +def find(pattern: str, text: str): + return re.search(pattern, text) + +find(\"[0-9]+\", \"abc 123\") +", + ); + assert_snapshot!(test.injections(), @r#" + info[injection]: PythonRegExp (propagated) + --> main.by:6:7 + | + 6 | find("[0-9]+", "abc 123") + | ^^^^^^ + "#); + } } diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 515eeff6bb..43aec42518 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -2937,6 +2937,26 @@ pub fn builtin_scalar_kinds( (collect(model.db(), &env, ty, exact, &mut kinds) && !kinds.is_empty()).then_some(kinds) } +/// whether `parameter` of the function `definition` defines is the regular expression of one of +/// python's `re` functions — the `pattern` of `re.compile`, `re.match`, `re.sub` and the others +/// that compile one — so that a string handed to it is written in python's regular expression +/// syntax +pub fn is_re_pattern_parameter<'db>( + db: &'db dyn Db, + definition: Definition<'db>, + parameter: &str, +) -> bool { + let DefinitionKind::Function(function) = definition.kind(db) else { + return false; + }; + let module = parsed_module(db, definition.python_file(db)).load(db); + parameter == "pattern" + && crate::types::regex::RegexCall::from_name(function.node(&module).name.as_str()).is_some() + && ty_module_resolver::file_to_module(db, definition.program_file(db).resolver_file(db)) + .and_then(|module| module.known(db)) + == Some(ty_module_resolver::KnownModule::Re) +} + /// Whether `expr` evaluates to a module, so that reading an attribute of it looks the name up /// in the module's namespace and runs nothing. pub fn is_module_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { diff --git a/docs/basedpython/features/language-injection.md b/docs/basedpython/features/language-injection.md index 10d343a159..7108474a26 100644 --- a/docs/basedpython/features/language-injection.md +++ b/docs/basedpython/features/language-injection.md @@ -14,6 +14,34 @@ and nothing in basedpython interprets it: your editor matches it against the languages it has, so `sql`, `html`, `json` and `basedpython` all work as well as your editor supports them +the marker can trail the statement instead, at the end of its last line: + +```by +query = "select name from users" # language=sql +``` + +a comment trailing one statement is that statement's alone — the statement on the +next line is not marked by it + +## a regular expression is one already + +what python's `re` functions take as their pattern — `re.compile`, `re.match`, +`re.search`, `re.sub`, `re.split` and the rest — is written in python's regular +expression syntax, and is reported as `PythonRegExp` with nothing written to say so: + +```by +import re + +word = re.compile(r"(?P\w+)") # PythonRegExp +``` + +which function is called is resolved, not read off the spelling, so +`from re import sub as replace` is found as well, and a `re.compile` of some other +module is not. a pattern handed in through a parameter that passes it straight on +travels the way [a declared language does](#the-language-travels-to-the-call). +`PythonRegExp` is what PyCharm calls python's dialect of regular expressions; an +editor with only a generic regular expression language reads it as that + ## a parameter can say it instead a function that takes source code says so once, on the parameter, rather than From 1afae1fcde9f9f5fad7747285e64eed59d304b26 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:15:49 +1000 Subject: [PATCH 033/110] place an import a completion, quick fix, paste or inlay hint adds where buff's isort configuration sorts it, in its section of the file's import block, in order among the section's imports and with the blank lines and heading isort writes around a new section, and a member among an existing from import's members, resolving the configuration as buff does, rather than at the top of the block --- Cargo.lock | 3 + crates/ruff_linter/src/rules/isort/mod.rs | 1 + .../ruff_linter/src/rules/isort/placement.rs | 360 +++++++++ crates/ruff_linter/src/rules/isort/sorting.rs | 35 + crates/ruff_python_importer/src/lib.rs | 2 + crates/ruff_python_importer/src/order.rs | 62 ++ crates/ty_ide/Cargo.toml | 3 + crates/ty_ide/src/code_action.rs | 21 +- crates/ty_ide/src/completion.rs | 13 +- crates/ty_ide/src/import_order.rs | 203 +++++ crates/ty_ide/src/importer.rs | 195 ++++- crates/ty_ide/src/inlay_hints.rs | 67 +- crates/ty_ide/src/lib.rs | 1 + crates/ty_ide/src/override_members.rs | 4 +- crates/ty_ide/src/pasted_imports.rs | 23 +- .../ty_ide/src/refactor/return_annotation.rs | 2 +- crates/ty_ide/src/test_links.rs | 7 +- crates/ty_python_semantic/src/importer.rs | 749 +++++++++++++++++- crates/ty_server/tests/e2e/code_actions.rs | 101 +++ crates/ty_server/tests/e2e/completions.rs | 38 +- crates/ty_server/tests/e2e/inlay_hints.rs | 6 +- .../ty_server/tests/e2e/override_members.rs | 9 +- crates/ty_server/tests/e2e/pasted_imports.rs | 4 +- ...n_existing_import_undefined_decorator.snap | 10 +- crates/ty_server/tests/e2e/test_subjects.rs | 5 +- docs/basedpython/features/editor.md | 27 + 26 files changed, 1828 insertions(+), 123 deletions(-) create mode 100644 crates/ruff_linter/src/rules/isort/placement.rs create mode 100644 crates/ruff_python_importer/src/order.rs create mode 100644 crates/ty_ide/src/import_order.rs diff --git a/Cargo.lock b/Cargo.lock index 32e71bca83..49fd4eb10c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4978,9 +4978,11 @@ dependencies = [ "ruff_db", "ruff_diagnostics", "ruff_index", + "ruff_linter", "ruff_memory_usage", "ruff_python_ast", "ruff_python_codegen", + "ruff_python_importer", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", @@ -4988,6 +4990,7 @@ dependencies = [ "ruff_ranged_value", "ruff_source_file", "ruff_text_size", + "ruff_workspace", "rustc-hash", "salsa", "smallvec", diff --git a/crates/ruff_linter/src/rules/isort/mod.rs b/crates/ruff_linter/src/rules/isort/mod.rs index c0619e396a..cf56ec8a20 100644 --- a/crates/ruff_linter/src/rules/isort/mod.rs +++ b/crates/ruff_linter/src/rules/isort/mod.rs @@ -30,6 +30,7 @@ mod format; mod helpers; mod normalize; mod order; +pub mod placement; pub(crate) mod rules; pub mod settings; mod sorting; diff --git a/crates/ruff_linter/src/rules/isort/placement.rs b/crates/ruff_linter/src/rules/isort/placement.rs new file mode 100644 index 0000000000..70f0b1c8c4 --- /dev/null +++ b/crates/ruff_linter/src/rules/isort/placement.rs @@ -0,0 +1,360 @@ +//! Where isort puts one import among the imports of a block it has already sorted. +//! +//! An editor that adds an import (an auto-import completion, a quick fix for an unresolved +//! name) inserts a single statement, or a single member into an existing `from` import, and +//! leaves every other line alone. To leave the block as `I001` would write it, the statement +//! has to land in its section, in the order isort sorts that section, separated from its +//! neighbours by the blank lines isort puts there. [`ImportOrder`] answers those questions +//! with the same keys, sections and settings [`super::format_imports`] sorts with, so the two +//! cannot disagree. + +use std::cmp::Ordering; +use std::path::PathBuf; + +use ruff_python_ast::PythonVersion; +pub use ruff_python_importer::SortedImport; + +use super::categorize::{ImportSection, ImportType, categorize}; +use super::settings::Settings; +use super::sorting::{ImportStyle, MemberKey, ModuleKey}; +use crate::package::PackageRoot; + +/// The import style isort groups `import` by within a section. +fn style(import: &SortedImport) -> ImportStyle { + match import { + SortedImport::Straight { .. } => ImportStyle::Straight, + SortedImport::From { .. } => ImportStyle::From, + } +} + +/// Where an import statement sorts in its block: its section, its group within the section +/// (plain or `from`, eager or lazy) and its key within the group. Positions compare as isort +/// orders the statements. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ImportPosition<'a> { + section: usize, + group: u8, + key: ModuleKey<'a>, + style: ImportStyle, +} + +impl ImportPosition<'_> { + /// The index of the statement's section in the configured section order. + pub fn section(&self) -> usize { + self.section + } + + /// The position, owning the names it compares. + pub fn into_owned(self) -> ImportPosition<'static> { + ImportPosition { + section: self.section, + group: self.group, + key: self.key.into_owned(), + style: self.style, + } + } +} + +/// The ordering isort applies to the imports of one file. +#[derive(Debug, Clone, Copy)] +pub struct ImportOrder<'a> { + settings: &'a Settings, + src: &'a [PathBuf], + package: Option>, + target_version: PythonVersion, +} + +impl<'a> ImportOrder<'a> { + /// The ordering for a file in `package` (as `detect_package_root` finds it), under the + /// project's isort `settings`, its `src` roots and the Python version it targets. + pub fn new( + settings: &'a Settings, + src: &'a [PathBuf], + package: Option>, + target_version: PythonVersion, + ) -> Self { + Self { + settings, + src, + package, + target_version, + } + } + + /// Where `import` sorts. + pub fn position<'b>(&self, import: &SortedImport<'b>) -> ImportPosition<'b> { + let (module_name, is_relative) = match import { + SortedImport::Straight { name, .. } => ((*name).to_string(), false), + SortedImport::From { module, level, .. } => ( + ruff_python_ast::helpers::format_import_from(*level, *module).into_owned(), + *level > 0, + ), + }; + let section = categorize( + &module_name, + is_relative, + self.src, + self.package, + self.settings.detect_same_package, + &self.settings.known_modules, + self.target_version, + self.settings.no_sections, + &self.settings.section_order, + &self.settings.default_section, + ); + let is_future = matches!(section, ImportSection::Known(ImportType::Future)); + let section = self + .settings + .section_order + .iter() + .position(|candidate| candidate == section) + .unwrap_or(self.settings.section_order.len()); + + let style = style(import); + let key = match import { + SortedImport::Straight { name, asname, .. } => { + ModuleKey::from_module(Some(name), *asname, 0, None, style, self.settings) + } + SortedImport::From { + module, + level, + first_member, + .. + } => ModuleKey::from_module(*module, None, *level, *first_member, style, self.settings), + }; + + // Mirrors `order_imports`: the `__future__` section always puts its `from` imports + // first, `force-sort-within-sections` mixes the two kinds, and otherwise each kind is + // its own run, plain imports first unless `from-first`. Lazy imports follow the eager + // ones in every case. + let from_first = is_future || self.settings.from_first; + let kind = if !is_future && self.settings.force_sort_within_sections { + 0 + } else { + match (style, from_first) { + (ImportStyle::Straight, false) | (ImportStyle::From, true) => 0, + (ImportStyle::From, false) | (ImportStyle::Straight, true) => 1, + } + }; + let group = u8::from(import.is_lazy()) * 2 + kind; + + ImportPosition { + section, + group, + key, + style, + } + } + + /// How many blank lines isort writes between a statement at `before` and the statement at + /// `after` that follows it. + pub fn blank_lines_between(&self, before: &ImportPosition, after: &ImportPosition) -> usize { + if before.section != after.section { + // `format_import_block` separates two sections with a blank line unless every + // section from the one after `before` up to `after`'s is listed in + // `no-lines-before`. + let separated = self + .settings + .section_order + .iter() + .take(after.section + 1) + .skip(before.section + 1) + .any(|section| !self.settings.no_lines_before.contains(section)); + return usize::from(separated); + } + if self.settings.force_sort_within_sections || self.settings.lines_between_types == 0 { + return 0; + } + let first_kind = if self.settings.from_first { + ImportStyle::From + } else { + ImportStyle::Straight + }; + if before.style == first_kind && after.style != first_kind { + self.settings.lines_between_types + } else { + 0 + } + } + + /// The comment isort writes above the first import of the section at `position`, if one is + /// configured. + pub fn section_heading(&self, position: &ImportPosition) -> Option<&'a str> { + let section = self.settings.section_order.get(position.section)?; + self.settings + .import_headings + .get(section) + .map(String::as_str) + } + + /// How isort orders two members of one `from` import. + pub fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { + MemberKey::from_member(a.0, a.1, self.settings).cmp(&MemberKey::from_member( + b.0, + b.1, + self.settings, + )) + } + + /// Whether isort writes `member` (imported `as asname`, if given) from `module` in a `from` + /// import of its own rather than among the members of an existing one, under + /// `force-single-line` or, for an aliased member, without `combine-as-imports`. + pub fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + (self.settings.force_single_line + && module.is_none_or(|module| !self.settings.single_line_exclusions.contains(module))) + || (asname.is_some() && !self.settings.combine_as_imports) + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::PythonVersion; + use rustc_hash::{FxHashMap, FxHashSet}; + + use super::{ImportOrder, SortedImport}; + use crate::rules::isort::categorize::{ImportSection, ImportType, KnownModules}; + use crate::rules::isort::settings::Settings; + use crate::settings::types::IdentifierPattern; + + fn straight(name: &str) -> SortedImport<'_> { + SortedImport::Straight { + name, + asname: None, + is_lazy: false, + } + } + + fn from<'a>(module: &'a str, member: &'a str) -> SortedImport<'a> { + SortedImport::From { + module: Some(module), + level: 0, + first_member: Some((member, None)), + is_lazy: false, + } + } + + /// Sorts `imports` by position and returns them as statements. + fn sorted(order: &ImportOrder, imports: &[SortedImport]) -> Vec { + let mut imports: Vec<_> = imports + .iter() + .map(|import| (order.position(import), import)) + .collect(); + imports.sort_by(|a, b| a.0.cmp(&b.0)); + imports + .into_iter() + .map(|(_, import)| match import { + SortedImport::Straight { name, .. } => format!("import {name}"), + SortedImport::From { + module, + first_member, + .. + } => format!( + "from {} import {}", + module.unwrap_or_default(), + first_member.map(|(name, _)| name).unwrap_or_default() + ), + }) + .collect() + } + + #[test] + fn sections_then_plain_before_from() { + let settings = Settings::default(); + let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + assert_eq!( + sorted( + &order, + &[ + from("attrs", "define"), + straight("sys"), + from("collections", "OrderedDict"), + straight("attr"), + from("__future__", "annotations"), + straight("json"), + ] + ), + [ + "from __future__ import annotations", + "import json", + "import sys", + "from collections import OrderedDict", + "import attr", + "from attrs import define", + ] + ); + } + + #[test] + fn force_sort_within_sections_mixes_kinds() { + let settings = Settings { + force_sort_within_sections: true, + ..Settings::default() + }; + let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + assert_eq!( + sorted( + &order, + &[ + straight("sys"), + from("collections", "OrderedDict"), + straight("json") + ] + ), + [ + "from collections import OrderedDict", + "import json", + "import sys" + ] + ); + } + + #[test] + fn blank_lines() { + let settings = Settings { + lines_between_types: 1, + known_modules: KnownModules::new( + vec![IdentifierPattern::new("pkg").unwrap()], + vec![], + vec![], + vec![], + FxHashMap::default(), + ), + no_lines_before: FxHashSet::from_iter([ImportSection::Known(ImportType::FirstParty)]), + ..Settings::default() + }; + let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + let json = order.position(&straight("json")); + let collections = order.position(&from("collections", "OrderedDict")); + let attr = order.position(&straight("attr")); + let pkg = order.position(&straight("pkg")); + // a plain import and a `from` import of one section, under `lines-between-types` + assert_eq!(order.blank_lines_between(&json, &collections), 1); + // two sections + assert_eq!(order.blank_lines_between(&collections, &attr), 1); + // a section listed in `no-lines-before` + assert_eq!(order.blank_lines_between(&attr, &pkg), 0); + // past a section that is absent but separated + assert_eq!(order.blank_lines_between(&json, &pkg), 1); + } + + #[test] + fn members_by_type() { + let settings = Settings::default(); + let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + assert!( + order + .compare_members(("Counter", None), ("OrderedDict", None)) + .is_lt() + ); + assert!( + order + .compare_members(("OrderedDict", None), ("defaultdict", None)) + .is_lt() + ); + assert!( + order + .compare_members(("CONST", None), ("Class", None)) + .is_lt() + ); + } +} diff --git a/crates/ruff_linter/src/rules/isort/sorting.rs b/crates/ruff_linter/src/rules/isort/sorting.rs index 0db5d80423..df5a4bedea 100644 --- a/crates/ruff_linter/src/rules/isort/sorting.rs +++ b/crates/ruff_linter/src/rules/isort/sorting.rs @@ -53,6 +53,12 @@ impl PartialOrd for NatOrdStr<'_> { } } +impl NatOrdStr<'_> { + pub(crate) fn into_owned(self) -> NatOrdStr<'static> { + NatOrdStr(Cow::Owned(self.0.into_owned())) + } +} + impl<'a> From<&'a str> for NatOrdStr<'a> { fn from(s: &'a str) -> Self { NatOrdStr(Cow::Borrowed(s)) @@ -143,6 +149,21 @@ impl<'a> ModuleKey<'a> { } } +impl ModuleKey<'_> { + /// The key, owning the names it compares. + pub(crate) fn into_owned(self) -> ModuleKey<'static> { + ModuleKey { + force_to_top: self.force_to_top, + maybe_length: self.maybe_length, + distance: self.distance, + maybe_lowercase_name: self.maybe_lowercase_name.map(NatOrdStr::into_owned), + module_name: self.module_name.map(NatOrdStr::into_owned), + first_alias: self.first_alias.map(MemberKey::into_owned), + asname: self.asname.map(NatOrdStr::into_owned), + } + } +} + /// A comparable key to capture the desired sorting order for an imported member (e.g., `bar` in /// `from foo import bar`). #[derive(Debug, PartialOrd, Ord, PartialEq, Eq)] @@ -180,6 +201,20 @@ impl<'a> MemberKey<'a> { } } +impl MemberKey<'_> { + /// The key, owning the names it compares. + pub(crate) fn into_owned(self) -> MemberKey<'static> { + MemberKey { + not_star_import: self.not_star_import, + member_type: self.member_type, + maybe_length: self.maybe_length, + maybe_lowercase_name: self.maybe_lowercase_name.map(NatOrdStr::into_owned), + module_name: self.module_name.into_owned(), + asname: self.asname.map(NatOrdStr::into_owned), + } + } +} + /// Lowercase the given string, if it contains any uppercase characters. fn maybe_lowercase(name: &str) -> Cow<'_, str> { if name.chars().all(char::is_lowercase) { diff --git a/crates/ruff_python_importer/src/lib.rs b/crates/ruff_python_importer/src/lib.rs index 557d9f5013..154e690c22 100644 --- a/crates/ruff_python_importer/src/lib.rs +++ b/crates/ruff_python_importer/src/lib.rs @@ -3,5 +3,7 @@ Low-level helpers for manipulating Python import statements. */ pub use self::insertion::Insertion; +pub use self::order::{ImportOrdering, SortedImport}; mod insertion; +mod order; diff --git a/crates/ruff_python_importer/src/order.rs b/crates/ruff_python_importer/src/order.rs new file mode 100644 index 0000000000..b94aa2f129 --- /dev/null +++ b/crates/ruff_python_importer/src/order.rs @@ -0,0 +1,62 @@ +//! How an import sorter orders the imports of a file, for an importer to add an import where the +//! sorter would put it. + +use std::cmp::Ordering; + +/// One import statement, as an import sorter reads it to sort it. +#[derive(Debug, Clone, Copy)] +pub enum SortedImport<'a> { + /// `import name` or `import name as asname`. isort sorts a statement importing several + /// modules by its first. + Straight { + name: &'a str, + asname: Option<&'a str>, + is_lazy: bool, + }, + /// `from module import member`, with `level` leading dots. isort sorts it by its module and + /// then by its first member. + From { + module: Option<&'a str>, + level: u32, + first_member: Option<(&'a str, Option<&'a str>)>, + is_lazy: bool, + }, +} + +impl SortedImport<'_> { + pub fn is_lazy(&self) -> bool { + match self { + Self::Straight { is_lazy, .. } | Self::From { is_lazy, .. } => *is_lazy, + } + } +} + +/// The order a sorter puts the imports of one file in: which section each import belongs to, +/// where it sorts within it, and the blank lines and headings written between them. +pub trait ImportOrdering { + /// Where an import sorts in its block. Positions compare as the sorter orders the + /// statements. + type Position: Ord; + + /// Where `import` sorts. + fn position(&self, import: &SortedImport<'_>) -> Self::Position; + + /// The index of the section an import at `position` belongs to, in the sorter's section + /// order. + fn section(&self, position: &Self::Position) -> usize; + + /// How many blank lines the sorter writes between a statement at `before` and the statement + /// at `after` that follows it. + fn blank_lines_between(&self, before: &Self::Position, after: &Self::Position) -> usize; + + /// The comment the sorter writes above the first import of the section at `position`, if one + /// is configured. + fn section_heading(&self, position: &Self::Position) -> Option<&str>; + + /// How the sorter orders two members of one `from` import, each a name and its alias. + fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering; + + /// Whether the sorter writes `member` (imported `as asname`, if given) from `module` in a + /// `from` import of its own rather than among the members of an existing one. + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; +} diff --git a/crates/ty_ide/Cargo.toml b/crates/ty_ide/Cargo.toml index 6114e1b193..723e45ebd4 100644 --- a/crates/ty_ide/Cargo.toml +++ b/crates/ty_ide/Cargo.toml @@ -18,15 +18,18 @@ bitflags = { workspace = true } ruff_db = { workspace = true } ruff_diagnostics = { workspace = true } ruff_index = { workspace = true } +ruff_linter = { workspace = true } ruff_memory_usage = { workspace = true } ruff_python_ast = { workspace = true } ruff_python_codegen = { workspace = true } +ruff_python_importer = { workspace = true } ruff_python_literal = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } +ruff_workspace = { workspace = true } toml_edit = { workspace = true } ty_module_resolver = { workspace = true } ty_project = { workspace = true } diff --git a/crates/ty_ide/src/code_action.rs b/crates/ty_ide/src/code_action.rs index 6042a04b09..8a23e207df 100644 --- a/crates/ty_ide/src/code_action.rs +++ b/crates/ty_ide/src/code_action.rs @@ -1161,8 +1161,9 @@ mod tests { | ^^^^^^^^^^ help: This is a preferred code action | - 1 + from warnings import deprecated - 2 | + 2 | import warnings + 3 + from warnings import deprecated + 4 | | info[code-action]: import typing_extensions.deprecated @@ -1172,8 +1173,10 @@ mod tests { | ^^^^^^^^^^ help: This is a preferred code action | - 1 + from typing_extensions import deprecated - 2 | + 3 | + 4 + from typing_extensions import deprecated + 5 + + 6 | @deprecated("do not use") | info[code-action]: qualify warnings.deprecated @@ -1280,8 +1283,9 @@ mod tests { help: This is a preferred code action help: This is the only import that resolves the name | - 1 + from importlib.abc import ExecutionLoader - 2 | + 2 | import importlib + 3 + from importlib.abc import ExecutionLoader + 4 | ExecutionLoader | info[code-action]: Ignore 'unresolved-reference' for this line @@ -1326,8 +1330,9 @@ mod tests { help: This is a preferred code action help: This is the only import that resolves the name | - 1 + from importlib.abc import ExecutionLoader - 2 | + 2 | import importlib.abc + 3 + from importlib.abc import ExecutionLoader + 4 | ExecutionLoader | info[code-action]: qualify importlib.abc.ExecutionLoader diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 5eb3aaf30c..9d073fa8f3 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -44,6 +44,7 @@ use crate::common_aliases; use crate::django_template::django_string_completions; use crate::docstring::Docstring; use crate::goto::Definitions; +use crate::import_order; use crate::symbols::QueryPattern; use crate::{Db, all_symbols, signature_help}; use ruff_db::files::File; @@ -155,7 +156,7 @@ pub fn completion<'db>( &mut completions, ); if settings.auto_import { - let importer = Importer::new(db, program_file, &parsed); + let importer = import_order::importer(db, program_file, &parsed); let members = importer.members_in_scope_at(expr.into(), expr.start()); add_aliased_module_completions( db, @@ -251,7 +252,7 @@ pub fn completion<'db>( // an import is only ever offered for something the user has begun to name: a // query that matches everything would drag in every symbol of every module if settings.auto_import && !completions.query.will_match_everything() { - let importer = Importer::new(db, program_file, &parsed); + let importer = import_order::importer(db, program_file, &parsed); let members = importer.members_in_scope_at(scoped.node, scoped.node.start()); add_alias_completions( db, @@ -2931,7 +2932,7 @@ pub(crate) fn unresolved_fixes<'db>( let query = UserQuery::exactly(symbol); let ctx = CollectionContext::none(); - let importer = Importer::new(db, file, parsed); + let importer = import_order::importer(db, file, parsed); let members = importer.members_in_scope_at(node, node.start()); // Request imports we could add to put the symbol in scope @@ -15436,9 +15437,11 @@ TypedDi @" TypedDict :: , TypedDict is_typeddict() :: , is_typeddict - TypedDict :: from typing_extensions import TypedDict + TypedDict :: + from typing_extensions import TypedDict - is_typeddict() :: from typing_extensions import is_typeddict + is_typeddict() :: + from typing_extensions import is_typeddict ", ); } diff --git a/crates/ty_ide/src/import_order.rs b/crates/ty_ide/src/import_order.rs new file mode 100644 index 0000000000..93720bf87e --- /dev/null +++ b/crates/ty_ide/src/import_order.rs @@ -0,0 +1,203 @@ +//! The isort configuration buff sorts a file's imports with. +//! +//! An import added to a file is placed where `buff check --select I001 --fix` would put it, so +//! adding one never leaves a block buff then reports as unsorted. buff reads its configuration +//! the way ruff does: from the closest `ruff.toml`, `.ruff.toml` or `pyproject.toml` with a +//! `[tool.ruff]` table above the file, then from the user's own configuration, and otherwise its +//! defaults with the `target-version` the closest `requires-python` implies. This resolves it +//! with the same `ruff_workspace` code, so the two agree on sections, known modules and order. +//! +//! The files are read from disk, not through the database: they are not Python sources and the +//! server does not watch them. A resolved configuration is cached against the revision (the +//! modification time) of the file it was read from, so an edit to it is seen by the next import +//! added. + +use std::cmp::Ordering; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, LazyLock, Mutex}; + +use ruff_db::file_revision::FileRevision; +use ruff_db::parsed::ParsedModuleRef; +use ruff_db::source::source_text; +use ruff_db::system::{System, SystemPath}; +use ruff_linter::package::PackageRoot; +use ruff_linter::packaging::detect_package_root; +use ruff_linter::rules::isort::placement::{ImportOrder, ImportPosition}; +use ruff_python_ast::PythonVersion; +use ruff_python_importer::{ImportOrdering, SortedImport}; +use ruff_workspace::configuration::Configuration; +use ruff_workspace::pyproject::{ + find_fallback_target_version, find_settings_toml, find_user_settings_toml, +}; +use ruff_workspace::resolver::{ + ConfigurationOrigin, ConfigurationTransformer, resolve_root_settings, +}; +use rustc_hash::FxHashMap; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_semantic::importer::Importer; + +/// An importer for `file` that adds each import where buff's isort configuration sorts it, read +/// the first time an import is placed. +pub(crate) fn importer<'a>( + db: &'a dyn Db, + file: ProgramFile<'a>, + parsed: &'a ParsedModuleRef, +) -> Importer<'a> { + Importer::new(db, file, parsed).sorted_by(move || IsortConfig::for_program_file(db, file)) +} + +/// The isort settings that apply to one file, and what isort needs to know about the file to +/// categorize its imports. +#[derive(Debug)] +pub(crate) struct IsortConfig { + settings: Arc, + package_root: Option, + target_version: PythonVersion, +} + +impl IsortConfig { + /// The configuration that applies to `file`, or `None` for a file that is not on disk or is + /// a notebook. + fn for_program_file(db: &dyn Db, file: ProgramFile<'_>) -> Option { + if source_text(db, file.file(db)).is_notebook() { + return None; + } + let path = file.file(db).path(db).as_system_path()?; + Some(Self::for_file( + db.system(), + path.as_std_path(), + db.project().root(db).as_std_path(), + )) + } + + /// The configuration that applies to the file at `path`. `project_root` stands in for the + /// directory buff resolves `src` against when no configuration file is found. + pub(crate) fn for_file(system: &dyn System, path: &Path, project_root: &Path) -> Self { + let directory = path.parent().unwrap_or(path); + let settings = settings_for(system, directory, project_root); + let package_root = detect_package_root(directory, &settings.linter.namespace_packages) + .map(Path::to_path_buf); + let target_version = settings + .linter + .resolve_target_version(path) + .linter_version(); + Self { + settings, + package_root, + target_version, + } + } + + pub(crate) fn order(&self) -> ImportOrder<'_> { + ImportOrder::new( + &self.settings.linter.isort, + &self.settings.linter.src, + self.package_root.as_deref().map(PackageRoot::root), + self.target_version, + ) + } +} + +impl ImportOrdering for IsortConfig { + type Position = ImportPosition<'static>; + + fn position(&self, import: &SortedImport<'_>) -> Self::Position { + self.order().position(import).into_owned() + } + + fn section(&self, position: &Self::Position) -> usize { + position.section() + } + + fn blank_lines_between(&self, before: &Self::Position, after: &Self::Position) -> usize { + self.order().blank_lines_between(before, after) + } + + fn section_heading(&self, position: &Self::Position) -> Option<&str> { + self.order().section_heading(position) + } + + fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { + self.order().compare_members(a, b) + } + + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + self.order().imports_member_alone(module, asname) + } +} + +struct Unchanged; + +impl ConfigurationTransformer for Unchanged { + fn transform(&self, config: Configuration) -> Configuration { + config + } +} + +/// A resolved configuration file and the revision it was resolved at. +struct Resolved { + revision: Option, + settings: Arc, +} + +static RESOLVED: LazyLock>> = LazyLock::new(Mutex::default); + +fn settings_for( + system: &dyn System, + directory: &Path, + project_root: &Path, +) -> Arc { + let project = find_settings_toml(directory).ok().flatten(); + if let Some(path) = &project + && let Some(settings) = resolved(system, path, ConfigurationOrigin::Ancestor) + { + return settings; + } + // the user's own configuration is left out of tests, which would otherwise depend on the + // machine they run on + if !cfg!(test) + && let Some(path) = find_user_settings_toml() + && let Some(settings) = resolved(system, &path, ConfigurationOrigin::UserSettings) + { + return settings; + } + let configuration = Configuration { + target_version: find_fallback_target_version(directory).map(Into::into), + ..Configuration::default() + }; + Arc::new( + configuration + .into_settings(project_root) + .unwrap_or_default(), + ) +} + +/// The settings resolved from the configuration file at `path`, or `None` when it cannot be +/// read or is invalid, as buff then falls back on the next configuration too. +fn resolved( + system: &dyn System, + path: &Path, + origin: ConfigurationOrigin, +) -> Option> { + let revision = SystemPath::from_std_path(path) + .and_then(|path| system.path_metadata(path).ok()) + .map(|metadata| metadata.revision()); + let mut cache = RESOLVED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(resolved) = cache.get(path) + && resolved.revision == revision + { + return Some(resolved.settings.clone()); + } + let settings = Arc::new(resolve_root_settings(path, &Unchanged, origin).ok()?); + cache.insert( + path.to_path_buf(), + Resolved { + revision, + settings: settings.clone(), + }, + ); + Some(settings) +} diff --git a/crates/ty_ide/src/importer.rs b/crates/ty_ide/src/importer.rs index 889d5784f9..b6522e71f7 100644 --- a/crates/ty_ide/src/importer.rs +++ b/crates/ty_ide/src/importer.rs @@ -4,6 +4,7 @@ mod tests { use insta::assert_snapshot; + use crate::import_order; use crate::tests::{CursorTest, cursor_test}; use ruff_python_ast::find_node::covering_node; use ruff_text_size::{Ranged, TextRange}; @@ -57,7 +58,7 @@ mod tests { } fn importer(&self) -> Importer<'_> { - Importer::new( + import_order::importer( &self.db, ProgramFile::new( &self.db, @@ -69,6 +70,171 @@ mod tests { } } + /// A new import goes where buff's isort would sort it: into its section of the block, in + /// order among the section's imports, rather than at the top of the block. + #[test] + fn sorted_into_its_section() { + let source = r#" +"""Module docstring.""" + +from __future__ import annotations + +import json +import sys +from collections import OrderedDict + +import attr + + +"#; + let test = cursor_test(source); + assert_snapshot!(test.module("math"), @r#" + """Module docstring.""" + + from __future__ import annotations + + import json + import math + import sys + from collections import OrderedDict + + import attr + + math + "#); + assert_snapshot!(test.import_from("pathlib", "Path"), @r#" + """Module docstring.""" + + from __future__ import annotations + + import json + import sys + from collections import OrderedDict + from pathlib import Path + + import attr + + Path + "#); + assert_snapshot!(test.module("attrs"), @r#" + """Module docstring.""" + + from __future__ import annotations + + import json + import sys + from collections import OrderedDict + + import attr + import attrs + + attrs + "#); + } + + /// A member added to a `from` import is listed where isort orders it: constants, then + /// classes, then everything else, each alphabetically. + #[test] + fn member_sorted_among_members() { + let test = cursor_test( + "\ +from collections import OrderedDict, deque + +", + ); + assert_snapshot!(test.import("collections", "Counter"), @" + from collections import Counter, OrderedDict, deque + Counter + "); + assert_snapshot!(test.import("collections", "defaultdict"), @" + from collections import OrderedDict, defaultdict, deque + defaultdict + "); + assert_snapshot!(test.import("collections", "namedtuple"), @" + from collections import OrderedDict, deque, namedtuple + namedtuple + "); + } + + /// One per line in an import written one per line. + #[test] + fn member_sorted_among_members_one_per_line() { + let test = cursor_test( + "\ +from collections import ( + OrderedDict, + deque, +) + +", + ); + assert_snapshot!(test.import("collections", "defaultdict"), @" + from collections import ( + OrderedDict, + defaultdict, + deque, + ) + defaultdict + "); + } + + /// An import of a section the block has none of yet is set apart by a blank line on each + /// side, as isort separates sections. + #[test] + fn new_section_is_set_apart() { + let test = cursor_test( + "\ +import json + + +", + ); + assert_snapshot!(test.module("attrs"), @" + import json + + import attrs + + attrs + "); + + let test = cursor_test( + "\ +import attr + + +", + ); + assert_snapshot!(test.module("abc"), @" + import abc + + import attr + + abc + "); + } + + /// Comments above an import stay with it, as isort keeps them. + #[test] + fn placed_above_the_comments_of_the_import_after() { + let test = cursor_test( + "\ +import json +# the system +import sys + + +", + ); + assert_snapshot!(test.module("os"), @" + import json + import os + # the system + import sys + + os + "); + } + #[test] fn empty_source_qualified() { let test = cursor_test(""); @@ -340,7 +506,7 @@ from collections import ( .build(); assert_snapshot!( test.import("package.foo", "Bar"), @" - from ...foo import Foo, Bar + from ...foo import Bar, Foo Bar "); } @@ -548,6 +714,7 @@ from __future__ import annotations assert_snapshot!( test.import("typing", "TypeVar"), @" from __future__ import annotations + import typing typing.TypeVar @@ -569,6 +736,7 @@ from __future__ import annotations "This is a module level docstring" from __future__ import annotations + import typing typing.TypeVar @@ -587,9 +755,10 @@ lazy from __future__ import annotations ); assert_snapshot!( test.import("typing", "TypeVar"), @" - import typing lazy from __future__ import annotations + import typing + typing.TypeVar "); } @@ -766,6 +935,7 @@ import defaultdict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + import defaultdict (collections.defaultdict) @@ -773,6 +943,7 @@ import defaultdict assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + import defaultdict (collections.defaultdict) @@ -792,6 +963,7 @@ from foo import collections assert_snapshot!( test.import("collections", "defaultdict"), @" from collections import defaultdict + from foo import collections (defaultdict) @@ -799,6 +971,7 @@ from foo import collections assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + from foo import collections (defaultdict) @@ -818,6 +991,7 @@ from othermodule import defaultdict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + from othermodule import defaultdict (collections.defaultdict) @@ -825,6 +999,7 @@ from othermodule import defaultdict assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + from othermodule import defaultdict (collections.defaultdict) @@ -844,6 +1019,7 @@ import defaultdict as ddict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + import defaultdict as ddict (collections.defaultdict) @@ -851,6 +1027,7 @@ import defaultdict as ddict assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + import defaultdict as ddict (defaultdict) @@ -870,6 +1047,7 @@ from othermodule import something as defaultdict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + from othermodule import something as defaultdict (collections.defaultdict) @@ -877,6 +1055,7 @@ from othermodule import something as defaultdict assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + from othermodule import something as defaultdict (collections.defaultdict) @@ -896,6 +1075,7 @@ import defaultdict as ddict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + import defaultdict as ddict (collections.defaultdict) @@ -903,6 +1083,7 @@ import defaultdict as ddict assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + import defaultdict as ddict (defaultdict) @@ -922,6 +1103,7 @@ from foo import defaultdict as ddict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + from foo import defaultdict as ddict (collections.defaultdict) @@ -929,6 +1111,7 @@ from foo import defaultdict as ddict assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + from foo import defaultdict as ddict (defaultdict) @@ -962,9 +1145,9 @@ import numpy as np "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" - from collections import defaultdict import json import re + from collections import defaultdict from whenever import ZonedDateTime import numpy as np @@ -993,10 +1176,10 @@ import numpy as np assert_snapshot!( test.import("foo", "Bar"), @" - import foo import json import re + import foo from whenever import ZonedDateTime import numpy as np @@ -1004,10 +1187,10 @@ import numpy as np "); assert_snapshot!( test.import_from("foo", "Bar"), @" - from foo import Bar import json import re + from foo import Bar from whenever import ZonedDateTime import numpy as np diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 60139f9ca3..0a9ed4a903 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -5,6 +5,7 @@ use itertools::{Either, Itertools}; use rustc_hash::FxHashMap; use strum_macros::EnumIter; +use crate::import_order; use crate::{Db, HasNavigationTargets, NavigationTarget}; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; @@ -920,7 +921,7 @@ fn inlay_hints_inner( ) -> Vec { let ast = parsed_module(db, file.python_file(db)).load(db); let source = source_text(db, file.file(db)); - let importer = Importer::new(db, file, &ast); + let importer = import_order::importer(db, file, &ast); let mut visitor = InlayHintVisitor::new(db, file, importer, source.as_str(), range, settings); @@ -1546,7 +1547,7 @@ pub(crate) fn annotation_for_type<'db>( type_params_in_scope: &[(BoundTypeVarInstance<'db>, Name)], ) -> Option<(String, Vec)> { let ast = parsed_module(db, file.python_file(db)).load(db); - let importer = Importer::new(db, file, &ast); + let importer = import_order::importer(db, file, &ast); // a basedpython file writes types in basedpython's own spelling with_display_for_file(db, file.file(db), || { let details = type_display_details(db, file, ty, false, type_params_in_scope); @@ -1652,7 +1653,7 @@ pub(crate) fn with_annotation_speller<'db, R>( spell: impl FnOnce(&mut AnnotationSpeller<'_, 'db>) -> R, ) -> (R, Vec) { let ast = parsed_module(db, file.python_file(db)).load(db); - let importer = Importer::new(db, file, &ast); + let importer = import_order::importer(db, file, &ast); let mut speller = AnnotationSpeller { db, file, @@ -3198,15 +3199,18 @@ impl<'a, 'db> DynamicImporter<'a, 'db> { /// Builds the text edits from all collected imports. fn text_edits(&self) -> Vec { - self.imported_members - .iter() - .filter_map(|member| self.dynamic_imports.get(member)) - .filter_map(|import_action| { - import_action.import().and_then(|edit| { - edit.content().map(|content| InlayHintTextEdit { - range: edit.range(), - new_text: content.to_string(), - }) + // placed together, so imports added at one place come out in isort's order + self.importer + .combined_edits( + self.imported_members + .iter() + .filter_map(|member| self.dynamic_imports.get(member)), + ) + .into_iter() + .filter_map(|edit| { + edit.content().map(|content| InlayHintTextEdit { + range: edit.range(), + new_text: content.to_string(), }) }) .collect() @@ -3711,7 +3715,7 @@ mod tests { | 1 | - from module import make - 2 + from module import make, Item + 2 + from module import Item, make 3 | - def create(): 4 + def create() -> Item: @@ -10813,7 +10817,7 @@ Source with applied edits: | 1 | - from module import make_unknown - 2 + from module import make_unknown, Unknown + 2 + from module import Unknown, make_unknown 3 | - def custom_unknown(): 4 + def custom_unknown() -> Unknown: @@ -11061,9 +11065,8 @@ Source with applied edits: info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | - 1 + from bar import D - 2 | - 3 | import foo + 2 | import foo + 3 + from bar import D 4 | - a = foo.C().foo() 5 + a: foo.B[foo.A[D[int, list[str | foo.A[foo.B[int]]]]]] = foo.C().foo() @@ -11213,10 +11216,10 @@ Source with applied edits: info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | - 1 + from bar import D - 2 | + 1 | - from foo import C - 3 + from foo import C, B, A + 2 + from bar import D + 3 + from foo import A, B, C 4 | - a = C().foo() 5 + a: B[A[D[int, list[str | A[B[int]]]]]] = C().foo() @@ -11362,12 +11365,14 @@ Source with applied edits: assert_snapshot!(edited_sources, @" from typing import Literal - from module import y, Y + + from module import Y, y x: tuple[Y, Literal[1], Y, Literal[1]] = (y, 1, y, 1) x = (y, 1, y, 1) from typing import Literal - from module import y, Y + + from module import Y, y x = (y, 1, y, 1) x: tuple[Y, Literal[1], Y, Literal[1]] = (y, 1, y, 1) "); @@ -11525,9 +11530,9 @@ Source with applied edits: info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | - 1 + from typing import TypeVar - 2 + from typing import Any - 3 | + 1 | + 2 + from typing import Any, TypeVar + 3 + 4 | from foo import foo 5 | - a = foo() @@ -11607,9 +11612,9 @@ Source with applied edits: info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | - 1 + import bar - 2 + import baz - 3 | + 1 | + 2 + import bar + 3 + import baz 4 | from foo import foo 5 | - a = foo() @@ -11727,9 +11732,9 @@ Source with applied edits: info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | - 1 + import bar - 2 + import baz - 3 | + 1 | + 2 + import bar + 3 + import baz 4 | from foo import foo 5 | from bar import B 6 | diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 682ddb5b0f..ade47e81d9 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -30,6 +30,7 @@ mod goto_implementation; mod goto_type_definition; mod hints; mod hover; +mod import_order; #[cfg(test)] mod importer; mod injection; diff --git a/crates/ty_ide/src/override_members.rs b/crates/ty_ide/src/override_members.rs index acdd8134b0..552857946b 100644 --- a/crates/ty_ide/src/override_members.rs +++ b/crates/ty_ide/src/override_members.rs @@ -1383,8 +1383,8 @@ class Square(Shape): ) .build(); assert_snapshot!(test.override_members(&["draw"]), @" - import shapes + import shapes from shapes import Shape def make_config() -> list[int]: @@ -1414,8 +1414,8 @@ class M(Mapping[str, int]): ) .build(); assert_snapshot!(test.override_members(&["__iter__"]), @" - import typing + import typing from collections.abc import Mapping from mylib import Iterator diff --git a/crates/ty_ide/src/pasted_imports.rs b/crates/ty_ide/src/pasted_imports.rs index f290470cec..8e8f0ba028 100644 --- a/crates/ty_ide/src/pasted_imports.rs +++ b/crates/ty_ide/src/pasted_imports.rs @@ -31,7 +31,9 @@ use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_semantic::SemanticModel; -use ty_python_semantic::importer::{ImportRequest, Importer}; +use ty_python_semantic::importer::ImportRequest; + +use crate::import_order; /// A name copied code uses, and the import that binds it. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -319,7 +321,7 @@ pub fn pasted_imports( imports: &[CopiedImport], ) -> PastedImports { let parsed = parsed_module(db, file.python_file(db)).load(db); - let importer = Importer::new(db, file, &parsed); + let importer = import_order::importer(db, file, &parsed); let node = landing_node(&parsed, range); let members = importer.members_in_scope_at(node, range.start()); let model = SemanticModel::new(db, file); @@ -327,6 +329,7 @@ pub fn pasted_imports( .map(|module| module.name(db).as_str().to_string()); let mut answer = PastedImports::default(); + let mut actions = Vec::new(); let mut seen = FxHashSet::default(); for import in imports { if !seen.insert(import.name.as_str()) @@ -344,13 +347,13 @@ pub fn pasted_imports( // an import the file can only satisfy by spelling the name another way — as // `pathlib.Path` under an `import pathlib` — would leave the pasted name as // unresolved as it was - if let Some(edit) = action.import() - && action.symbol_text() == import.name - { + if action.import().is_some() && action.symbol_text() == import.name { answer.missing.push(import.clone()); - answer.edits.push(edit.clone()); + actions.push(action); } } + // placed together, so imports added at one place come out in isort's order + answer.edits = importer.combined_edits(&actions); answer } @@ -625,11 +628,13 @@ mod tests { from pathlib import PurePosixPath as P import numbers as np --- - from shapes import SCALE - import numbers as np import math - from pathlib import PurePath, Path, PurePosixPath as P + import numbers as np + from pathlib import Path, PurePath + from pathlib import PurePosixPath as P + + from shapes import SCALE def f(): return math.pi * SCALE, Path(), P(), np diff --git a/crates/ty_ide/src/refactor/return_annotation.rs b/crates/ty_ide/src/refactor/return_annotation.rs index 39c169ea6b..b4aab4554a 100644 --- a/crates/ty_ide/src/refactor/return_annotation.rs +++ b/crates/ty_ide/src/refactor/return_annotation.rs @@ -174,7 +174,7 @@ mod tests { Add return annotation `-> Circle` --- - from shapes import make, Circle + from shapes import Circle, make def f() -> Circle: return make() diff --git a/crates/ty_ide/src/test_links.rs b/crates/ty_ide/src/test_links.rs index 39b5d3bb49..ed53c5450b 100644 --- a/crates/ty_ide/src/test_links.rs +++ b/crates/ty_ide/src/test_links.rs @@ -27,8 +27,9 @@ use ty_python_core::ProgramFile; use ty_python_core::program::Program; use ty_python_semantic::{CollectedTest, collected_pytest_tests}; +use crate::import_order; use crate::{CallHierarchyItem, Db, SymbolKind, incoming_calls, outgoing_calls}; -use ty_python_semantic::importer::{ImportRequest, Importer}; +use ty_python_semantic::importer::ImportRequest; /// a test, or a subject a test exercises #[derive(Debug, Clone, PartialEq, Eq)] @@ -506,7 +507,7 @@ fn test_directory(db: &dyn Db, file: ProgramFile<'_>) -> SystemPathBuf { fn import_edit(db: &dyn Db, file: ProgramFile<'_>, module: &str, name: &str) -> Option { let parsed: ParsedModuleRef = parsed_module(db, file.python_file(db)).load(db); let source = source_text(db, file.file(db)); - let importer = Importer::new(db, file, &parsed); + let importer = import_order::importer(db, file, &parsed); // in scope at the end of the module, which is where a new test goes let end = TextSize::of(source.as_str()); let members = importer.members_in_scope_at(parsed.syntax().into(), end); @@ -760,7 +761,7 @@ class TestShapes: .build(); assert_snapshot!(test.subject_tests(), @r#" - new test test_perimeter added in /tests/test_geometry.py importing perimeter from geometry with "from geometry import perimeter\n" + new test test_perimeter added in /tests/test_geometry.py importing perimeter from geometry with "/nfrom geometry import perimeter\n" "#); } diff --git a/crates/ty_python_semantic/src/importer.rs b/crates/ty_python_semantic/src/importer.rs index 753847e418..f661a44528 100644 --- a/crates/ty_python_semantic/src/importer.rs +++ b/crates/ty_python_semantic/src/importer.rs @@ -11,6 +11,9 @@ The main differences here are: 3. It doesn't have as many facilities as `ruff_linter`'s importer. */ +use std::cell::OnceCell; +use std::ops::Range; + use rustc_hash::FxHashMap; use ruff_db::PythonFile; @@ -25,7 +28,8 @@ use ruff_python_ast::name::Name; use ruff_python_ast::token::Tokens; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal, walk_stmt}; use ruff_python_codegen::Stylist; -use ruff_python_importer::Insertion; +use ruff_python_importer::{ImportOrdering, Insertion, SortedImport}; +use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_module_resolver::{ImportingFile, ModuleName}; use ty_python_core::ast_node_ref::AstNodeRef; @@ -49,6 +53,10 @@ pub struct Importer<'a> { stylist: &'a Stylist<'static>, /// The list of visited, top-level runtime imports in the Python AST. imports: &'a [AstNodeRef], + /// The order an import sorter puts this file's imports in, for imports to be added where it + /// would put them. `None` adds a statement at the top of the file and a member after an + /// import's last. + sorting: Option>, } impl<'a> Importer<'a> { @@ -70,9 +78,29 @@ impl<'a> Importer<'a> { source: source_text(db, file.file(db)), stylist: stylist(db, file.python_file(db)), imports, + sorting: None, } } + /// This importer, adding each import where `order` sorts it: a statement in its section of + /// the file's import block and a member among the members of a `from` import. + /// + /// `resolve` is called the first time an import is placed, as finding the order can be + /// costly; when it returns `None` the imports are added as without an order. + #[must_use] + pub fn sorted_by(mut self, resolve: F) -> Self + where + O: ImportOrdering + 'a, + F: Fn() -> Option + 'a, + { + self.sorting = Some(Box::new(LazyOrder { + resolve, + order: OnceCell::new(), + block: OnceCell::new(), + })); + self + } + /// The file's indentation unit, shared with inserted imports and diagnostic fixes. pub(crate) fn indentation(&self) -> &str { self.stylist.indentation().as_str() @@ -140,17 +168,14 @@ impl<'a> Importer<'a> { let request = request.avoid_conflicts(self.db, importing_file, members); let mut symbol_text: Box = request.symbol_text().into(); let Some(response) = self.find(importing_file, &request, members.at) else { - let insertion = if let Some(future) = self.find_last_future_import(members.at) { - Insertion::end_of_statement(future.stmt, &self.source, self.stylist) - } else { - let range = self - .source - .as_notebook() - .and_then(|notebook| notebook.cell_offsets().containing_range(members.at)); - - Insertion::start_of_file(self.parsed.suite(), &self.source, self.stylist, range) - }; - let import = insertion.into_edit(&request.to_string()); + let placement = Placement::statement(&request, members.at); + let import = self + .placed_edits(std::slice::from_ref(&placement)) + .and_then(|mut edits| edits.pop()) + .unwrap_or_else(|| { + self.statement_insertion(members.at) + .into_edit(&request.to_string()) + }); if let Some(member) = request.member && matches!(request.style, ImportStyle::Import) { @@ -159,6 +184,7 @@ impl<'a> Importer<'a> { return ImportAction { import: Some(import), symbol_text, + placement: Some(placement), }; }; @@ -170,6 +196,7 @@ impl<'a> Importer<'a> { return ImportAction { import: None, symbol_text, + placement: None, }; }; match response.kind { @@ -183,6 +210,7 @@ impl<'a> Importer<'a> { ImportAction { import: None, symbol_text, + placement: None, } } ImportResponseKind::Qualified { alias } => { @@ -190,21 +218,54 @@ impl<'a> Importer<'a> { ImportAction { import: None, symbol_text: format!("{module}.{member}").into(), + placement: None, } } - ImportResponseKind::Partial => { - let imported = request.imported_member(); - let import = if let Some(insertion) = - Insertion::existing_import(response.import.stmt, self.tokens) - { - insertion.into_edit(&imported) + ImportResponseKind::Partial(from) => { + let module = from.module.as_ref().map(ast::Identifier::as_str); + let alone = self + .sorting + .as_ref() + .is_some_and(|sorting| sorting.imports_member_alone(module, request.asname)); + // a sorter that won't list the member among others writes it in a `from` import + // of its own, so that is the statement added + let placement = if alone { + Placement::Statement { + module: module.map(str::to_string), + level: from.level, + member: Some(member.to_string()), + asname: request.asname.map(str::to_string), + at: members.at, + } } else { - Insertion::end_of_statement(response.import.stmt, &self.source, self.stylist) - .into_edit(&format!("from {} import {imported}", request.module)) + Placement::Member { + import: from.start(), + member: member.to_string(), + asname: request.asname.map(str::to_string), + } }; + let import = self + .placed_edits(std::slice::from_ref(&placement)) + .and_then(|mut edits| edits.pop()) + .unwrap_or_else(|| { + let imported = request.imported_member(); + if let Some(insertion) = + Insertion::existing_import(response.import.stmt, self.tokens) + { + insertion.into_edit(&imported) + } else { + Insertion::end_of_statement( + response.import.stmt, + &self.source, + self.stylist, + ) + .into_edit(&format!("from {} import {imported}", request.module)) + } + }); ImportAction { import: Some(import), symbol_text, + placement: Some(placement), } } } @@ -402,6 +463,426 @@ impl<'a> Importer<'a> { None } + /// The edits that add everything `actions` import at once. + /// + /// Each action's own edit is computed as if it were the only import added, so two of them + /// can land at one place in an order the file's import sorter wouldn't write, or both + /// separate themselves from the imports around them. These are placed together instead: + /// statements of one section are sorted among each other and separated as the sorter + /// separates them, two members added to one `from` import are sorted, and two members added + /// from a module the file doesn't import yet are listed by one new `from` import. + pub fn combined_edits<'b>( + &self, + actions: impl IntoIterator, + ) -> Vec { + let mut placements = Vec::new(); + let mut edits = Vec::new(); + for action in actions { + match (&action.placement, &action.import) { + (Some(placement), Some(_)) => placements.push(placement.clone()), + (None, Some(edit)) => edits.push(edit.clone()), + (_, None) => {} + } + } + if let Some(placed) = self.placed_edits(&placements) { + edits.extend(placed); + } else { + edits.extend( + placements + .iter() + .filter_map(|placement| self.unsorted_edit(placement)), + ); + } + edits + } + + /// Where a new statement goes without a sorter's order: after the last `from __future__` + /// import before `at`, or otherwise at the top of the file (or of `at`'s notebook cell). + fn statement_insertion(&self, at: TextSize) -> Insertion<'static> { + if let Some(future) = self.find_last_future_import(at) { + Insertion::end_of_statement(future.stmt, &self.source, self.stylist) + } else { + let range = self + .source + .as_notebook() + .and_then(|notebook| notebook.cell_offsets().containing_range(at)); + Insertion::start_of_file(self.parsed.suite(), &self.source, self.stylist, range) + } + } + + /// The edit that adds `placement` without a sorter's order: a statement at the top of the + /// file, a member after an import's last. + fn unsorted_edit(&self, placement: &Placement) -> Option { + match placement { + Placement::Statement { at, .. } => Some( + self.statement_insertion(*at) + .into_edit(&placement.statement_text()), + ), + Placement::Member { import, .. } => { + let stmt = self + .imports() + .find(|candidate| candidate.stmt.start() == *import)?; + Insertion::existing_import(stmt.stmt, self.tokens) + .map(|insertion| insertion.into_edit(&placement.imported_member())) + } + } + } + + /// The edits that make the additions in `placements` where the file's import sorter would + /// make them, or `None` when the importer has no order to follow, or a statement has no + /// import block to join, for the caller to add them without one. + fn placed_edits(&self, placements: &[Placement]) -> Option> { + self.sorting.as_ref()?.placed_edits(self, placements) + } + + /// The imports the sorter sorts as one block that new imports join, the run of consecutive + /// import statements the module's body starts its imports with, and where `order` sorts + /// each of them, worked out once into `block`. `None` when it starts at or after `at`, + /// where the imports have to be available. + fn import_block<'b, O: ImportOrdering>( + &self, + order: &O, + block: &'b ImportBlock, + at: TextSize, + ) -> Option<(&'a [ast::Stmt], &'b [O::Position])> { + let body: &'a [ast::Stmt] = self.parsed.suite(); + let (range, positions) = block + .get_or_init(|| { + let start = body.iter().position(is_import)?; + let len = body[start..] + .iter() + .take_while(|stmt| is_import(stmt)) + .count(); + let positions = body[start..start + len] + .iter() + .map(|stmt| sorted_import(stmt, order).map(|import| order.position(&import))) + .collect::>()?; + Some((start..start + len, positions)) + }) + .as_ref()?; + let statements = &body[range.clone()]; + (statements.first()?.start() < at).then_some((statements, positions.as_slice())) + } + + /// The edits that make the additions in `placements` where `order` puts them. + fn placed_edits_in( + &self, + order: &O, + block: &ImportBlock, + placements: &[Placement], + ) -> Option> { + let mut edits = Vec::new(); + + // Statements: two `from` imports of one module, as a member the file imports nothing + // from yet makes, are one statement listing both + let mut statements: Vec = Vec::new(); + for placement in placements { + let Placement::Statement { + module, + level, + member, + asname, + at, + } = placement + else { + continue; + }; + let joins = member.is_some() + && !order.imports_member_alone(module.as_deref(), asname.as_deref()); + if joins + && let Some(existing) = statements.iter_mut().find(|existing| { + existing.module == module.as_deref() + && existing.level == *level + && !existing.members.is_empty() + && !order.imports_member_alone(existing.module, existing.members[0].1) + }) + { + if let Some(member) = member { + existing.members.push((member.as_str(), asname.as_deref())); + } + existing.at = existing.at.min(*at); + continue; + } + statements.push(NewStatement { + module: module.as_deref(), + level: *level, + members: member + .iter() + .map(|member| (member.as_str(), asname.as_deref())) + .collect(), + asname: if member.is_some() { + None + } else { + asname.as_deref() + }, + at: *at, + }); + } + for statement in &mut statements { + statement + .members + .sort_by(|a, b| order.compare_members(*a, *b)); + statement.members.dedup(); + } + if !statements.is_empty() { + let at = statements.iter().map(|statement| statement.at).min()?; + edits.extend(self.placed_statements(order, block, &statements, at)?); + } + + // Members added to `from` imports the file has + let mut by_import: Vec<(TextSize, Vec)> = Vec::new(); + for placement in placements { + if let Placement::Member { + import, + member, + asname, + } = placement + { + let entry = match by_import.iter_mut().find(|(start, _)| start == import) { + Some(entry) => entry, + None => { + by_import.push((*import, Vec::new())); + by_import.last_mut()? + } + }; + entry.1.push((member.as_str(), asname.as_deref())); + } + } + for (import, mut added) in by_import { + added.sort_by(|a, b| order.compare_members(*a, *b)); + added.dedup(); + let AstImportKind::ImportFrom(from) = self + .imports() + .find(|candidate| candidate.stmt.start() == import)? + .kind + else { + return None; + }; + edits.extend(self.placed_members(order, from, &added)?); + } + Some(edits) + } + + /// The edits adding `statements` to the file's import block, each in its section, sorted + /// among the section's imports and each other, with the blank lines and section headings + /// the sorter writes around them. + fn placed_statements( + &self, + order: &O, + block: &ImportBlock, + statements: &[NewStatement], + at: TextSize, + ) -> Option> { + let (block, positions) = self.import_block(order, block, at)?; + + // The sorter's order is total, so in a sorted block a new statement goes before the + // first import that sorts after it. In a block that isn't sorted, that is still a place + // among the imports it sorts with. Statements that go before the same import are sorted + // among each other. + let mut slots: Vec> = Vec::new(); + for statement in statements { + let position = order.position(&statement.sorted_import()); + let after = positions.iter().position(|existing| *existing > position); + let slot = match slots.iter_mut().find(|(slot, _)| *slot == after) { + Some(slot) => slot, + None => { + slots.push((after, Vec::new())); + slots.last_mut()? + } + }; + slot.1.push((position, statement.text())); + } + + let source: &str = &self.source; + let line_ending = self.stylist.line_ending().as_str(); + let mut edits = Vec::new(); + for (after, mut added) in slots { + added.sort_by(|a, b| a.0.cmp(&b.0)); + let before = match after { + Some(0) => None, + Some(after) => Some(after - 1), + None => Some(positions.len() - 1), + }; + + // The comments directly above an import are the sorter's to move with it, except + // the heading of its section, so statements placed before an import go above them. + let floor = before.map_or_else( + || source.line_start(block[0].start()), + |before| source.full_line_end(block[before].end()), + ); + let anchor = after.map(|after| { + let section_heading = order.section_heading(&positions[after]); + let mut anchor = source.line_start(block[after].start()); + while anchor > floor { + let previous = source.line_start(anchor - TextSize::from(1)); + let line = source[TextRange::new(previous, anchor)].trim(); + if !line.starts_with('#') || Some(line) == section_heading { + break; + } + anchor = previous; + } + anchor + }); + let existing_blank_lines = anchor.map_or(0, |anchor| { + source[TextRange::new(floor, anchor)] + .lines() + .filter(|line| line.trim().is_empty()) + .count() + }); + + // The added statements and the blank lines and headings between them and the + // imports they go between + let mut text = String::new(); + let mut previous = before.map(|before| &positions[before]); + for (position, statement) in &added { + let blank_lines = + previous.map_or(0, |previous| order.blank_lines_between(previous, position)); + for _ in 0..blank_lines { + text.push_str(line_ending); + } + if previous + .is_none_or(|previous| order.section(previous) != order.section(position)) + && after.is_none_or(|after| { + order.section(&positions[after]) != order.section(position) + }) + && let Some(heading) = order.section_heading(position) + { + text.push_str(heading); + text.push_str(line_ending); + } + text.push_str(statement); + text.push_str(line_ending); + previous = Some(position); + } + let first = &added.first()?.0; + let last = &added.last()?.0; + let blank_before = before.map_or(0, |before| { + order.blank_lines_between(&positions[before], first) + }); + let blank_after = after.map_or(0, |after| { + order.blank_lines_between(last, &positions[after]) + }); + + let (offset, text) = match (before, anchor) { + // Joins the imports after them: placed above those, as far below the imports + // before them as the sorter separates the two, which the lines already there + // are. + (_, Some(anchor)) if blank_after == 0 && (before.is_none() || blank_before > 0) => { + let text = text + .strip_prefix(&line_ending.repeat(blank_before)) + .unwrap_or(&text); + let mut padded = + line_ending.repeat(blank_before.saturating_sub(existing_blank_lines)); + padded.push_str(text); + (anchor, padded) + } + // Placed below the imports before them, as far above the imports after them as + // the sorter separates the two. + (Some(before), _) => { + let offset = source.full_line_end(block[before].end()); + let mut padded = String::new(); + if !source[..offset.to_usize()].ends_with(['\n', '\r']) { + padded.push_str(line_ending); + } + padded.push_str(&text); + if anchor.is_some() { + padded.push_str( + &line_ending.repeat(blank_after.saturating_sub(existing_blank_lines)), + ); + } + (offset, padded) + } + // First in the block, in a section of their own. + (None, Some(anchor)) => { + let mut padded = text; + padded.push_str(&line_ending.repeat(blank_after)); + (anchor, padded) + } + (None, None) => return None, + }; + edits.push(Edit::insertion(text, offset)); + } + Some(edits) + } + + /// The edits listing `added`, sorted, among the members of the existing `from` import + /// `from`, each where `order` sorts it. + fn placed_members( + &self, + order: &O, + from: &ast::StmtImportFrom, + added: &[Member], + ) -> Option> { + let source: &str = &self.source; + let names = &from.names; + let line_ending = self.stylist.line_ending().as_str(); + let mut edits = Vec::new(); + let mut appended = Vec::new(); + let mut slots: Vec<(usize, Vec)> = Vec::new(); + for (member, asname) in added { + let imported = match asname { + Some(asname) => format!("{member} as {asname}"), + None => (*member).to_string(), + }; + let next = names.iter().position(|alias| { + order + .compare_members( + (member, *asname), + ( + alias.name.as_str(), + alias.asname.as_ref().map(ast::Identifier::as_str), + ), + ) + .is_lt() + }); + match next { + Some(next) => match slots.iter_mut().find(|(slot, _)| *slot == next) { + Some(slot) => slot.1.push(imported), + None => slots.push((next, vec![imported])), + }, + None => appended.push(imported), + } + } + for (next, members) in slots { + // Written with the separator the import already puts between its members, so a + // member lands on a line of its own in an import that writes one per line. + let separator = if next + 1 < names.len() { + TextRange::new(names[next].end(), names[next + 1].start()) + } else if next > 0 { + TextRange::new(names[next - 1].end(), names[next].start()) + } else { + TextRange::empty(names[next].start()) + }; + let separator = &source[separator]; + let separator = if separator.is_empty() { + ", ".to_string() + } else if separator.contains('#') { + // a comment trails the member before; only the line break and indentation + // are reused + let indentation = &source + [TextRange::new(source.line_start(names[next].start()), names[next].start())]; + format!(",{line_ending}{indentation}") + } else { + separator.to_string() + }; + let mut text = String::new(); + for member in members { + text.push_str(&member); + text.push_str(&separator); + } + edits.push(Edit::insertion(text, names[next].start())); + } + if !appended.is_empty() { + let stmt = self + .imports() + .find(|candidate| candidate.stmt.start() == from.start())?; + edits.push( + Insertion::existing_import(stmt.stmt, self.tokens)?.into_edit(&appended.join(", ")), + ); + } + Some(edits) + } + /// Look for an import already in this importer's module that /// satisfies the given request. If found, the corresponding /// import is returned along with the way in which the import @@ -434,7 +915,7 @@ impl<'a> Importer<'a> { } if let Some(response) = import.satisfies(self.db, importing_file, request) { - let partial = matches!(response.kind, ImportResponseKind::Partial); + let partial = matches!(response.kind, ImportResponseKind::Partial(_)); // The LSP doesn't support edits across cell boundaries. // Skip over imports that only partially satisfy the import @@ -505,6 +986,84 @@ impl<'a> Importer<'a> { } } +/// The import block new imports join, as the range of the module's body it spans, and where an +/// order sorts each of its statements. `None` when the module has no import block, or one with +/// a statement the order can't read. +type ImportBlock = OnceCell, Vec<::Position>)>>; + +/// An importer's sorting, with the type of its order erased. +trait Sorting { + /// The edits that make the additions in `placements` where the order puts them, or `None` + /// when there is no order or the additions can't follow it. + fn placed_edits(&self, importer: &Importer<'_>, placements: &[Placement]) -> Option>; + + /// Whether the order writes `member` (imported `as asname`, if given) from `module` in a + /// `from` import of its own. + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; +} + +/// An order found the first time an import is placed, and the import block as it sorts it. +struct LazyOrder { + resolve: F, + order: OnceCell>, + block: ImportBlock, +} + +impl Option> LazyOrder { + fn order(&self) -> Option<&O> { + self.order.get_or_init(&self.resolve).as_ref() + } +} + +impl Option> Sorting for LazyOrder { + fn placed_edits(&self, importer: &Importer<'_>, placements: &[Placement]) -> Option> { + importer.placed_edits_in(self.order()?, &self.block, placements) + } + + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + self.order() + .is_some_and(|order| order.imports_member_alone(module, asname)) + } +} + +fn is_import(stmt: &ast::Stmt) -> bool { + matches!(stmt, ast::Stmt::Import(_) | ast::Stmt::ImportFrom(_)) +} + +/// An import statement as a sorter reads it to sort it: an `import` by its first module, a +/// `from` import by its module and the member `order` lists first. +fn sorted_import<'a>(stmt: &'a ast::Stmt, order: &impl ImportOrdering) -> Option> { + match stmt { + ast::Stmt::Import(import) => { + let first = import.names.first()?; + Some(SortedImport::Straight { + name: first.name.as_str(), + asname: first.asname.as_ref().map(ast::Identifier::as_str), + is_lazy: import.is_lazy, + }) + } + ast::Stmt::ImportFrom(from) => { + let first_member = from + .names + .iter() + .map(|alias| { + ( + alias.name.as_str(), + alias.asname.as_ref().map(ast::Identifier::as_str), + ) + }) + .min_by(|a, b| order.compare_members(*a, *b)); + Some(SortedImport::From { + module: from.module.as_ref().map(ast::Identifier::as_str), + level: from.level, + first_member, + is_lazy: from.is_lazy, + }) + } + _ => None, + } +} + /// Detects source style once per file revision, shared by diagnostics and IDE features. #[salsa::tracked(returns(ref), no_eq)] fn stylist(db: &dyn Db, file: PythonFile<'_>) -> Stylist<'static> { @@ -649,6 +1208,148 @@ enum MemberImportKind<'ast> { pub struct ImportAction { import: Option, symbol_text: Box, + /// What `import` adds, for [`Importer::combined_edits`] to place it together with what + /// other actions add. + placement: Option, +} + +/// What an import action adds to the file. +#[derive(Debug, Clone)] +enum Placement { + /// A new import statement: `import module as asname`, or `from module import member as + /// asname`, `level` dots up. + Statement { + module: Option, + level: u32, + member: Option, + asname: Option, + /// Where what is imported has to be available. + at: TextSize, + }, + /// A member listed by the existing `from` import starting at `import`. + Member { + import: TextSize, + member: String, + asname: Option, + }, +} + +impl Placement { + /// The statement `request` adds when the file has no import that satisfies it. + fn statement(request: &ImportRequest<'_>, at: TextSize) -> Self { + let member = match (&request.style, request.member) { + (ImportStyle::ImportFrom, Some(member)) => Some(member.to_string()), + _ => None, + }; + Placement::Statement { + module: Some(request.module.to_string()), + level: 0, + member, + asname: request.asname.map(str::to_string), + at, + } + } + + /// The statement a [`Placement::Statement`] adds. + fn statement_text(&self) -> String { + match self { + Placement::Statement { + module, + level, + member, + asname, + .. + } => NewStatement { + module: module.as_deref(), + level: *level, + members: member + .iter() + .map(|member| (member.as_str(), asname.as_deref())) + .collect(), + asname: if member.is_some() { + None + } else { + asname.as_deref() + }, + at: TextSize::default(), + } + .text(), + Placement::Member { .. } => String::new(), + } + } + + /// The member a [`Placement::Member`] lists: `member`, or `member as asname`. + fn imported_member(&self) -> String { + match self { + Placement::Member { + member, + asname: Some(asname), + .. + } => format!("{member} as {asname}"), + Placement::Member { member, .. } => member.clone(), + Placement::Statement { .. } => String::new(), + } + } +} + +/// A member of a `from` import, and the name it is imported as. +type Member<'a> = (&'a str, Option<&'a str>); + +/// The statements added before the import of the block at the index, or after the block's last +/// import for `None`, each with where it sorts. +type StatementSlot

= (Option, Vec<(P, String)>); + +/// An import statement to add: `import module as asname`, or `from module import members`. +#[derive(Debug)] +struct NewStatement<'a> { + module: Option<&'a str>, + level: u32, + /// The members a `from` import lists, sorted; empty for an `import`. + members: Vec>, + asname: Option<&'a str>, + at: TextSize, +} + +impl<'a> NewStatement<'a> { + fn sorted_import(&self) -> SortedImport<'a> { + if self.members.is_empty() { + SortedImport::Straight { + name: self.module.unwrap_or_default(), + asname: self.asname, + is_lazy: false, + } + } else { + SortedImport::From { + module: self.module, + level: self.level, + first_member: self.members.first().copied(), + is_lazy: false, + } + } + } + + fn text(&self) -> String { + let module = self.module.unwrap_or_default(); + if self.members.is_empty() { + return match self.asname { + Some(asname) => format!("import {module} as {asname}"), + None => format!("import {module}"), + }; + } + let members = self + .members + .iter() + .map(|(member, asname)| match asname { + Some(asname) => format!("{member} as {asname}"), + None => (*member).to_string(), + }) + .collect::>() + .join(", "); + format!( + "from {}{module} import {members}", + ".".repeat(self.level as usize) + ) + } } impl ImportAction { @@ -770,7 +1471,7 @@ impl<'ast> AstImportKind<'ast> { None => alias.name.as_str() == "*" || alias.name.as_str() == member, }) .map(|alias| ImportResponseKind::Unqualified { alias }) - .unwrap_or(ImportResponseKind::Partial); + .unwrap_or(ImportResponseKind::Partial(ast)); Some(kind) } } @@ -1035,7 +1736,7 @@ enum ImportResponseKind<'ast> { /// /// It is guaranteed that this never contains a wildcard import. /// (otherwise, this import wouldn't be partial). - Partial, + Partial(&'ast ast::StmtImportFrom), } impl ImportResponseKind<'_> { @@ -1055,7 +1756,7 @@ impl ImportResponseKind<'_> { fn priority(&self) -> usize { match *self { ImportResponseKind::Unqualified { .. } => 0, - ImportResponseKind::Partial => 1, + ImportResponseKind::Partial(_) => 1, // N.B. When given the choice between adding a // name to an existing `from ... import ...` // statement and using an existing `import ...` diff --git a/crates/ty_server/tests/e2e/code_actions.rs b/crates/ty_server/tests/e2e/code_actions.rs index ebf9526068..4acefc1b9f 100644 --- a/crates/ty_server/tests/e2e/code_actions.rs +++ b/crates/ty_server/tests/e2e/code_actions.rs @@ -512,3 +512,104 @@ def f(user: User | None): Ok(()) } + +/// The text of `content` with the edits of the code action titled `title` applied. +fn applied(actions: &[lsp_types::CodeActionResponse], title: &str, content: &str) -> String { + let action = actions + .iter() + .find_map(|action| match action { + lsp_types::CodeActionResponse::CodeAction(action) if action.title == title => { + Some(action) + } + _ => None, + }) + .unwrap_or_else(|| panic!("no code action `{title}`")); + let mut edits: Vec<_> = action + .edit + .iter() + .flat_map(|edit| edit.changes.iter().flatten()) + .flat_map(|(_, edits)| edits.iter().cloned()) + .collect(); + edits + .sort_by_key(|edit| std::cmp::Reverse((edit.range.start.line, edit.range.start.character))); + let mut lines: Vec = content.split('\n').map(str::to_string).collect(); + for edit in edits { + let start = edit.range.start; + let end = edit.range.end; + let head = lines[start.line as usize][..start.character as usize].to_string(); + let tail = lines[end.line as usize][end.character as usize..].to_string(); + let replaced = format!("{head}{}{tail}", edit.new_text); + lines.splice( + start.line as usize..=end.line as usize, + replaced.split('\n').map(str::to_string), + ); + } + lines.join("\n") +} + +/// An import a quick fix adds goes where buff's isort configuration sorts it: here, with +/// `force-sort-within-sections`, among the standard library's imports by module whatever their +/// kind, and after the project's other import of its own package, which buff finds is +/// first-party because the package is in the project. +#[test] +fn an_import_is_added_where_the_project_s_isort_configuration_sorts_it() -> Result<()> { + let main = SystemPath::new("main.py"); + let main_content = "\ +from collections import OrderedDict +import json +import sys + +from shop import other + +Path, helper, OrderedDict, json, sys, other +"; + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new(""), None)? + .with_file( + "pyproject.toml", + "\ +[project] +name = \"shop\" +version = \"0\" + +[tool.ruff.lint.isort] +force-sort-within-sections = true +", + )? + .with_file("shop/__init__.py", "other = 1\n")? + .with_file("shop/util.py", "def helper() -> None: ...\n")? + .with_file(main, main_content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(main, main_content, 1); + let diagnostics = server.document_diagnostic_request(main, None); + let code_action_params = code_actions_at(&server, diagnostics, main, full_range(main_content)); + let code_actions = server + .send_request_await::(code_action_params) + .unwrap_or_default(); + + insta::assert_snapshot!(applied(&code_actions, "import pathlib.Path", main_content), @" + from collections import OrderedDict + import json + from pathlib import Path + import sys + + from shop import other + + Path, helper, OrderedDict, json, sys, other + "); + insta::assert_snapshot!(applied(&code_actions, "import shop.util.helper", main_content), @" + from collections import OrderedDict + import json + import sys + + from shop import other + from shop.util import helper + + Path, helper, OrderedDict, json, sys, other + "); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index a82eb77ec1..56ad18168a 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -402,15 +402,15 @@ is_typedd { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, - "newText": "from typing_extensions import is_typeddict\n" + "newText": "\nfrom typing_extensions import is_typeddict\n" } ], "command": { @@ -472,15 +472,15 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, - "newText": "from typing_extensions import TypedDict\n" + "newText": "\nfrom typing_extensions import TypedDict\n" } ] }, @@ -493,15 +493,15 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, - "newText": "from _typeshed._type_checker_internals import TypedDictFallback\n" + "newText": "\nfrom _typeshed._type_checker_internals import TypedDictFallback\n" } ] }, @@ -514,15 +514,15 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, - "newText": "from typing_extensions import is_typeddict\n" + "newText": "\nfrom typing_extensions import is_typeddict\n" } ] }, @@ -535,11 +535,11 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, @@ -556,11 +556,11 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, @@ -577,15 +577,15 @@ TypedDi { "range": { "start": { - "line": 0, + "line": 1, "character": 0 }, "end": { - "line": 0, + "line": 1, "character": 0 } }, - "newText": "import _typeshed.dbapi\n" + "newText": "\nimport _typeshed.dbapi\n" } ] } diff --git a/crates/ty_server/tests/e2e/inlay_hints.rs b/crates/ty_server/tests/e2e/inlay_hints.rs index e0076cfb2e..fccb28ec7c 100644 --- a/crates/ty_server/tests/e2e/inlay_hints.rs +++ b/crates/ty_server/tests/e2e/inlay_hints.rs @@ -464,14 +464,14 @@ def get_a() -> A: "range": { "start": { "line": 0, - "character": 21 + "character": 16 }, "end": { "line": 0, - "character": 21 + "character": 16 } }, - "newText": ", A" + "newText": "A, " } ], "data": { diff --git a/crates/ty_server/tests/e2e/override_members.rs b/crates/ty_server/tests/e2e/override_members.rs index e2dcf6af93..d977c1859d 100644 --- a/crates/ty_server/tests/e2e/override_members.rs +++ b/crates/ty_server/tests/e2e/override_members.rs @@ -139,13 +139,14 @@ fn the_members_picked_are_written_with_their_imports_and_the_first_body_selected "{answer:#}" ); assert_eq!(edits.len(), 2, "{answer:#}"); - assert_eq!(edits[1]["newText"], "from pathlib import Path\n"); - // the `...` of `area`, once the import has moved it down a line + // the standard library's section, above the project's own imports + assert_eq!(edits[1]["newText"], "from pathlib import Path\n\n"); + // the `...` of `area`, once the import has moved it down two lines assert_eq!( answer["selection"], serde_json::json!({ - "start": { "line": 7, "character": 8 }, - "end": { "line": 7, "character": 11 }, + "start": { "line": 8, "character": 8 }, + "end": { "line": 8, "character": 11 }, }), ); diff --git a/crates/ty_server/tests/e2e/pasted_imports.rs b/crates/ty_server/tests/e2e/pasted_imports.rs index 22c33ecd1f..8ee756e194 100644 --- a/crates/ty_server/tests/e2e/pasted_imports.rs +++ b/crates/ty_server/tests/e2e/pasted_imports.rs @@ -108,7 +108,9 @@ fn a_copy_answers_its_imports_and_a_paste_the_ones_it_is_missing() -> Result<()> .collect(); assert_eq!( added, - ["import math\n", "from shapes import SCALE\n"], + // `math` joins the standard library's imports, above `pathlib`'s, and `shapes`, which is + // the project's own, starts a section of its own below them + ["import math\n", "\nfrom shapes import SCALE\n"], "{pasted:#}" ); Ok(()) diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap index 2d2beb7110..e355a5ca3c 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_existing_import_undefined_decorator.snap @@ -34,11 +34,11 @@ expression: code_actions { "range": { "start": { - "line": 0, + "line": 2, "character": 0 }, "end": { - "line": 0, + "line": 2, "character": 0 } }, @@ -79,15 +79,15 @@ expression: code_actions { "range": { "start": { - "line": 0, + "line": 2, "character": 0 }, "end": { - "line": 0, + "line": 2, "character": 0 } }, - "newText": "from typing_extensions import deprecated\n" + "newText": "\nfrom typing_extensions import deprecated\n" } ] } diff --git a/crates/ty_server/tests/e2e/test_subjects.rs b/crates/ty_server/tests/e2e/test_subjects.rs index 3f07a1e31f..2e90ad1fe3 100644 --- a/crates/ty_server/tests/e2e/test_subjects.rs +++ b/crates/ty_server/tests/e2e/test_subjects.rs @@ -196,8 +196,9 @@ fn a_new_test_in_a_test_module_that_does_not_import_the_code_comes_with_the_impo assert_eq!( answer["newTest"]["importEdits"], json!([{ - "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, - "newText": "from shapes.circle import Circle\n", + // the project's own import, in a section of its own below the standard library's + "range": { "start": { "line": 1, "character": 0 }, "end": { "line": 1, "character": 0 } }, + "newText": "\nfrom shapes.circle import Circle\n", }]), "{answer:#}", ); diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 76ad6b337b..ae0783f73f 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -181,6 +181,33 @@ written through it, and one from a module with a `from ... import` is added to t statement. how imports are sorted, grouped and split is not an editor setting: it is the project's isort configuration, which `buff` reads and fixes +an import any of these write — a completion's, a quick fix's, a pasted name's, an +inlay hint's — goes where `buff check --select I --fix` would put it, so adding one +never leaves the block unsorted. it joins its section of the file's first block of +imports (future, standard library, third party, first party, local, or the +sections the configuration defines) in the order isort sorts that section, with a +blank line around a section the block did not have yet, and a member added to an +existing `from` import is listed where isort orders the members: + +```py +import json +import sys +from collections import OrderedDict + +import attr +``` + +`math` is imported between `json` and `sys`, `Path` below the `collections` +import, `defaultdict` as `from collections import OrderedDict, defaultdict`, and +`attrs` below `attr`. the configuration is `buff`'s own, resolved as `buff` +resolves it — the closest `ruff.toml`, `.ruff.toml` or `pyproject.toml` with a +`[tool.ruff]` table, then the user's, then the defaults — so +`force-sort-within-sections`, `from-first`, `known-first-party`, `section-order`, +`no-lines-before`, `lines-between-types`, `force-single-line` and the rest apply +as they do to `buff`. the import has to run before the name it binds is used, so a +file whose imports all come after that point, or that has none, still gets it at +the top, below a docstring and any `from __future__` import + ### names inside a string a name written inside a plain string's braces completes as a name, and taking From f5370079594eea32817ee2d93e44d25ffa7a6fe4 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:30:21 +1000 Subject: [PATCH 034/110] move a module's top-level function, class or variable to another module with by/moveSymbol, after by/prepareMoveSymbol names it from its definition or a reference, carrying the imports and names it reads, pointing every import, module attribute and star import of it at its new module, and refusing a move that reads a private name, makes two modules import each other, or collides with a name the target binds --- crates/ty_ide/src/lib.rs | 5 +- crates/ty_ide/src/refactor.rs | 3 + crates/ty_ide/src/refactor/move_symbol.rs | 1405 +++++++++++++++++ crates/ty_python_semantic/src/importer.rs | 7 +- crates/ty_server/src/server/api.rs | 9 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/move_symbol.rs | 248 +++ crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/move_symbol.rs | 159 ++ docs/basedpython/features/editor.md | 57 + 10 files changed, 1893 insertions(+), 3 deletions(-) create mode 100644 crates/ty_ide/src/refactor/move_symbol.rs create mode 100644 crates/ty_server/src/server/api/requests/move_symbol.rs create mode 100644 crates/ty_server/tests/e2e/move_symbol.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index ade47e81d9..af43162436 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -109,7 +109,10 @@ pub use override_members::{ override_members, }; pub use pasted_imports::{CopiedCode, CopiedImport, PastedImports, copied_imports, pasted_imports}; -pub use refactor::{Refactor, RefactorKind, RefactorOffer, refactor, refactors}; +pub use refactor::{ + MovableKind, MovableSymbol, Refactor, RefactorKind, RefactorOffer, SymbolMove, movable_symbol, + move_symbol, refactor, refactors, +}; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; pub use selection_range::selection_range; diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 8996ee1b2a..527f3a7023 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -22,6 +22,7 @@ mod extract_variable; mod flow; mod hazards; mod inline_variable; +mod move_symbol; mod names; mod return_annotation; mod split_join; @@ -39,6 +40,8 @@ use ty_python_semantic::SemanticModel; use crate::FileEdit; +pub use move_symbol::{MovableKind, MovableSymbol, SymbolMove, movable_symbol, move_symbol}; + /// The refactorings the server knows how to perform. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum RefactorKind { diff --git a/crates/ty_ide/src/refactor/move_symbol.rs b/crates/ty_ide/src/refactor/move_symbol.rs new file mode 100644 index 0000000000..efca6b6f29 --- /dev/null +++ b/crates/ty_ide/src/refactor/move_symbol.rs @@ -0,0 +1,1405 @@ +//! Moving a module's top-level function, class or variable to another module. +//! +//! ```py +//! # shapes/util.py # shapes/geometry.py +//! import math import math +//! +//! RATE = 2 from shapes.util import RATE +//! +//! def area(r): ──► def area(r): +//! return math.pi * r * RATE return math.pi * r * RATE +//! ``` +//! +//! The definition is cut from its module, with its decorators and the comments directly above +//! it, and appended to the target. What it reads from its old module comes with it: an import it +//! uses is written in the target too (and dropped from the old module when nothing else there +//! uses it and no other module imports it from there), and another of the old module's names is +//! imported from there. Every reference to it in the project follows it: +//! `from shapes.util import area` imports it from `shapes.geometry`, `shapes.util.area` reads +//! `shapes.geometry.area`, a module that reached it through a star import of the old module +//! imports it by name, and the old module imports it back when it still uses it. +//! +//! The move is refused, with the reason, when the result would not run as the original did: +//! when the definition reads a name private to its module, which nothing outside the module is +//! meant to import; when it reads a name its module binds more than once, which an import would +//! see only as it was when imported; when it assigns a name of its module through `global`, which +//! would name the target's once moved; when it reads an attribute of its module object, `__name__` +//! or `__file__`, which would be the target's; when it reads a builtin the target binds to +//! something else; when the two modules would end up importing each other, which python cannot +//! load; when the target already binds the name; when the name is bound more than once in its +//! module; and when the modules are not the same kind of file. + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; +use ruff_python_codegen::Stylist; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_module_resolver::{ImportingFile, ModuleName, file_to_module}; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_core::definition::DefinitionKind; +use ty_python_core::scope::FileScopeId; +use ty_python_core::semantic_index; +use ty_python_semantic::importer::{ImportAction, ImportRequest, MembersInScope}; +use ty_python_semantic::types::Type; +use ty_python_semantic::{HasType, SemanticModel}; + +use super::flow::{bindings_of_symbol, resolving_scope}; +use crate::code_action::FileEdit; +use crate::goto::find_goto_target; +use crate::import_order; +use crate::references::{ReferencesMode, references}; + +/// What a top-level definition that can be moved is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MovableKind { + Function, + Class, + Variable, +} + +/// A top-level definition [`move_symbol`] can move. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MovableSymbol { + /// The file it is defined in, which need not be the one asked about. + pub file: File, + pub name: String, + pub kind: MovableKind, + /// The range of its name where it is defined. + pub name_range: TextRange, + /// The qualified name of the module it is defined in. + pub module: String, +} + +/// What [`move_symbol`] does to the project. +#[derive(Debug, Default)] +pub struct SymbolMove { + pub edits: Vec, + /// References the move leaves as they are, which may no longer reach the definition once it + /// has moved, each described with where it is. + pub warnings: Vec, +} + +/// The top-level definition named at `offset` — at its definition or at a reference to it — when +/// it is one [`move_symbol`] can move: a function, a class, or a variable a single name is +/// assigned to, at the top level of a module of the project. +pub fn movable_symbol( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option { + let (file, offset) = declaration_site(db, file, offset)?; + let parsed = parsed_module(db, file.python_file(db)).load(db); + let declaration = Declaration::at(parsed.suite(), offset)?; + let module = file_to_module(db, file.resolver_file(db))?; + Some(MovableSymbol { + file: file.file(db), + name: declaration.name.to_string(), + kind: declaration.kind, + name_range: declaration.name_range, + module: module.name(db).to_string(), + }) +} + +/// Moves the top-level definition named at `offset` to the end of the module `target`, and +/// updates the project to find it there. `Err` says why it cannot be moved. +pub fn move_symbol( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + target: File, +) -> Result { + let (source, offset) = declaration_site(db, file, offset) + .ok_or_else(|| "there is no top-level function, class or variable here".to_string())?; + let parsed = parsed_module(db, source.python_file(db)).load(db); + let declaration = Declaration::at(parsed.suite(), offset) + .ok_or_else(|| "there is no top-level function, class or variable here".to_string())?; + let name = declaration.name; + let source_file = source.file(db); + let target_file = ProgramFile::new(db, target, source.program(db)); + + let source_module = file_to_module(db, source.resolver_file(db)) + .ok_or_else(|| format!("`{name}` is not in a module anything can import"))? + .name(db) + .clone(); + if target == source_file { + return Err(format!("`{name}` is in {source_module} already")); + } + let target_module = file_to_module(db, target_file.resolver_file(db)) + .ok_or_else(|| "the target is not a module anything can import".to_string())? + .name(db) + .clone(); + let source_type = source_file.source_type(db); + let target_type = target.source_type(db); + if !target_type.is_py_file() { + return Err(format!("{target_module} is not a module of source code")); + } + if source_type.is_basedpython() != target_type.is_basedpython() { + return Err(if source_type.is_basedpython() { + format!( + "{target_module} is a python module, and `{name}` is basedpython, which python does \ + not read" + ) + } else { + format!( + "{target_module} is a basedpython module, which reads `{name}`'s python \ + differently" + ) + }); + } + + let index = semantic_index(db, source); + let globals = index.place_table(FileScopeId::global()); + let symbol = globals + .symbol_id(name) + .ok_or_else(|| format!("`{name}` is not a name of {source_module}"))?; + let bindings = bindings_of_symbol( + db, + index, + &parsed, + FileScopeId::global(), + symbol, + source_type.is_stub(), + ); + if bindings.len() != 1 { + return Err(format!( + "`{name}` is bound more than once in {source_module}, so moving one of them would \ + change which the module ends with" + )); + } + + let target_parsed = parsed_module(db, target_file.python_file(db)).load(db); + let target_index = semantic_index(db, target_file); + let target_globals = target_index.place_table(FileScopeId::global()); + let target_binds = |name: &str| { + target_globals + .symbol_id(name) + .is_some_and(|symbol| target_globals.symbol(symbol).is_bound()) + }; + if target_binds(name) { + return Err(format!("{target_module} has a `{name}` already")); + } + + // What the definition reads from its module comes with it. + let dependencies = dependencies(db, source, &parsed, declaration.statement, name); + let mut target_imports: Vec = Vec::new(); + let mut from_source: Vec = Vec::new(); + for dependency in &dependencies { + match dependency { + Dependency::Imported { request, .. } => { + if target_binds(&request.binds) + && !target_imports_same(db, target_file, &target_parsed, request) + { + return Err(format!( + "`{name}` uses `{}`, which {target_module} binds to something else", + request.binds + )); + } + target_imports.push(request.clone()); + } + Dependency::Sibling(sibling) => { + if is_private(sibling) { + return Err(format!( + "`{name}` uses `{sibling}`, which is private to {source_module}; make it \ + public, or move it too" + )); + } + if target_binds(sibling) { + return Err(format!( + "`{name}` uses {source_module}'s `{sibling}`, which {target_module} binds \ + to something else" + )); + } + from_source.push(sibling.clone()); + } + } + } + + // Where the definition is referred to, other than by itself. + let statement_range = declaration.statement.range(); + let model = SemanticModel::new(db, source); + let goto_target = find_goto_target(&model, &parsed, declaration.name_range.start()) + .ok_or_else(|| format!("`{name}` cannot be searched for"))?; + let mut by_file: FxHashMap> = FxHashMap::default(); + for reference in + references(db, source, &goto_target, ReferencesMode::RenameMultiFile).unwrap_or_default() + { + if reference.file() == source_file && statement_range.contains_range(reference.range()) { + continue; + } + by_file + .entry(reference.file()) + .or_default() + .push(reference.range()); + } + let source_uses: Vec = by_file.remove(&source_file).unwrap_or_default(); + + // Two modules that import each other at the top cannot both be loaded: the second one would + // read a name from the first before the first has bound it. + let source_needs_target = + !source_uses.is_empty() || imports_module(db, source, &parsed, &target_module); + let target_needs_source = + !from_source.is_empty() || imports_module(db, target_file, &target_parsed, &source_module); + if source_needs_target && target_needs_source { + let reason = if from_source.is_empty() { + format!("{target_module} imports {source_module}") + } else { + format!( + "`{name}` uses {}", + from_source + .iter() + .map(|sibling| format!("`{sibling}`")) + .collect::>() + .join(", ") + ) + }; + let because = if source_uses.is_empty() { + format!("{source_module} imports {target_module}") + } else { + format!("{source_module} still uses `{name}`") + }; + return Err(format!( + "{source_module} and {target_module} would import each other: {reason}, and \ + {because}" + )); + } + + let mut moved = SymbolMove::default(); + + // The old module: the definition cut, the imports only it used dropped, and the definition + // imported back where the module still uses it. + let source_text = source_text(db, source_file); + let source_text = source_text.as_str(); + let cut = cut_range(source_text, parsed.suite(), declaration.index); + moved.edits.push(FileEdit { + file: source_file, + edit: Edit::range_deletion(cut.removed), + }); + for dependency in &dependencies { + if let Dependency::Imported { + binding_statement: Some(binding_statement), + alias_index, + request, + } = dependency + && !used_elsewhere(&parsed, &request.binds, statement_range) + && let Some(edit) = remove_alias( + source_text, + parsed.suite(), + *binding_statement, + *alias_index, + ) + { + moved.edits.push(FileEdit { + file: source_file, + edit, + }); + } + } + if let Some(edit) = remove_from_all(parsed.suite(), name) { + moved.edits.push(FileEdit { + file: source_file, + edit, + }); + } + if let Some(first_use) = source_uses.iter().map(Ranged::start).min() { + let importer = import_order::importer(db, source, &parsed); + let action = importer.import( + ImportRequest::import_from(target_module.as_str(), name), + &MembersInScope::empty(first_use), + ); + moved.edits.extend( + importer + .combined_edits([&action]) + .into_iter() + .map(|edit| FileEdit { + file: source_file, + edit, + }), + ); + } + + // The target: what the definition imports, then the definition. + let target_text = ruff_db::source::source_text(db, target); + let target_text = target_text.as_str(); + let target_end = TextSize::of(target_text); + let stylist = Stylist::from_tokens(target_parsed.tokens(), target_text); + let importer = import_order::importer(db, target_file, &target_parsed); + let members = MembersInScope::empty(target_end); + let mut actions: Vec = Vec::new(); + for request in &target_imports { + let action = importer.import(request.request(), &members); + if action.symbol_text() != request.binds { + return Err(format!( + "`{name}` uses `{}`, which {target_module} cannot import under that name", + request.binds + )); + } + actions.push(action); + } + for sibling in &from_source { + actions.push(importer.import( + ImportRequest::import_from(source_module.as_str(), sibling), + &members, + )); + } + let import_edits = importer.combined_edits(&actions); + let adds_imports = !import_edits.is_empty(); + moved.edits.extend( + import_edits + .into_iter() + .map(|edit| FileEdit { file: target, edit }), + ); + moved.edits.push(FileEdit { + file: target, + edit: Edit::insertion( + appended_text( + target_text, + target_parsed.suite(), + &cut.text, + declaration.kind, + adds_imports, + stylist.line_ending().as_str(), + ), + target_end, + ), + }); + if remove_from_all(parsed.suite(), name).is_some() + && let Some(edit) = add_to_all(target_text, target_parsed.suite(), name) + { + moved.edits.push(FileEdit { file: target, edit }); + } + + // Every other module that refers to it. + let mut files: Vec = by_file.keys().copied().collect(); + files.sort_by_key(|file| file.path(db).as_str().to_string()); + for file in files { + let ranges = &by_file[&file]; + rewrite_references( + db, + ProgramFile::new(db, file, source.program(db)), + ranges, + name, + &source_module, + &target_module, + &mut moved, + ); + } + Ok(moved) +} + +/// The file and offset of the top-level declaration named at `offset`: `offset` itself when it is +/// in a declaration's name, or else where the name at `offset` is defined. +fn declaration_site<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + offset: TextSize, +) -> Option<(ProgramFile<'db>, TextSize)> { + let parsed = parsed_module(db, file.python_file(db)).load(db); + if Declaration::at(parsed.suite(), offset).is_some() { + return Some((file, offset)); + } + let targets = crate::goto_definition(db, file, offset)?; + let mut targets = targets.value.into_iter(); + let target = targets.next()?; + if targets.next().is_some() { + return None; + } + let target_file = ProgramFile::new(db, target.file(), file.program(db)); + if !db.project().files(db).contains(target.file()) { + return None; + } + let target_parsed = parsed_module(db, target_file.python_file(db)).load(db); + let declaration = Declaration::at(target_parsed.suite(), target.focus_range().start())?; + Some((target_file, declaration.name_range.start())) +} + +/// A top-level statement that defines one name. +struct Declaration<'a> { + index: usize, + statement: &'a Stmt, + name: &'a str, + name_range: TextRange, + kind: MovableKind, +} + +impl<'a> Declaration<'a> { + /// The top-level declaration whose name `offset` is in. + fn at(suite: &'a [Stmt], offset: TextSize) -> Option { + suite.iter().enumerate().find_map(|(index, statement)| { + let (name, kind) = match statement { + Stmt::FunctionDef(function) => (&function.name, MovableKind::Function), + Stmt::ClassDef(class) => (&class.name, MovableKind::Class), + Stmt::Assign(assign) => match assign.targets.as_slice() { + [Expr::Name(target)] => { + return target.range().contains_inclusive(offset).then_some(Self { + index, + statement, + name: target.id.as_str(), + name_range: target.range(), + kind: MovableKind::Variable, + }); + } + _ => return None, + }, + Stmt::AnnAssign(assign) if assign.value.is_some() => match assign.target.as_ref() { + Expr::Name(target) => { + return target.range().contains_inclusive(offset).then_some(Self { + index, + statement, + name: target.id.as_str(), + name_range: target.range(), + kind: MovableKind::Variable, + }); + } + _ => return None, + }, + _ => return None, + }; + name.range().contains_inclusive(offset).then_some(Self { + index, + statement, + name: name.as_str(), + name_range: name.range(), + kind, + }) + }) + } +} + +/// An import request whose strings it owns. +#[derive(Debug, Clone, PartialEq, Eq)] +struct OwnedRequest { + module: String, + member: Option, + asname: Option, + /// The name the import binds. + binds: String, +} + +impl OwnedRequest { + fn request(&self) -> ImportRequest<'_> { + match (&self.member, &self.asname) { + (Some(member), Some(asname)) => { + ImportRequest::import_from_as(&self.module, member, asname) + } + (Some(member), None) => ImportRequest::import_from(&self.module, member), + (None, Some(asname)) => ImportRequest::module_as(&self.module, asname), + (None, None) => ImportRequest::module(&self.module), + } + } +} + +/// A name a definition reads from the top level of its module. +#[derive(Debug)] +enum Dependency { + /// A name one of the module's imports binds, as the import that binds it. + Imported { + /// The start of the statement that imports it, or `None` for a star import, which binds + /// more than this name and so is never dropped. + binding_statement: Option, + alias_index: usize, + request: OwnedRequest, + }, + /// Another of the module's own top-level names. + Sibling(String), +} + +/// The names `statement` reads from the top level of `file`, other than `name`, which it defines. +fn dependencies( + db: &dyn Db, + file: ProgramFile<'_>, + parsed: &ParsedModuleRef, + statement: &Stmt, + name: &str, +) -> Vec { + struct Loads<'a> { + names: Vec<&'a ast::ExprName>, + } + impl<'a> SourceOrderVisitor<'a> for Loads<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + { + self.names.push(name); + } + walk_expr(self, expr); + } + } + let mut loads = Loads { names: Vec::new() }; + loads.visit_stmt(statement); + + let index = semantic_index(db, file); + let globals = index.place_table(FileScopeId::global()); + let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); + let mut seen = FxHashSet::default(); + let mut dependencies = Vec::new(); + for load in loads.names { + let id = load.id.as_str(); + if id == name || !seen.insert(id) { + continue; + } + let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { + continue; + }; + if resolving_scope(index, scope, id) != Some(FileScopeId::global()) { + continue; + } + let Some(symbol) = globals.symbol_id(id) else { + continue; + }; + let bindings = bindings_of_symbol( + db, + index, + parsed, + FileScopeId::global(), + symbol, + file.file(db).is_stub(db), + ); + let imported = bindings + .iter() + .find_map(|definition| match definition.kind(db) { + DefinitionKind::Import(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + Some(Dependency::Imported { + binding_statement: Some(statement.start()), + alias_index: import.alias_index(), + request: OwnedRequest { + module: alias.name.to_string(), + member: None, + asname: alias.asname.as_ref().map(ToString::to_string), + binds: id.to_string(), + }, + }) + } + DefinitionKind::ImportFrom(import) => { + let statement = import.import(parsed); + let alias = import.alias(parsed); + let module = + ModuleName::from_import_statement(db, importing_file, statement).ok()?; + Some(Dependency::Imported { + binding_statement: Some(statement.start()), + alias_index: import.alias_index(), + request: OwnedRequest { + module: module.to_string(), + member: Some(alias.name.to_string()), + asname: alias.asname.as_ref().map(ToString::to_string), + binds: id.to_string(), + }, + }) + } + DefinitionKind::StarImport(import) => { + let statement = import.import(parsed); + let module = + ModuleName::from_import_statement(db, importing_file, statement).ok()?; + Some(Dependency::Imported { + binding_statement: None, + alias_index: 0, + request: OwnedRequest { + module: module.to_string(), + member: Some(id.to_string()), + asname: None, + binds: id.to_string(), + }, + }) + } + _ => None, + }); + dependencies.push(imported.unwrap_or_else(|| Dependency::Sibling(id.to_string()))); + } + dependencies +} + +/// Whether `name` is private to its module by python's convention: it starts with an underscore +/// and is not a dunder. +fn is_private(name: &str) -> bool { + name.starts_with('_') && !(name.starts_with("__") && name.ends_with("__")) +} + +/// Whether `file` already imports what `request` asks for, under the name it asks for. +fn target_imports_same( + db: &dyn Db, + file: ProgramFile<'_>, + parsed: &ParsedModuleRef, + request: &OwnedRequest, +) -> bool { + let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); + parsed.suite().iter().any(|stmt| match stmt { + Stmt::Import(import) => { + request.member.is_none() + && import.names.iter().any(|alias| { + alias.name.as_str() == request.module + && alias.asname.as_ref().map(ToString::to_string) == request.asname + }) + } + Stmt::ImportFrom(from) => request.member.as_ref().is_some_and(|member| { + ModuleName::from_import_statement(db, importing_file, from) + .is_ok_and(|module| module.as_str() == request.module) + && from.names.iter().any(|alias| { + alias.name.as_str() == member + && alias.asname.as_ref().map(ToString::to_string) == request.asname + }) + }), + _ => false, + }) +} + +/// Whether a top-level import of `file` imports `module`, or something from it. +fn imports_module( + db: &dyn Db, + file: ProgramFile<'_>, + parsed: &ParsedModuleRef, + module: &ModuleName, +) -> bool { + let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); + parsed.suite().iter().any(|stmt| match stmt { + Stmt::Import(import) => import + .names + .iter() + .any(|alias| alias.name.as_str() == module.as_str()), + Stmt::ImportFrom(from) => ModuleName::from_import_statement(db, importing_file, from) + .is_ok_and(|imported| { + &imported == module + || from.names.iter().any(|alias| { + format!("{imported}.{}", alias.name.as_str()) == module.as_str() + }) + }), + _ => false, + }) +} + +/// Whether `name` is read anywhere in the module outside `except`, or listed in its `__all__`. +fn used_elsewhere(parsed: &ParsedModuleRef, name: &str, except: TextRange) -> bool { + struct Uses<'a> { + name: &'a str, + except: TextRange, + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for Uses<'_> { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) + if name.id.as_str() == self.name + && !self.except.contains_range(name.range()) => + { + self.found = true; + } + Expr::StringLiteral(string) + if string.value.to_str() == self.name + && !self.except.contains_range(string.range()) => + { + // a name listed in `__all__`, or otherwise spelled out, is kept + self.found = true; + } + _ => walk_expr(self, expr), + } + } + } + let mut uses = Uses { + name, + except, + found: false, + }; + uses.visit_body(parsed.suite()); + uses.found +} + +/// The text a cut takes, and the range it removes from the module. +struct Cut { + /// The definition with its decorators and the comments directly above it, ending in a line + /// break. + text: String, + /// Its lines and the blank lines that separated it from the statement after it, or, when it is + /// the last statement, from the one before. + removed: TextRange, +} + +fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { + let statement = &suite[index]; + let floor = index + .checked_sub(1) + .map_or(TextSize::default(), |previous| { + source.full_line_end(suite[previous].end()) + }); + let mut start = source.line_start(statement.start()); + // the comments directly above it go with it + while start > floor { + let previous = source.line_start(start - TextSize::from(1)); + if !source[TextRange::new(previous, start)] + .trim_start() + .starts_with('#') + { + break; + } + start = previous; + } + let end = source.full_line_end(statement.end()); + let mut text = source[TextRange::new(start, end)].to_string(); + if !text.ends_with(['\n', '\r']) { + text.push('\n'); + } + + let removed = if let Some(next) = suite.get(index + 1) { + // up to the next statement, or the comments above it + let mut next_start = source.line_start(next.start()); + while next_start > end { + let previous = source.line_start(next_start - TextSize::from(1)); + if source[TextRange::new(previous, next_start)] + .trim() + .is_empty() + { + break; + } + next_start = previous; + } + let mut removed_end = end; + while removed_end < next_start + && source[TextRange::new(removed_end, source.full_line_end(removed_end))] + .trim() + .is_empty() + { + removed_end = source.full_line_end(removed_end); + } + TextRange::new(start, removed_end) + } else { + // the last statement takes the blank lines above it, and everything after it + let mut removed_start = start; + while removed_start > floor { + let previous = source.line_start(removed_start - TextSize::from(1)); + if !source[TextRange::new(previous, removed_start)] + .trim() + .is_empty() + { + break; + } + removed_start = previous; + } + TextRange::new(removed_start, TextSize::of(source)) + }; + Cut { text, removed } +} + +/// The text appended to the end of the target: the definition, set apart from what is above it +/// as PEP 8 sets apart a top-level definition. +fn appended_text( + target: &str, + suite: &[Stmt], + definition: &str, + kind: MovableKind, + adds_imports: bool, + line_ending: &str, +) -> String { + let mut text = String::new(); + if !target.is_empty() && !target.ends_with(['\n', '\r']) { + text.push_str(line_ending); + } + let trailing_blank_lines = target + .lines() + .rev() + .take_while(|line| line.trim().is_empty()) + .count(); + let follows_a_variable = matches!( + suite.last(), + Some(Stmt::Assign(_) | Stmt::AnnAssign(_) | Stmt::AugAssign(_)) + ); + let blank_lines = if (suite.is_empty() && !adds_imports) + || (kind == MovableKind::Variable && follows_a_variable) + { + 0 + } else { + 2 + }; + for _ in trailing_blank_lines.min(blank_lines)..blank_lines { + text.push_str(line_ending); + } + text.push_str(definition); + text +} + +/// The edit removing alias `alias_index` from the import statement starting at `statement`, or the +/// whole statement when it is its only alias. +fn remove_alias( + source: &str, + suite: &[Stmt], + statement: TextSize, + alias_index: usize, +) -> Option { + let (position, stmt) = suite + .iter() + .enumerate() + .find(|(_, stmt)| stmt.start() == statement)?; + let names = match stmt { + Stmt::Import(import) => &import.names, + Stmt::ImportFrom(from) => &from.names, + _ => return None, + }; + if names.len() == 1 { + return Some(Edit::range_deletion( + cut_range(source, suite, position).removed, + )); + } + let alias = names.get(alias_index)?; + let range = if let Some(next) = names.get(alias_index + 1) { + TextRange::new(alias.start(), next.start()) + } else { + TextRange::new(names[alias_index - 1].end(), alias.end()) + }; + Some(Edit::range_deletion(range)) +} + +/// The `__all__ = [...]` (or tuple) assignment at the top level of a module. +fn all_list(suite: &[Stmt]) -> Option<&[Expr]> { + suite.iter().find_map(|stmt| { + let (target, value) = match stmt { + Stmt::Assign(assign) => (assign.targets.first()?, assign.value.as_ref()), + Stmt::AnnAssign(assign) => (assign.target.as_ref(), assign.value.as_deref()?), + _ => return None, + }; + if !matches!(target, Expr::Name(name) if name.id.as_str() == "__all__") { + return None; + } + match value { + Expr::List(list) => Some(list.elts.as_slice()), + Expr::Tuple(tuple) => Some(tuple.elts.as_slice()), + _ => None, + } + }) +} + +/// The edit taking `name` out of the module's `__all__`, when it lists it. +fn remove_from_all(suite: &[Stmt], name: &str) -> Option { + let elements = all_list(suite)?; + let position = elements.iter().position( + |element| matches!(element, Expr::StringLiteral(string) if string.value.to_str() == name), + )?; + let element = &elements[position]; + let range = if let Some(next) = elements.get(position + 1) { + TextRange::new(element.start(), next.start()) + } else if position > 0 { + TextRange::new(elements[position - 1].end(), element.end()) + } else { + element.range() + }; + Some(Edit::range_deletion(range)) +} + +/// The edit adding `name` to the end of the module's `__all__`, when it has one. +fn add_to_all(source: &str, suite: &[Stmt], name: &str) -> Option { + let elements = all_list(suite)?; + let quote = elements + .first() + .and_then(|element| source[element.range()].chars().next()) + .filter(|quote| matches!(quote, '"' | '\'')) + .unwrap_or('"'); + elements + .last() + .map(|last| Edit::insertion(format!(", {quote}{name}{quote}"), last.end())) +} + +/// Points the references `ranges` in `file` at `name`'s new module. +fn rewrite_references( + db: &dyn Db, + file: ProgramFile<'_>, + ranges: &[TextRange], + name: &str, + source_module: &ModuleName, + target_module: &ModuleName, + moved: &mut SymbolMove, +) { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let text = source_text(db, file.file(db)); + let text = text.as_str(); + let stylist = Stylist::from_tokens(parsed.tokens(), text); + let importer = import_order::importer(db, file, &parsed); + let model = SemanticModel::new(db, file); + let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); + let file_id = file.file(db); + let path = file.file(db).path(db).as_str().to_string(); + + let mut actions: Vec = Vec::new(); + let mut imported_by_statement = false; + let mut names = Vec::new(); + for range in ranges { + let covering = covering_node(parsed.syntax().into(), *range); + // the name of an attribute is an identifier of the attribute expression + let node = match covering.node() { + AnyNodeRef::Identifier(_) => match covering.parent() { + Some(attribute @ AnyNodeRef::ExprAttribute(_)) => attribute, + _ => covering.node(), + }, + node => node, + }; + match node { + // `from module import name`, as a statement of its own or among others + AnyNodeRef::Alias(_) | AnyNodeRef::Identifier(_) => { + let Some(from) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtImportFrom(from) => Some(from), + _ => None, + }) else { + continue; + }; + imported_by_statement = true; + let Ok(module) = ModuleName::from_import_statement(db, importing_file, from) else { + continue; + }; + if &module != source_module { + // imported through a module that re-exports it, which keeps doing so + continue; + } + let Some((alias_index, alias)) = from + .names + .iter() + .enumerate() + .find(|(_, alias)| alias.name.range() == *range) + else { + continue; + }; + if from.names.len() == 1 { + if let Some(module_range) = from_module_range(text, from) { + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::range_replacement(target_module.to_string(), module_range), + }); + } + } else { + let range = if let Some(next) = from.names.get(alias_index + 1) { + TextRange::new(alias.start(), next.start()) + } else { + TextRange::new(from.names[alias_index - 1].end(), alias.end()) + }; + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::range_deletion(range), + }); + let request = match &alias.asname { + Some(asname) => ImportRequest::import_from_as( + target_module.as_str(), + name, + asname.as_str(), + ), + None => ImportRequest::import_from(target_module.as_str(), name), + }; + if parsed.suite().iter().any(|stmt| stmt.range() == from.range) { + actions.push(importer.import(request, &MembersInScope::empty(from.end()))); + } else { + // an import inside a function: the new one goes on the line after it + let indentation = + &text[TextRange::new(text.line_start(from.start()), from.start())]; + let statement = match &alias.asname { + Some(asname) => { + format!("from {target_module} import {name} as {asname}") + } + None => format!("from {target_module} import {name}"), + }; + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::insertion( + format!( + "{indentation}{statement}{}", + stylist.line_ending().as_str() + ), + text.full_line_end(from.end()), + ), + }); + } + } + } + // `module.name`, through an import of the module + AnyNodeRef::ExprAttribute(attribute) + if attribute.attr.range() == *range + && matches!( + attribute.value.inferred_type(&model), + Some(Type::ModuleLiteral(literal)) if literal.module(db).name(db) == source_module + ) => + { + let action = importer.import( + ImportRequest::import(target_module.as_str(), name), + &MembersInScope::empty(attribute.start()), + ); + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::range_replacement( + action.symbol_text().to_string(), + attribute.range(), + ), + }); + actions.push(action); + } + AnyNodeRef::ExprName(_) => names.push(*range), + _ => { + let line = line_number(text, range.start()); + moved.warnings.push(format!( + "{path}:{line} refers to `{name}` in a way the move leaves as it is" + )); + } + } + } + + // a name the file reads without importing it by name reaches it through a star import + if !imported_by_statement && let Some(first) = names.iter().map(Ranged::start).min() { + let star = parsed.suite().iter().any(|stmt| { + matches!(stmt, Stmt::ImportFrom(from) + if from.names.iter().any(|alias| alias.name.as_str() == "*") + && ModuleName::from_import_statement(db, importing_file, from) + .is_ok_and(|module| &module == source_module)) + }); + if star { + actions.push(importer.import( + ImportRequest::import_from(target_module.as_str(), name), + &MembersInScope::empty(first), + )); + } else { + let line = line_number(text, first); + moved.warnings.push(format!( + "{path}:{line} reaches `{name}` through a module that will no longer provide it" + )); + } + } + + moved.edits.extend( + importer + .combined_edits(&actions) + .into_iter() + .map(|edit| FileEdit { + file: file_id, + edit, + }), + ); +} + +/// The range of the module a `from` import names, its leading dots included. +fn from_module_range(source: &str, from: &ast::StmtImportFrom) -> Option { + let start = from.start() + TextSize::try_from(source[from.range()].find("from")? + 4).ok()?; + let end = from.names.first()?.start(); + let between = &source[TextRange::new(start, end)]; + let keyword = between + .rfind(" import") + .or_else(|| between.rfind(" export"))?; + let module = &between[..keyword]; + let leading = module.len() - module.trim_start().len(); + let trailing = module.trim_end().len(); + Some(TextRange::new( + start + TextSize::try_from(leading).ok()?, + start + TextSize::try_from(trailing).ok()?, + )) +} + +/// The one-based line `offset` is on. +fn line_number(source: &str, offset: TextSize) -> usize { + source[..offset.to_usize()].matches('\n').count() + 1 +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use insta::assert_snapshot; + use ruff_db::files::system_path_to_file; + use ruff_db::source::source_text; + use ruff_text_size::Ranged; + + use super::move_symbol; + use crate::tests::{CursorTest, CursorTestBuilder}; + + const SRC: &str = r#"import math +from collections import OrderedDict + +__all__ = ["RATE", "area", "helper"] + +RATE = 2 +_SECRET = 3 + + +def helper(x): + return x * RATE + + +@staticmethod +def area(r): + """Area of a circle.""" + return math.pi * r * r * RATE + + +def uses_private(x): + return x + _SECRET + + +def free(x): + return OrderedDict(x=x) +"#; + + const DST: &str = r#""""Destination.""" + +import os + +X = os.sep +"#; + + /// A project of the modules above and modules using `area`, with the cursor where `src` + /// gives it, in `mv/src.py`. + fn project(src: &str) -> CursorTest { + CursorTestBuilder::default() + .source("mv/__init__.py", "") + .source("mv/src.py", src) + .source("mv/dst.py", DST) + .source( + "mv/user1.py", + "from mv.src import area, helper\n\nprint(area(1), helper(2))\n", + ) + .source("mv/user2.py", "import mv.src\n\nprint(mv.src.area(1))\n") + .source("mv/user3.py", "from mv import src\n\nprint(src.area(1))\n") + .source("mv/user4.py", "from mv.src import *\n\nprint(area(1))\n") + .source( + "mv/user5.py", + "from mv.src import area as a\n\nprint(a(1))\n", + ) + .build() + } + + impl CursorTest { + /// Every module the move to `mv/dst.py` changes, as it is afterwards, or why it refuses. + fn move_to_dst(&self) -> String { + let target = system_path_to_file(&self.db, "mv/dst.py").expect("dst to exist"); + let moved = salsa::attach(&self.db, || { + move_symbol( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + target, + ) + }); + let moved = match moved { + Ok(moved) => moved, + Err(reason) => return format!("refused: {reason}"), + }; + let mut files: Vec<_> = moved.edits.iter().map(|edit| edit.file).collect(); + files.sort_by_key(|file| file.path(&self.db).as_str().to_string()); + files.dedup(); + let mut out = String::new(); + for file in files { + let mut text = source_text(&self.db, file).as_str().to_string(); + let mut edits: Vec<_> = moved + .edits + .iter() + .filter(|edit| edit.file == file) + .enumerate() + .collect(); + // applied from the end; of two at one place, the one listed first ends up first + edits.sort_by_key(|(order, edit)| { + ( + std::cmp::Reverse(edit.edit.start()), + std::cmp::Reverse(*order), + ) + }); + for (_, edit) in edits { + text.replace_range( + edit.edit.range().to_std_range(), + edit.edit.content().unwrap_or_default(), + ); + } + let _ = write!(out, "--- {}\n{text}", file.path(&self.db)); + } + for warning in moved.warnings { + let _ = writeln!(out, "warning: {warning}"); + } + out + } + } + + /// The definition goes to the end of the target with the import it uses and the name of + /// its old module it reads; the old module drops the import nothing else uses and its + /// `__all__` entry; and every way the project refers to it follows it. + #[test] + fn a_function_moves_with_what_it_reads() { + let test = project(&SRC.replace("def area(r)", "def area(r)")); + assert_snapshot!(test.move_to_dst(), @r#" + --- /mv/dst.py + """Destination.""" + + import math + import os + + from mv.src import RATE + + X = os.sep + + + @staticmethod + def area(r): + """Area of a circle.""" + return math.pi * r * r * RATE + --- /mv/src.py + from collections import OrderedDict + + __all__ = ["RATE", "helper"] + + RATE = 2 + _SECRET = 3 + + + def helper(x): + return x * RATE + + + def uses_private(x): + return x + _SECRET + + + def free(x): + return OrderedDict(x=x) + --- /mv/user1.py + from mv.dst import area + from mv.src import helper + + print(area(1), helper(2)) + --- /mv/user2.py + import mv.dst + import mv.src + + print(mv.dst.area(1)) + --- /mv/user3.py + import mv.dst + from mv import src + + print(mv.dst.area(1)) + --- /mv/user4.py + from mv.dst import area + from mv.src import * + + print(area(1)) + --- /mv/user5.py + from mv.dst import area as a + + print(a(1)) + "#); + } + + /// The old module still calls it, and it reads `RATE` from the old module, so the two + /// modules would import each other, which python cannot load. + #[test] + fn a_move_that_makes_two_modules_import_each_other_is_refused() { + let test = project(&format!( + "{}\n\ndef user():\n return area(2)\n", + SRC.replace("def area(r)", "def area(r)") + )); + assert_snapshot!(test.move_to_dst(), @"refused: mv.src and mv.dst would import each other: `area` uses `RATE`, and mv.src still uses `area`"); + } + + #[test] + fn a_definition_that_reads_a_private_name_is_refused() { + let test = project(&SRC.replace("def uses_private", "def uses_private")); + assert_snapshot!(test.move_to_dst(), @"refused: `uses_private` uses `_SECRET`, which is private to mv.src; make it public, or move it too"); + } + + /// A variable the old module still reads is imported back from its new module. + #[test] + fn a_variable_the_old_module_reads_is_imported_back() { + let test = project(&SRC.replace("RATE = 2", "RATE = 2")); + assert_snapshot!(test.move_to_dst(), @r#" + --- /mv/dst.py + """Destination.""" + + import os + + X = os.sep + RATE = 2 + --- /mv/src.py + import math + from collections import OrderedDict + + from mv.dst import RATE + + __all__ = ["area", "helper"] + + _SECRET = 3 + + + def helper(x): + return x * RATE + + + @staticmethod + def area(r): + """Area of a circle.""" + return math.pi * r * r * RATE + + + def uses_private(x): + return x + _SECRET + + + def free(x): + return OrderedDict(x=x) + "#); + } + + /// Asked at a reference, the definition the reference is to is moved. + #[test] + fn asked_at_a_reference() { + let test = CursorTestBuilder::default() + .source("mv/__init__.py", "") + .source("mv/src.py", SRC) + .source("mv/dst.py", DST) + .source( + "mv/user.py", + "from mv.src import helper\n\nprint(helper(2))\n", + ) + .build(); + assert_snapshot!(test.move_to_dst(), @r#" + --- /mv/dst.py + """Destination.""" + + import os + + from mv.src import RATE + + X = os.sep + + + def helper(x): + return x * RATE + --- /mv/src.py + import math + from collections import OrderedDict + + __all__ = ["RATE", "area"] + + RATE = 2 + _SECRET = 3 + + + @staticmethod + def area(r): + """Area of a circle.""" + return math.pi * r * r * RATE + + + def uses_private(x): + return x + _SECRET + + + def free(x): + return OrderedDict(x=x) + --- /mv/user.py + from mv.dst import helper + + print(helper(2)) + "#); + } + + #[test] + fn a_name_the_target_binds_is_refused() { + let test = CursorTestBuilder::default() + .source("mv/__init__.py", "") + .source("mv/src.py", "def f():\n pass\n") + .source("mv/dst.py", "f = 1\n") + .build(); + assert_snapshot!(test.move_to_dst(), @"refused: mv.dst has a `f` already"); + } +} diff --git a/crates/ty_python_semantic/src/importer.rs b/crates/ty_python_semantic/src/importer.rs index f661a44528..01cc458537 100644 --- a/crates/ty_python_semantic/src/importer.rs +++ b/crates/ty_python_semantic/src/importer.rs @@ -1098,8 +1098,11 @@ pub struct MembersInScope<'ast> { impl<'ast> MembersInScope<'ast> { /// An empty scope for importing a name that is already known to be unbound. - /// This avoids querying inferred types while constructing a diagnostic fix. - pub(crate) fn empty(at: TextSize) -> Self { + /// This avoids querying inferred types while constructing a diagnostic fix, and serves a + /// refactoring that has checked itself that the names it imports are free where it imports + /// them, as one moving a definition does: the definition is still in scope where it is + /// moving from, and would otherwise be taken for a conflict. + pub fn empty(at: TextSize) -> Self { Self { at, map: FxHashMap::default(), diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 95b788446d..ca5b978d53 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -240,6 +240,15 @@ pub(super) fn request(req: server::Request) -> Task { requests::DeclarationRequestHandler::METHOD => background_document_request_task::< requests::DeclarationRequestHandler, >(req, BackgroundSchedule::Worker), + requests::PrepareMoveSymbolRequestHandler::METHOD => background_document_request_task::< + requests::PrepareMoveSymbolRequestHandler, + >( + req, BackgroundSchedule::Worker + ), + // searches the whole project for references, like a rename + requests::MoveSymbolRequestHandler::METHOD => background_document_request_task::< + requests::MoveSymbolRequestHandler, + >(req, BackgroundSchedule::Worker), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 451504525a..7ee8ecd521 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -47,6 +47,7 @@ mod hover; mod injections; mod inlay_hints; mod module_name; +mod move_symbol; mod on_type_formatting; mod overridable_members; mod override_members; @@ -110,6 +111,7 @@ pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; +pub(super) use move_symbol::{MoveSymbolRequestHandler, PrepareMoveSymbolRequestHandler}; pub(super) use on_type_formatting::OnTypeFormattingRequestHandler; pub(super) use overridable_members::OverridableMembersRequestHandler; pub(super) use override_members::OverrideMembersRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/move_symbol.rs b/crates/ty_server/src/server/api/requests/move_symbol.rs new file mode 100644 index 0000000000..2e496efbed --- /dev/null +++ b/crates/ty_server/src/server/api/requests/move_symbol.rs @@ -0,0 +1,248 @@ +//! `by/prepareMoveSymbol` and `by/moveSymbol`: moving a module's top-level function, class or +//! variable to another module, with every reference to it in the project following it. LSP has no +//! request for a move whose target the user picks: a code action carries no argument, and +//! `workspace/willRenameFiles` moves files. see [`ty_ide::move_symbol`] for what is moved and what +//! is refused + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, TextEdit, + Uri, WorkspaceEdit, +}; +use ruff_db::files::{FileRange, system_path_to_file}; +use ruff_db::system::SystemPathBuf; +use ruff_text_size::Ranged; +use rustc_hash::FxHashMap; +use ty_ide::{MovableKind, movable_symbol, move_symbol}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{FileRangeExt, PositionExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum PrepareMoveSymbolRequest {} + +impl Request for PrepareMoveSymbolRequest { + type Params = PrepareMoveSymbolParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareMoveSymbol"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// a position in a top-level definition's name, or in a reference to it +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PrepareMoveSymbolParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// the definition a move would move +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MovableSymbol { + name: String, + /// `function`, `class` or `variable` + kind: String, + /// the dotted name of the module it is defined in + module: String, + /// the file it is defined in, which need not be the document asked about + uri: Uri, + /// its name, where it is defined + range: Range, +} + +pub(crate) struct PrepareMoveSymbolRequestHandler; + +impl RequestHandler for PrepareMoveSymbolRequestHandler { + type RequestType = PrepareMoveSymbolRequest; +} + +impl BackgroundDocumentRequestHandler for PrepareMoveSymbolRequestHandler { + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &PrepareMoveSymbolParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` where there is no top-level function, class or variable of a project module to + /// move, when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PrepareMoveSymbolParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + if !file.source_type(db).is_py_file() { + return Ok(None); + } + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let Some(symbol) = movable_symbol(db, db.program_file(file), offset) else { + return Ok(None); + }; + let Some(location) = FileRange::new(symbol.file, symbol.name_range) + .to_lsp_range(db, snapshot.encoding()) + .and_then(|range| range.to_location()) + else { + return Ok(None); + }; + Ok(Some(MovableSymbol { + name: symbol.name, + kind: match symbol.kind { + MovableKind::Function => "function", + MovableKind::Class => "class", + MovableKind::Variable => "variable", + } + .to_string(), + module: symbol.module, + uri: location.uri, + range: location.range, + })) + } +} + +impl RetriableRequestHandler for PrepareMoveSymbolRequestHandler {} + +pub(crate) enum MoveSymbolRequest {} + +impl Request for MoveSymbolRequest { + type Params = MoveSymbolParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/moveSymbol"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the definition to move, named as for `by/prepareMoveSymbol`, and the module to move it to +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct MoveSymbolParams { + text_document: TextDocumentIdentifier, + position: Position, + /// an existing `.py` or `.by` file of the project, of the same kind as the definition's + target: Uri, +} + +/// either the edit that moves it, or why it is not moved +#[derive(Debug, Default, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MoveSymbolAnswer { + /// the edits to every file the move changes; absent when refused + #[serde(skip_serializing_if = "Option::is_none")] + edit: Option, + /// why the definition is not moved; absent when it is + #[serde(skip_serializing_if = "Option::is_none")] + refused: Option, + /// references the edit leaves as they are, which may no longer reach the definition, each + /// with the file and line it is on + warnings: Vec, +} + +pub(crate) struct MoveSymbolRequestHandler; + +impl RequestHandler for MoveSymbolRequestHandler { + type RequestType = MoveSymbolRequest; +} + +impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &MoveSymbolParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: MoveSymbolParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + if !file.source_type(db).is_py_file() { + return Ok(None); + } + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let refused = |reason: &str| { + Ok(Some(MoveSymbolAnswer { + refused: Some(reason.to_string()), + ..MoveSymbolAnswer::default() + })) + }; + let Some(target) = params + .target + .to_file_path() + .ok() + .and_then(|path| SystemPathBuf::from_path_buf(path).ok()) + .and_then(|path| system_path_to_file(db, &path).ok()) + else { + return refused("the target is not a file that exists"); + }; + + let moved = match move_symbol(db, db.program_file(file), offset, target) { + Ok(moved) => moved, + Err(reason) => return refused(&reason), + }; + let mut changes: FxHashMap> = FxHashMap::default(); + for file_edit in moved.edits { + let Some(location) = FileRange::new(file_edit.file, file_edit.edit.range()) + .to_lsp_range(db, snapshot.encoding()) + .and_then(|range| range.to_location()) + else { + return refused("a file the move changes cannot be edited"); + }; + changes.entry(location.uri).or_default().push(TextEdit { + range: location.range, + new_text: file_edit.edit.content().unwrap_or_default().to_string(), + }); + } + Ok(Some(MoveSymbolAnswer { + edit: Some(WorkspaceEdit { + changes: Some(changes.into_iter().collect()), + document_changes: None, + change_annotations: None, + }), + refused: None, + warnings: moved.warnings, + })) + } +} + +impl RetriableRequestHandler for MoveSymbolRequestHandler { + /// the user is waiting on the move they asked for + const RETRY_ON_CANCELLATION: bool = true; +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 4b9dd3e936..1019b2a0e8 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -53,6 +53,7 @@ mod implementation; mod initialize; mod injections; mod inlay_hints; +mod move_symbol; mod named_text; mod notebook; mod on_type_formatting; diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs new file mode 100644 index 0000000000..1eea0db91d --- /dev/null +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -0,0 +1,159 @@ +//! `by/prepareMoveSymbol` and `by/moveSymbol` over the wire +//! +//! what a move does to the project, and when it refuses, is `ty_ide`'s and its unit tests cover +//! it. what these cover is the contract a client reads: the definition a position names, the +//! workspace edit of a move, and the reason a move is refused + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use serde_json::json; + +use crate::{TestServer, TestServerBuilder}; + +/// `by/prepareMoveSymbol`, in json throughout +enum PrepareMoveSymbol {} + +impl Request for PrepareMoveSymbol { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareMoveSymbol"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/moveSymbol`, in json throughout +enum MoveSymbol {} + +impl Request for MoveSymbol { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/moveSymbol"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const UTIL: &str = "\ +RATE = 2 + + +def area(r: float) -> float: + return r * r * RATE + + +def _private() -> int: + return 1 + + +def uses_private() -> int: + return _private() +"; + +const MAIN: &str = "\ +from shapes.util import area + +print(area(1)) +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes/__init__.py"), "")? + .with_file(SystemPath::new("src/shapes/util.py"), UTIL)? + .with_file(SystemPath::new("src/shapes/geometry.py"), "")? + .with_file(SystemPath::new("src/main.py"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.py"), MAIN, 1); + Ok(server) +} + +#[test] +fn a_reference_names_the_definition_it_is_to() -> Result<()> { + let mut server = server()?; + let answer = server.send_request_await::(json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.py")) }, + "position": { "line": 2, "character": 7 }, + })); + assert_eq!( + answer, + Some(json!({ + "name": "area", + "kind": "function", + "module": "shapes.util", + "uri": server.file_uri(SystemPath::new("src/shapes/util.py")), + "range": { + "start": { "line": 3, "character": 4 }, + "end": { "line": 3, "character": 8 }, + }, + })) + ); + Ok(()) +} + +#[test] +fn a_move_answers_the_edit_to_every_file_it_changes() -> Result<()> { + let mut server = server()?; + let util = server.file_uri(SystemPath::new("src/shapes/util.py")); + let geometry = server.file_uri(SystemPath::new("src/shapes/geometry.py")); + let main = server.file_uri(SystemPath::new("src/main.py")); + let answer = server + .send_request_await::(json!({ + "textDocument": { "uri": main }, + "position": { "line": 2, "character": 7 }, + "target": geometry, + })) + .expect("an answer"); + assert_eq!(answer["warnings"], json!([]), "{answer:#}"); + assert!(answer.get("refused").is_none(), "{answer:#}"); + let changes = &answer["edit"]["changes"]; + assert_eq!( + changes[main.as_str()], + json!([{ + "range": { "start": { "line": 0, "character": 5 }, "end": { "line": 0, "character": 16 } }, + "newText": "shapes.geometry", + }]), + "{answer:#}" + ); + assert_eq!( + changes[geometry.as_str()], + json!([ + { + "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, + "newText": "from shapes.util import RATE\n", + }, + { + "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, + "newText": "\n\ndef area(r: float) -> float:\n return r * r * RATE\n", + }, + ]), + "{answer:#}" + ); + assert_eq!( + changes[util.as_str()], + json!([{ + "range": { "start": { "line": 3, "character": 0 }, "end": { "line": 7, "character": 0 } }, + "newText": "", + }]), + "{answer:#}" + ); + Ok(()) +} + +#[test] +fn a_refused_move_says_why() -> Result<()> { + let mut server = server()?; + server.open_text_document(SystemPath::new("src/shapes/util.py"), UTIL, 1); + let answer = server.send_request_await::(json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/shapes/util.py")) }, + "position": { "line": 11, "character": 5 }, + "target": server.file_uri(SystemPath::new("src/shapes/geometry.py")), + })); + assert_eq!( + answer, + Some(json!({ + "refused": "`uses_private` uses `_private`, which is private to shapes.util; make it \ + public, or move it too", + "warnings": [], + })) + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ae0783f73f..19d4df675c 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -570,6 +570,63 @@ a split list ends with a comma, which tells a formatter it is meant to stay spli and a joined one loses it unless it is a tuple of one. a list with a comment in it is refused, since joining would put what follows the comment inside it +### moving a definition + +a module's top-level function, class or variable moves to another module of the +project with `by/moveSymbol`, which a code action could not be: the user picks the +target. the definition is cut with its decorators and the comments directly above +it and appended to the target, and what it reads from its old module comes with +it — an import it uses is written in the target too, and dropped from the old module +when nothing else there uses it and no other module imports it from there, and another of the old module's names is imported +from there: + +```py +# shapes/util.py # shapes/geometry.py, after moving `area` +import math import math + +RATE = 2 from shapes.util import RATE + +def area(r): def area(r): + return math.pi * r * r * RATE return math.pi * r * r * RATE +``` + +every reference in the project follows it: `from shapes.util import area` imports +it from `shapes.geometry` (or, among other names, loses it to a new import), +`shapes.util.area` reads `shapes.geometry.area` with `import shapes.geometry` added, +a module that reached it through `from shapes.util import *` imports it by name, an +`__all__` that listed it moves it, and the old module imports it back when it still +uses it. each import added goes where the project's isort configuration sorts it + +the move is refused, with the reason, when the program would not run as before: + +- the definition reads a name private to its module, `_SECRET`, which nothing + outside the module is meant to import — make it public, or move it too +- the two modules would import each other, which python cannot load: the old module + still uses the definition and the definition reads the old module's names, or the + target imports the old module already +- the target binds the name already, or binds a name the definition reads to + something else +- the name is bound more than once in its module +- the definition assigns a name of its module through `global`, which would name + the target's once moved +- the definition reads a name its module binds more than once, which an import + would see only as it was when imported +- the definition reads an attribute of its module object, `__name__` or + `__file__`, which would be the target's +- the definition reads a builtin the target binds to something else +- the modules are not the same kind of file: basedpython does not move into python, + nor python into basedpython, which reads it differently + +`by/prepareMoveSymbol` takes `textDocument` and `position` — in a top-level +definition's name, or in a reference to one — and answers `{name, kind, module, uri, range}`: the name, `function`, `class` or `variable`, the dotted name of the module +it is defined in, and its name's range in the file `uri`, which need not be the +document asked about; `null` where there is nothing to move. `by/moveSymbol` takes +the same and `target`, the URI of an existing `.py` or `.by` file of the project +(an editor creates a new module before asking), and answers `{edit, warnings}`, a +`WorkspaceEdit` of `changes` to every file the move changes and the references it +left as they were, each with its file and line, or `{refused, warnings}`, the reason +nothing is moved. two edits at one position of a file apply in the order listed + ## unwrap and remove `by/unwrap` answers what an editor's *unwrap/remove* offers around a range, innermost From 1e54a6f974be2f3640ad16b6d38c5447f9aa03a6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:52:39 +1000 Subject: [PATCH 035/110] pull a class's members up into a superclass of the project with by/pullMembersUp and push them down into its subclasses with by/pushMembersDown, listed by by/prepareMoveMembers, moving the override modifier with them and refusing a move that leaves a member reading what its new class lacks, changes what another class inherits, or strands a use through the class itself --- crates/ty_ide/src/lib.rs | 5 +- crates/ty_ide/src/refactor.rs | 5 + crates/ty_ide/src/refactor/class_members.rs | 1724 +++++++++++++++++ crates/ty_ide/src/refactor/move_symbol.rs | 683 ++++++- crates/ty_python_semantic/src/lib.rs | 6 +- .../src/types/ide_support.rs | 167 +- crates/ty_server/src/server/api.rs | 13 + crates/ty_server/src/server/api/requests.rs | 4 + .../src/server/api/requests/move_members.rs | 289 +++ .../src/server/api/requests/move_symbol.rs | 43 +- crates/ty_server/tests/e2e/move_symbol.rs | 169 ++ docs/basedpython/features/editor.md | 53 + 12 files changed, 3047 insertions(+), 114 deletions(-) create mode 100644 crates/ty_ide/src/refactor/class_members.rs create mode 100644 crates/ty_server/src/server/api/requests/move_members.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index af43162436..d56068d6e0 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -110,8 +110,9 @@ pub use override_members::{ }; pub use pasted_imports::{CopiedCode, CopiedImport, PastedImports, copied_imports, pasted_imports}; pub use refactor::{ - MovableKind, MovableSymbol, Refactor, RefactorKind, RefactorOffer, SymbolMove, movable_symbol, - move_symbol, refactor, refactors, + ClassMember, HierarchyClass, MemberKind, MovableKind, MovableMembers, MovableSymbol, Refactor, + RefactorKind, RefactorOffer, SymbolMove, movable_members, movable_symbol, move_symbol, + pull_members_up, push_members_down, refactor, refactors, }; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 527f3a7023..5206ad57cd 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -15,6 +15,7 @@ //! hands its edits back rather than throwing them away, and the caller tells it //! which refactorings to consider rather than being given all of them. +mod class_members; mod data_class; mod evaluation; mod extract_function; @@ -40,6 +41,10 @@ use ty_python_semantic::SemanticModel; use crate::FileEdit; +pub use class_members::{ + ClassMember, HierarchyClass, MemberKind, MovableMembers, movable_members, pull_members_up, + push_members_down, +}; pub use move_symbol::{MovableKind, MovableSymbol, SymbolMove, movable_symbol, move_symbol}; /// The refactorings the server knows how to perform. diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs new file mode 100644 index 0000000000..283653b2f7 --- /dev/null +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -0,0 +1,1724 @@ +//! Pulling a class's members up into one of its superclasses, and pushing them down into its +//! subclasses. +//! +//! ```py +//! class Shape: class Shape: +//! pass def describe(self): +//! ──► return type(self).__name__ +//! class Square(Shape): +//! def describe(self): class Square(Shape): +//! return type(self).__name__ pass +//! ``` +//! +//! A member is every statement of a class body that binds its name: a method (with its property +//! accessors or overloads, which share the name), or a class attribute. It moves with its +//! decorators and the comments directly above it, reindented for the body it lands in. +//! +//! Either move is refused, with the reason, where the program would no longer mean what it +//! meant: +//! +//! - pulled up, a member that reads an attribute of its receiver the superclass does not have, +//! which the superclass's other subclasses would then miss; that uses `super()` from a class +//! whose base is not the superclass, since `super()` would then start from elsewhere; that +//! another subclass of the superclass inherits from further up already, and would inherit the +//! pulled one instead; or that the superclass binds already +//! - pushed down, a member something reads off the class itself or an instance that may be the +//! class rather than a subclass — another of its methods, through `self`, included — or of a +//! class without subclasses in the project +//! - either way, a name-mangled `__member`, whose name depends on the class it is written in, +//! and, between modules, a member that reads a name of its module that the other would have +//! to import from a module that imports it +//! +//! `override` follows the move: a method pulled up overrides nothing where it lands unless the +//! superclass inherits the name, so it loses the modifier there, and the methods of the same +//! name below the superclass then override it and gain the modifier; a method pushed down keeps +//! it only where something above still has the name. + +use ruff_db::files::File; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor, walk_expr}; +use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_python_codegen::Stylist; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashSet; +use ty_module_resolver::file_to_module; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_semantic::importer::{ImportAction, ImportRequest, MembersInScope}; +use ty_python_semantic::types::Type; +use ty_python_semantic::{ + HasType, ProgramEnvironment, SemanticModel, class_bases, instance_member_names, + is_within_classes, type_hierarchy_prepare, type_hierarchy_subtypes, +}; + +use super::move_symbol::{ + Dependency, OwnedRequest, SymbolMove, cut_range, dependencies, imports_module, line_number, +}; +use crate::code_action::FileEdit; +use crate::goto::find_goto_target; +use crate::import_order; +use crate::method_kind::{MethodKind, method_kind}; +use crate::references::{ReferencesMode, references}; + +/// What kind of member of a class a member is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MemberKind { + Method, + ClassMethod, + StaticMethod, + Property, + Attribute, +} + +/// A member [`pull_members_up`] or [`push_members_down`] can move. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClassMember { + pub name: String, + pub kind: MemberKind, + /// The range of its name in its first statement. + pub name_range: TextRange, +} + +/// A class of the project, where its name is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HierarchyClass { + pub file: File, + pub name: String, + /// Its module's dotted name and its own, the way [`pull_members_up`] is told which + /// superclass to pull into. + pub qualified_name: Option, + pub name_range: TextRange, +} + +/// The members of the class at a position, and the classes of the project above and below it +/// they could move to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MovableMembers { + pub class: HierarchyClass, + pub members: Vec, + /// The project's classes the class inherits from, nearest first. + pub superclasses: Vec, + /// The project's classes that name the class as a base. + pub subclasses: Vec, +} + +/// The class around `offset`, its members, and where they could go. +pub fn movable_members( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option { + let class = ClassSite::at(db, file, offset)?; + let def = class.def()?; + let mut members: Vec = Vec::new(); + for statement in &def.body { + let Some((name, kind)) = member_of(statement) else { + continue; + }; + if let Some(existing) = members + .iter_mut() + .find(|member| member.name == name.as_str()) + { + if kind == MemberKind::Property { + existing.kind = MemberKind::Property; + } + continue; + } + members.push(ClassMember { + name: name.to_string(), + kind, + name_range: member_name_range(statement)?, + }); + } + Some(MovableMembers { + class: class.item(db), + members, + superclasses: class + .ancestors(db) + .iter() + .map(|class| class.item(db)) + .collect(), + subclasses: class + .subclasses(db) + .iter() + .map(|class| class.item(db)) + .collect(), + }) +} + +/// Moves the members `names` of the class around `offset` into its superclass `target`, named by +/// its qualified name as [`movable_members`] lists it. `Err` says why they cannot be moved. +pub fn pull_members_up( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + names: &[String], + target: &str, +) -> Result { + let class = + ClassSite::at(db, file, offset).ok_or_else(|| "there is no class here".to_string())?; + let def = class + .def() + .ok_or_else(|| "there is no class here".to_string())?; + let ancestors = class.ancestors(db); + let superclass = ancestors + .iter() + .find(|ancestor| ancestor.qualified_name.as_deref() == Some(target)) + .ok_or_else(|| { + format!( + "{target} is not a class of the project {} inherits from", + class.name + ) + })?; + let super_def = superclass + .def() + .ok_or_else(|| format!("{target} cannot be read"))?; + let env = ProgramEnvironment::from_file(class.file); + let super_members = instance_member_names(db, &env, superclass.ty); + let pulled: FxHashSet<&str> = names.iter().map(String::as_str).collect(); + let source = source_text(db, class.file.file(db)); + let source = source.as_str(); + + let mut moved = SymbolMove::default(); + let mut texts: Vec<(usize, MemberKind, String)> = Vec::new(); + let mut removed: Vec = Vec::new(); + let mut target_imports: Vec = Vec::new(); + let mut overriding: Vec = Vec::new(); + for name in names { + refuse_mangled(name)?; + let statements = member_statements(def, name); + if statements.is_empty() { + return Err(format!("{} has no member `{name}`", class.name)); + } + if super_def + .body + .iter() + .any(|statement| binds(statement, name)) + || assigns_on_receiver(super_def, name) + { + return Err(format!("{} has a `{name}` already", superclass.name)); + } + let inherited_above = super_members.contains(&Name::new(name)); + // a class between the two that has the name would be what the class inherits instead + if let Some(between) = ancestors + .iter() + .take_while(|ancestor| !ancestor.same(superclass)) + .find(|ancestor| { + superclass.is_ancestor_of(db, ancestor) + && ancestor + .def() + .is_some_and(|def| !member_statements(def, name).is_empty()) + }) + { + return Err(format!( + "{} would inherit {}'s `{name}` rather than its own", + class.name, between.name + )); + } + for &index in &statements { + let statement = &def.body[index]; + // what it reads off its receiver has to be there in the superclass + for (attribute, range) in receiver_attributes(statement) { + if pulled.contains(attribute.as_str()) || super_members.contains(&attribute) { + continue; + } + refuse_mangled(&attribute)?; + let is_member = def + .body + .iter() + .any(|statement| binds(statement, &attribute)); + return Err(format!( + "`{name}` uses `{}` on line {}, which {} does not have{}", + &source[range], + line_number(source, range.start()), + superclass.name, + if is_member { "; pull it up too" } else { "" } + )); + } + for (sibling, _) in class_scope_reads(def, statement) { + if !pulled.contains(sibling.as_str()) && !super_members.contains(&sibling) { + return Err(format!( + "`{name}` reads {}'s `{sibling}`, which {} does not have; pull it up too", + class.name, superclass.name + )); + } + } + if uses_super(statement) && !class.has_only_base(db, superclass) { + return Err(format!( + "`{name}` calls `super()`, which would start from {} rather than from {}", + superclass.name, class.name + )); + } + if superclass.file_id != class.file_id { + let parsed = &class.parsed; + for dependency in dependencies(db, class.file, parsed, statement, name) { + match dependency { + Dependency::Imported { request, .. } => target_imports.push(request), + Dependency::Sibling(sibling) => { + let module = class.module(db).unwrap_or_default(); + return Err(format!( + "`{name}` uses `{sibling}`, which {} would have to import from \ + {module}, which imports it", + superclass.module(db).unwrap_or_default() + )); + } + } + } + } + texts.push(( + index, + member_kind(statement), + member_text( + source, + &def.body, + index, + !inherited_above && has_override(statement), + ), + )); + removed.push(index); + } + if inherited_above { + // another subclass of the superclass that inherits it from further up would + // inherit this one instead + for below in superclass.descendants(db) { + if below.same(&class) || class.is_ancestor_of(db, &below) { + continue; + } + if !below.inherits_own(db, name, superclass) { + return Err(format!( + "{} inherits `{name}` from above {}, and would inherit {}'s instead", + below.name, superclass.name, class.name + )); + } + } + } else { + overriding.push(name.clone()); + } + } + + // the subclass loses them, and the superclass gains them in the order they were written + texts.sort_by_key(|(index, _, _)| *index); + moved + .edits + .extend(remove_members(class.file.file(db), source, def, &removed)); + + // the superclass gains them, attributes before its methods and methods after them + let super_source = source_text(db, superclass.file.file(db)); + let super_source = super_source.as_str(); + let from_indent = body_indentation(source, def); + let to_indent = body_indentation(super_source, super_def); + moved.edits.extend(insert_members( + superclass.file.file(db), + super_source, + super_def, + &texts + .iter() + .map(|(_, kind, text)| (*kind, reindent(text, &from_indent, &to_indent))) + .collect::>(), + Stylist::from_tokens(superclass.parsed.tokens(), super_source) + .line_ending() + .as_str(), + )); + if !target_imports.is_empty() { + moved + .edits + .extend(add_imports(db, superclass, &target_imports)?); + } + + // what defines a name pulled up below the superclass now overrides it + for name in &overriding { + for below in superclass.descendants(db) { + if below.same(&class) || class.is_ancestor_of(db, &below) { + continue; + } + if let Some(below_def) = below.def() + && let Some(&first) = member_statements(below_def, name).first() + && !has_override(&below_def.body[first]) + { + moved + .edits + .extend(add_override(db, &below, &below_def.body[first])); + } + } + } + Ok(moved) +} + +/// Moves the members `names` of the class around `offset` into each of the project's classes +/// that name it as a base, as a copy into each that does not define the member itself. `Err` +/// says why they cannot be moved. +pub fn push_members_down( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + names: &[String], +) -> Result { + let class = + ClassSite::at(db, file, offset).ok_or_else(|| "there is no class here".to_string())?; + let def = class + .def() + .ok_or_else(|| "there is no class here".to_string())?; + let subclasses = class.subclasses(db); + if subclasses.is_empty() { + return Err(format!( + "no class of the project names {} as a base, so there is nowhere to push to", + class.name + )); + } + let env = ProgramEnvironment::from_file(class.file); + let subclass_types: Vec = subclasses.iter().map(|subclass| subclass.ty).collect(); + let inherited: FxHashSet = class_bases(db, &env, class.ty) + .into_iter() + .flat_map(|base| instance_member_names(db, &env, base)) + .collect(); + let source = source_text(db, class.file.file(db)); + let source = source.as_str(); + let model = SemanticModel::new(db, class.file); + + let mut moved = SymbolMove::default(); + let mut removed: Vec = Vec::new(); + let mut texts: Vec<(MemberKind, String, Vec)> = Vec::new(); + for name in names { + refuse_mangled(name)?; + let statements = member_statements(def, name); + let Some(&first) = statements.first() else { + return Err(format!("{} has no member `{name}`", class.name)); + }; + let inherited_above = inherited.contains(&Name::new(name)); + + // nothing may read it off the class, or what may be an instance of the class itself + let statement_ranges: Vec = statements + .iter() + .map(|&index| def.body[index].range()) + .collect(); + if let Some(name_range) = member_name_range(&def.body[first]) + && let Some(goto) = find_goto_target(&model, &class.parsed, name_range.start()) + { + for reference in references(db, class.file, &goto, ReferencesMode::RenameMultiFile) + .unwrap_or_default() + { + if reference.file() == class.file.file(db) + && statement_ranges + .iter() + .any(|range| range.contains_range(reference.range())) + { + continue; + } + let reference_file = ProgramFile::new(db, reference.file(), class.file.program(db)); + let Some(receiver) = attribute_receiver_type(db, reference_file, reference.range()) + else { + continue; + }; + let reference_env = ProgramEnvironment::from_file(reference_file); + if !is_within_classes(db, &reference_env, receiver, &subclass_types) { + let text = source_text(db, reference.file()); + let path = reference.file().path(db); + let root = db.project().root(db); + let path = path + .as_system_path() + .and_then(|path| path.strip_prefix(root).ok()) + .map_or_else(|| path.to_string(), ToString::to_string); + return Err(format!( + "{}:{} uses `{name}` on what may be a {} rather than a subclass", + path, + line_number(text.as_str(), reference.range().start()), + class.name + )); + } + } + } + + for &index in &statements { + let statement = &def.body[index]; + if uses_super(statement) { + for subclass in &subclasses { + if !subclass.has_only_base(db, &class) { + return Err(format!( + "`{name}` calls `super()`, which in {} would not start from {}", + subclass.name, class.name + )); + } + } + } + texts.push(( + member_kind(statement), + member_text( + source, + &def.body, + index, + !inherited_above && has_override(statement), + ), + vec![index], + )); + removed.push(index); + } + } + + moved + .edits + .extend(remove_members(class.file.file(db), source, def, &removed)); + + for subclass in &subclasses { + let Some(sub_def) = subclass.def() else { + continue; + }; + let sub_source = source_text(db, subclass.file.file(db)); + let sub_source = sub_source.as_str(); + let from_indent = body_indentation(source, def); + let to_indent = body_indentation(sub_source, sub_def); + let mut copies = Vec::new(); + let mut imports = Vec::new(); + for (kind, text, indices) in &texts { + let statement = &def.body[indices[0]]; + let Some((name, _)) = member_of(statement) else { + continue; + }; + let own = member_statements(sub_def, name.as_str()); + if let Some(&first) = own.first() { + // it keeps its own, which overrides nothing once the class no longer has it + let own_statement = &sub_def.body[first]; + if !inherited.contains(name) && has_override(own_statement) { + moved + .edits + .extend(remove_override(db, subclass, own_statement)); + } + continue; + } + copies.push((*kind, reindent(text, &from_indent, &to_indent))); + if subclass.file_id != class.file_id { + for dependency in dependencies(db, class.file, &class.parsed, statement, name) { + match dependency { + Dependency::Imported { request, .. } => imports.push(request), + Dependency::Sibling(sibling) => { + let module = class.module(db).unwrap_or_default(); + let sub_module = file_to_module(db, subclass.file.resolver_file(db)) + .map(|module| module.name(db).clone()); + if let Some(sub_module) = &sub_module + && imports_module(db, class.file, &class.parsed, sub_module) + { + return Err(format!( + "`{name}` uses `{sibling}`, which {sub_module} would have to \ + import from {module}, which imports it" + )); + } + imports.push(OwnedRequest { + module, + member: Some(sibling.clone()), + asname: None, + binds: sibling, + }); + } + } + } + } + } + if copies.is_empty() { + continue; + } + moved.edits.extend(insert_members( + subclass.file.file(db), + sub_source, + sub_def, + &copies, + Stylist::from_tokens(subclass.parsed.tokens(), sub_source) + .line_ending() + .as_str(), + )); + if !imports.is_empty() { + moved.edits.extend(add_imports(db, subclass, &imports)?); + } + } + Ok(moved) +} + +/// A class of the project, parsed. +struct ClassSite<'db> { + file: ProgramFile<'db>, + file_id: File, + parsed: ParsedModuleRef, + name: String, + name_range: TextRange, + ty: Type<'db>, + qualified_name: Option, +} + +impl<'db> ClassSite<'db> { + /// The innermost class whose definition `offset` is in. + fn at(db: &'db dyn Db, file: ProgramFile<'db>, offset: TextSize) -> Option { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let def = all_classes(&parsed) + .into_iter() + .filter(|class| class.range().contains_inclusive(offset)) + .min_by_key(|class| class.range().len())?; + let name_range = def.name.range(); + let ty = def.inferred_type(&SemanticModel::new(db, file))?; + Self::new(db, file, parsed, name_range, ty) + } + + /// The class whose name is at `name_range` in `file`, of the type `ty`. + fn new( + db: &'db dyn Db, + file: ProgramFile<'db>, + parsed: ParsedModuleRef, + name_range: TextRange, + ty: Type<'db>, + ) -> Option { + if !db.project().files(db).contains(file.file(db)) { + return None; + } + let def = all_classes(&parsed) + .into_iter() + .find(|class| class.name.range() == name_range)?; + let name = def.name.to_string(); + let qualified_name = file_to_module(db, file.resolver_file(db)) + .map(|module| format!("{}.{name}", module.name(db))); + Some(Self { + file, + file_id: file.file(db), + parsed, + name, + name_range, + ty, + qualified_name, + }) + } + + /// The class described by `class`, a class of the type hierarchy, of the type `ty`. + fn of_hierarchy( + db: &'db dyn Db, + program: ProgramFile<'db>, + class: &ty_python_semantic::TypeHierarchyClass<'db>, + ty: Type<'db>, + ) -> Option { + let file = ProgramFile::new(db, class.file.file(db), program.program(db)); + let parsed = parsed_module(db, file.python_file(db)).load(db); + Self::new(db, file, parsed, class.selection_range, ty) + } + + fn def(&self) -> Option<&ast::StmtClassDef> { + all_classes(&self.parsed) + .into_iter() + .find(|class| class.name.range() == self.name_range) + } + + fn same(&self, other: &ClassSite<'_>) -> bool { + self.file_id == other.file_id && self.name_range == other.name_range + } + + fn module(&self, db: &dyn Db) -> Option { + file_to_module(db, self.file.resolver_file(db)).map(|module| module.name(db).to_string()) + } + + fn item(&self, db: &dyn Db) -> HierarchyClass { + HierarchyClass { + file: self.file.file(db), + name: self.name.clone(), + qualified_name: self.qualified_name.clone(), + name_range: self.name_range, + } + } + + /// The project's classes this class names as its bases. + fn bases(&self, db: &'db dyn Db) -> Vec> { + let env = ProgramEnvironment::from_file(self.file); + class_bases(db, &env, self.ty) + .into_iter() + .filter_map(|base| { + let hierarchy = type_hierarchy_prepare(db, &env, base)?; + Self::of_hierarchy(db, self.file, &hierarchy, base) + }) + .collect() + } + + /// The project's classes this class inherits from, nearest first. + fn ancestors(&self, db: &'db dyn Db) -> Vec> { + let mut found: Vec> = Vec::new(); + let mut pending = self.bases(db); + pending.reverse(); + while let Some(next) = pending.pop() { + if found.iter().any(|seen| seen.same(&next)) || next.same(self) { + continue; + } + let mut bases = next.bases(db); + bases.reverse(); + found.push(next); + pending.extend(bases); + } + found + } + + /// Whether this class names `base` as its one base. + fn has_only_base(&self, db: &'db dyn Db, base: &ClassSite<'_>) -> bool { + let env = ProgramEnvironment::from_file(self.file); + let bases = class_bases(db, &env, self.ty); + bases.len() == 1 && self.bases(db).first().is_some_and(|only| only.same(base)) + } + + fn is_ancestor_of(&self, db: &'db dyn Db, other: &ClassSite<'db>) -> bool { + other + .ancestors(db) + .iter() + .any(|ancestor| ancestor.same(self)) + } + + /// The project's classes that name this class as a base. + fn subclasses(&self, db: &'db dyn Db) -> Vec> { + let env = ProgramEnvironment::from_file(self.file); + let modules: Vec<_> = + ty_module_resolver::all_modules(db, self.file.resolver_environment(db)) + .into_iter() + .filter(|module| { + module + .search_path(db) + .is_some_and(ty_module_resolver::SearchPath::is_first_party) + }) + .collect(); + type_hierarchy_subtypes(db, &env, self.ty, &modules) + .into_iter() + .filter_map(|class| { + let file = ProgramFile::new(db, class.file.file(db), self.file.program(db)); + let parsed = parsed_module(db, file.python_file(db)).load(db); + let def = all_classes(&parsed) + .into_iter() + .find(|def| def.name.range() == class.selection_range)?; + let ty = def.inferred_type(&SemanticModel::new(db, file))?; + Self::new(db, file, parsed, class.selection_range, ty) + }) + .collect() + } + + /// The project's classes below this one, at any depth. + fn descendants(&self, db: &'db dyn Db) -> Vec> { + let mut found: Vec> = Vec::new(); + let mut pending = self.subclasses(db); + while let Some(next) = pending.pop() { + if found.iter().any(|seen| seen.same(&next)) { + continue; + } + pending.extend(next.subclasses(db)); + found.push(next); + } + found + } + + /// Whether this class, or a class between it and `top`, defines `name` itself, so that it + /// does not inherit `name` from `top` or above. + fn inherits_own(&self, db: &'db dyn Db, name: &str, top: &ClassSite<'db>) -> bool { + if self + .def() + .is_some_and(|def| !member_statements(def, name).is_empty()) + { + return true; + } + self.ancestors(db) + .iter() + .take_while(|ancestor| !ancestor.same(top)) + .any(|ancestor| { + top.is_ancestor_of(db, ancestor) + && ancestor + .def() + .is_some_and(|def| !member_statements(def, name).is_empty()) + }) + } +} + +/// Every class definition of a module, at any depth. +fn all_classes(parsed: &ParsedModuleRef) -> Vec<&ast::StmtClassDef> { + #[derive(Default)] + struct Classes<'a> { + found: Vec<&'a ast::StmtClassDef>, + } + impl<'a> SourceOrderVisitor<'a> for Classes<'a> { + fn visit_stmt(&mut self, statement: &'a Stmt) { + if let Stmt::ClassDef(class) = statement { + self.found.push(class); + } + source_order::walk_stmt(self, statement); + } + fn visit_expr(&mut self, _expression: &'a Expr) {} + } + let mut classes = Classes::default(); + classes.visit_body(parsed.suite()); + classes.found +} + +/// The member a class body statement binds, and what kind of member it is. +fn member_of(statement: &Stmt) -> Option<(&Name, MemberKind)> { + match statement { + Stmt::FunctionDef(function) => Some((&function.name.id, member_kind(statement))), + Stmt::Assign(assign) => match assign.targets.as_slice() { + [Expr::Name(name)] => Some((&name.id, MemberKind::Attribute)), + _ => None, + }, + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(name) => Some((&name.id, MemberKind::Attribute)), + _ => None, + }, + _ => None, + } +} + +fn member_kind(statement: &Stmt) -> MemberKind { + let Stmt::FunctionDef(function) = statement else { + return MemberKind::Attribute; + }; + let is_property = function.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, Expr::Name(name) if name.id.as_str() == "property") + || matches!(&decorator.expression, Expr::Attribute(attribute) + if matches!(attribute.attr.as_str(), "setter" | "getter" | "deleter")) + }); + if is_property { + return MemberKind::Property; + } + match method_kind(function) { + MethodKind::Instance => MemberKind::Method, + MethodKind::Class => MemberKind::ClassMethod, + MethodKind::Static => MemberKind::StaticMethod, + } +} + +fn member_name_range(statement: &Stmt) -> Option { + match statement { + Stmt::FunctionDef(function) => Some(function.name.range()), + Stmt::Assign(assign) => assign.targets.first().map(Ranged::range), + Stmt::AnnAssign(assign) => Some(assign.target.range()), + _ => None, + } +} + +fn binds(statement: &Stmt, name: &str) -> bool { + member_of(statement).is_some_and(|(bound, _)| bound.as_str() == name) +} + +/// The indices of the statements of `class`'s body that bind `name`. +fn member_statements(class: &ast::StmtClassDef, name: &str) -> Vec { + class + .body + .iter() + .enumerate() + .filter(|(_, statement)| binds(statement, name)) + .map(|(index, _)| index) + .collect() +} + +fn refuse_mangled(name: &str) -> Result<(), String> { + if name.starts_with("__") && !name.ends_with("__") { + return Err(format!( + "`{name}` is name-mangled: its name depends on the class it is written in" + )); + } + Ok(()) +} + +/// Whether a method of `class` assigns `name` on its receiver: `self.name = ...`. +fn assigns_on_receiver(class: &ast::StmtClassDef, name: &str) -> bool { + class.body.iter().any(|statement| { + receiver_attributes(statement) + .iter() + .any(|(attribute, _)| attribute.as_str() == name) + && matches!(statement, Stmt::FunctionDef(_)) + && assigned_on_receiver(statement, name) + }) +} + +fn assigned_on_receiver(statement: &Stmt, name: &str) -> bool { + struct Stores<'a> { + receiver: &'a str, + name: &'a str, + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for Stores<'_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Attribute(attribute) = expr + && attribute.ctx.is_store() + && attribute.attr.as_str() == self.name + && matches!(&*attribute.value, Expr::Name(value) if value.id.as_str() == self.receiver) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let Some(receiver) = receiver_name(statement) else { + return false; + }; + let mut stores = Stores { + receiver: receiver.as_str(), + name, + found: false, + }; + stores.visit_stmt(statement); + stores.found +} + +/// The name of a method's receiver parameter, `self` or `cls`; `None` for a static method or +/// what is no method. +fn receiver_name(statement: &Stmt) -> Option { + let Stmt::FunctionDef(function) = statement else { + return None; + }; + if method_kind(function) == MethodKind::Static { + return None; + } + function + .parameters + .posonlyargs + .iter() + .chain(&function.parameters.args) + .next() + .map(|parameter| parameter.parameter.name.id.clone()) +} + +/// The attributes a method reads or writes on its receiver, `x` of `self.x`, with where. +fn receiver_attributes(statement: &Stmt) -> Vec<(Name, TextRange)> { + struct Attributes<'a> { + receiver: &'a str, + found: Vec<(Name, TextRange)>, + } + impl<'a> SourceOrderVisitor<'a> for Attributes<'_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Attribute(attribute) = expr + && matches!(&*attribute.value, Expr::Name(value) if value.id.as_str() == self.receiver) + && !self + .found + .iter() + .any(|(name, _)| name == attribute.attr.as_str()) + { + self.found + .push((attribute.attr.id.clone(), attribute.range())); + } + walk_expr(self, expr); + } + } + let Some(receiver) = receiver_name(statement) else { + return Vec::new(); + }; + let mut attributes = Attributes { + receiver: receiver.as_str(), + found: Vec::new(), + }; + if let Stmt::FunctionDef(function) = statement { + attributes.visit_body(&function.body); + } + attributes.found +} + +/// The names an attribute's value reads from the class body it is in: other members bound +/// before it, like `B` in `B = A + 1`. +fn class_scope_reads(class: &ast::StmtClassDef, statement: &Stmt) -> Vec<(Name, TextRange)> { + struct Loads { + found: Vec<(Name, TextRange)>, + } + impl<'a> SourceOrderVisitor<'a> for Loads { + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) if name.ctx.is_load() => { + self.found.push((name.id.clone(), name.range())); + } + // a lambda or comprehension does not see the class body + Expr::Lambda(_) + | Expr::ListComp(_) + | Expr::SetComp(_) + | Expr::DictComp(_) + | Expr::Generator(_) => {} + _ => walk_expr(self, expr), + } + } + } + let value = match statement { + Stmt::Assign(assign) => &*assign.value, + Stmt::AnnAssign(assign) => match &assign.value { + Some(value) => value, + None => return Vec::new(), + }, + _ => return Vec::new(), + }; + let mut loads = Loads { found: Vec::new() }; + loads.visit_expr(value); + loads + .found + .into_iter() + .filter(|(name, _)| { + class + .body + .iter() + .any(|other| !std::ptr::eq(other, statement) && binds(other, name)) + }) + .collect() +} + +/// Whether a method calls `super()`, or reads basedpython's `super`. +fn uses_super(statement: &Stmt) -> bool { + struct Super { + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for Super { + fn visit_expr(&mut self, expr: &'a Expr) { + if matches!(expr, Expr::Name(name) if name.id.as_str() == "super") { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut uses = Super { found: false }; + uses.visit_stmt(statement); + uses.found +} + +/// The `override` a method is marked with: basedpython's modifier, or `@override`. +fn override_marker(statement: &Stmt) -> Option<&ast::Decorator> { + let Stmt::FunctionDef(function) = statement else { + return None; + }; + function.decorator_list.iter().find(|decorator| { + matches!(&decorator.expression, Expr::Name(name) if name.id.as_str() == "override") + || matches!(&decorator.expression, Expr::Attribute(attribute) if attribute.attr.as_str() == "override") + }) +} + +fn has_override(statement: &Stmt) -> bool { + override_marker(statement).is_some() +} + +/// The range that takes `decorator` out: an `@override` line with its line break, or the +/// `override` modifier keyword with the space after it. +fn override_range(source: &str, decorator: &ast::Decorator) -> TextRange { + if source[decorator.range()].starts_with('@') { + TextRange::new( + source.line_start(decorator.start()), + source.full_line_end(decorator.end()), + ) + } else { + let spaces = source[decorator.end().to_usize()..] + .chars() + .take_while(|character| *character == ' ') + .count(); + TextRange::new( + decorator.start(), + decorator.end() + TextSize::try_from(spaces).unwrap_or_default(), + ) + } +} + +/// The text of the body statement `index`, as [`cut_range`] cuts it, without its `override` when +/// `strip_override`. +fn member_text(source: &str, body: &[Stmt], index: usize, strip_override: bool) -> String { + let statement = &body[index]; + let cut = cut_range(source, body, index); + if strip_override && let Some(decorator) = override_marker(statement) { + let local = override_range(source, decorator) - cut.start; + let mut text = cut.text; + text.replace_range(local.to_std_range(), ""); + return text; + } + cut.text +} + +/// The indentation the statements of `class`'s body are written with. +fn body_indentation(source: &str, class: &ast::StmtClassDef) -> String { + class.body.first().map_or_else( + || " ".to_string(), + |statement| { + let start = source.line_start(statement.start()); + source[TextRange::new(start, statement.start())].to_string() + }, + ) +} + +/// `text` with each line's leading `from` replaced by `to`. +fn reindent(text: &str, from: &str, to: &str) -> String { + if from == to { + return text.to_string(); + } + text.split_inclusive('\n') + .map(|line| match line.strip_prefix(from) { + Some(rest) => format!("{to}{rest}"), + None => line.to_string(), + }) + .collect() +} + +/// Whether a body statement stands in for no body: `pass` or `...`. +fn is_placeholder(statement: &Stmt) -> bool { + match statement { + Stmt::Pass(_) => true, + Stmt::Expr(expression) => matches!(*expression.value, Expr::EllipsisLiteral(_)), + _ => false, + } +} + +fn is_docstring(statement: &Stmt) -> bool { + matches!(statement, Stmt::Expr(expression) if expression.value.is_string_literal_expr()) +} + +/// The edits taking the statements `removed` out of `class`'s body, leaving `pass` when nothing +/// but a docstring would be left. +fn remove_members( + file: File, + source: &str, + class: &ast::StmtClassDef, + removed: &[usize], +) -> Vec { + let left = class + .body + .iter() + .enumerate() + .filter(|(index, statement)| !removed.contains(index) && !is_docstring(statement)) + .count(); + let mut edits = Vec::new(); + let mut ranges: Vec = removed + .iter() + .map(|&index| cut_range(source, &class.body, index).removed) + .collect(); + ranges.sort_by_key(Ranged::start); + // two adjacent cuts that would overlap at a blank line are one + let mut merged: Vec = Vec::new(); + for range in ranges { + match merged.last_mut() { + Some(last) if last.end() >= range.start() => *last = last.cover(range), + _ => merged.push(range), + } + } + // cuts that end the body take the blank lines above them, as the last one alone does + let body_end = class.body.last().map_or(class.end(), Ranged::end); + for range in &mut merged { + if range.end() >= body_end { + let mut start = range.start(); + while start > class.start() { + let previous = source.line_start(start - TextSize::from(1)); + if !source[TextRange::new(previous, start)].trim().is_empty() { + break; + } + start = previous; + } + *range = TextRange::new(start, range.end()); + } + } + let indentation = body_indentation(source, class); + for (position, range) in merged.into_iter().enumerate() { + let replacement = if left == 0 && position == 0 { + format!("{indentation}pass\n") + } else { + String::new() + }; + // the last member's cut takes the blank lines above it; a `pass` needs only its line + let range = if replacement.is_empty() { + range + } else { + let first = removed.iter().min().copied().unwrap_or_default(); + TextRange::new( + source.line_start(class.body[first].start()), + source + .full_line_end(range.end() - TextSize::from(1)) + .min(range.end()), + ) + }; + edits.push(FileEdit { + file, + edit: if replacement.is_empty() { + Edit::range_deletion(range) + } else { + Edit::range_replacement(replacement, range) + }, + }); + } + edits +} + +/// The edits adding `members` to `class`'s body: an attribute after its last attribute, before +/// its first method; a method after its last statement, a blank line away; in place of a body +/// that is only `pass` or `...`. +fn insert_members( + file: File, + source: &str, + class: &ast::StmtClassDef, + members: &[(MemberKind, String)], + line_ending: &str, +) -> Vec { + let attributes: Vec<&str> = members + .iter() + .filter(|(kind, _)| *kind == MemberKind::Attribute) + .map(|(_, text)| text.as_str()) + .collect(); + let methods: Vec<&str> = members + .iter() + .filter(|(kind, _)| *kind != MemberKind::Attribute) + .map(|(_, text)| text.as_str()) + .collect(); + let placeholder = class + .body + .iter() + .find(|statement| is_placeholder(statement)) + .filter(|_| { + class + .body + .iter() + .all(|statement| is_placeholder(statement) || is_docstring(statement)) + }); + if let Some(placeholder) = placeholder { + let mut text = attributes.concat(); + if !attributes.is_empty() && !methods.is_empty() { + text.push_str(line_ending); + } + text.push_str(&methods.join(line_ending)); + return vec![FileEdit { + file, + edit: Edit::range_replacement( + text, + TextRange::new( + source.line_start(placeholder.start()), + source.full_line_end(placeholder.end()), + ), + ), + }]; + } + + let mut edits = Vec::new(); + if !attributes.is_empty() { + let first_method = class + .body + .iter() + .find(|statement| matches!(statement, Stmt::FunctionDef(_) | Stmt::ClassDef(_))); + let (offset, text) = match first_method { + Some(method) => { + let mut text = attributes.concat(); + let previous_is_attribute = class + .body + .iter() + .take_while(|statement| !std::ptr::eq(*statement, method)) + .any(|statement| !is_docstring(statement)); + if !previous_is_attribute { + text.push_str(line_ending); + } + (source.line_start(method.start()), text) + } + None => { + let last = class.body.last().map_or(class.end(), Ranged::end); + (source.full_line_end(last), attributes.concat()) + } + }; + edits.push(FileEdit { + file, + edit: Edit::insertion(text, offset), + }); + } + if !methods.is_empty() { + let last = class.body.last().map_or(class.end(), Ranged::end); + let offset = source.full_line_end(last); + let mut text = String::new(); + if !source[..offset.to_usize()].ends_with(['\n', '\r']) { + text.push_str(line_ending); + } + for method in methods { + text.push_str(line_ending); + text.push_str(method); + } + edits.push(FileEdit { + file, + edit: Edit::insertion(text, offset), + }); + } + edits +} + +/// The edits importing `requests` into `class`'s module, each under the name the moved code +/// reads it by. +fn add_imports( + db: &dyn Db, + class: &ClassSite<'_>, + requests: &[OwnedRequest], +) -> Result, String> { + let importer = import_order::importer(db, class.file, &class.parsed); + let members = MembersInScope::empty(class.name_range.start()); + let mut actions: Vec = Vec::new(); + for request in requests { + let action = importer.import(request.request(), &members); + if action.symbol_text() != request.binds { + return Err(format!( + "the moved code reads `{}`, which {} cannot import under that name", + request.binds, + class.module(db).unwrap_or_default() + )); + } + actions.push(action); + } + Ok(importer + .combined_edits(&actions) + .into_iter() + .map(|edit| FileEdit { + file: class.file.file(db), + edit, + }) + .collect()) +} + +/// The edits marking `statement`, a method of `class`, as an override: basedpython's `override` +/// modifier, or python's `@override` with its import. +fn add_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec { + let Stmt::FunctionDef(function) = statement else { + return Vec::new(); + }; + let file = class.file.file(db); + let source = source_text(db, file); + let source = source.as_str(); + let header = source.line_start(function.name.start()); + let indentation = &source[TextRange::new( + header, + header + + TextSize::try_from( + source[header.to_usize()..].len() - source[header.to_usize()..].trim_start().len(), + ) + .unwrap_or_default(), + )]; + if file.source_type(db).is_basedpython() { + let at = header + TextSize::of(indentation); + return vec![FileEdit { + file, + edit: Edit::insertion("override ".to_string(), at), + }]; + } + let module = if class.file.python_version(db) >= ruff_python_ast::PythonVersion::PY312 { + "typing" + } else { + "typing_extensions" + }; + let stylist = Stylist::from_tokens(class.parsed.tokens(), source); + let importer = import_order::importer(db, class.file, &class.parsed); + let action = importer.import( + ImportRequest::import_from(module, "override"), + &MembersInScope::empty(statement.start()), + ); + let first_line = source.line_start(statement.start()); + let mut edits = vec![FileEdit { + file, + edit: Edit::insertion( + format!( + "{indentation}@{}{}", + action.symbol_text(), + stylist.line_ending().as_str() + ), + first_line, + ), + }]; + edits.extend( + importer + .combined_edits([&action]) + .into_iter() + .map(|edit| FileEdit { file, edit }), + ); + edits +} + +/// The edit taking the `override` off `statement`, a method of `class`. +fn remove_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec { + let Some(decorator) = override_marker(statement) else { + return Vec::new(); + }; + let source = source_text(db, class.file_id); + vec![FileEdit { + file: class.file_id, + edit: Edit::range_deletion(override_range(source.as_str(), decorator)), + }] +} + +/// The type of the receiver of the attribute whose name is at `range` in `file`: `x` of `x.name`. +fn attribute_receiver_type<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + range: TextRange, +) -> Option> { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let covering = ruff_python_ast::find_node::covering_node(parsed.syntax().into(), range); + let attribute = covering.ancestors().find_map(|node| match node { + ast::AnyNodeRef::ExprAttribute(attribute) if attribute.attr.range() == range => { + Some(attribute) + } + _ => None, + })?; + attribute.value.inferred_type(&SemanticModel::new(db, file)) +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + + use super::{movable_members, pull_members_up, push_members_down}; + use crate::refactor::move_symbol::tests::rendered; + use crate::tests::{CursorTest, CursorTestBuilder}; + + const SHAPES: &str = r#"class Base: + def keep(self): + return 1 + + +class Child(Base): + LIMIT = 3 + + def __init__(self, x): + self.x = x + + def area(self): + return self.x * self.LIMIT + + def plain(self): + return 2 + + def helper(self): + return self.plain() + 1 + + +class Other(Base): + pass + + +class GrandChild(Child): + def area(self): + return 0 +"#; + + fn project(shapes: &str) -> CursorTest { + CursorTestBuilder::default() + .source("cls/__init__.py", "") + .source("cls/shapes.py", shapes) + .source( + "cls/user.py", + "from cls.shapes import Base, Child\n\nprint(Child(1).area(), Base().keep())\n", + ) + .build() + } + + impl CursorTest { + fn pull_up(&self, names: &[&str], target: &str) -> String { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let moved = salsa::attach(&self.db, || { + pull_members_up( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + &names, + target, + ) + }); + rendered(&self.db, moved) + } + + fn push_down(&self, names: &[&str]) -> String { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let moved = salsa::attach(&self.db, || { + push_members_down( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + &names, + ) + }); + rendered(&self.db, moved) + } + + fn movable(&self) -> String { + let movable = salsa::attach(&self.db, || { + movable_members( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ) + }); + let Some(movable) = movable else { + return "nothing".to_string(); + }; + format!( + "{}\nmembers: {}\nsuperclasses: {}\nsubclasses: {}", + movable.class.qualified_name.unwrap_or_default(), + movable + .members + .iter() + .map(|member| format!("{} ({:?})", member.name, member.kind)) + .collect::>() + .join(", "), + movable + .superclasses + .iter() + .filter_map(|class| class.qualified_name.clone()) + .collect::>() + .join(", "), + movable + .subclasses + .iter() + .filter_map(|class| class.qualified_name.clone()) + .collect::>() + .join(", "), + ) + } + } + + #[test] + fn what_can_move_where() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.movable(), @" + cls.shapes.Child + members: LIMIT (Attribute), __init__ (Method), area (Method), plain (Method), helper (Method) + superclasses: cls.shapes.Base + subclasses: cls.shapes.GrandChild + "); + } + + #[test] + fn a_method_that_needs_nothing_else_is_pulled_up() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.pull_up(&["plain"], "cls.shapes.Base"), @" + --- /cls/shapes.py + class Base: + def keep(self): + return 1 + + def plain(self): + return 2 + + + class Child(Base): + LIMIT = 3 + + def __init__(self, x): + self.x = x + + def area(self): + return self.x * self.LIMIT + + def helper(self): + return self.plain() + 1 + + + class Other(Base): + pass + + + class GrandChild(Child): + def area(self): + return 0 + "); + } + + /// `area` reads `self.x`, which only a `Child` has; `Other` would inherit an `area` that + /// fails. + #[test] + fn a_method_that_reads_what_the_superclass_lacks_is_refused() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.pull_up(&["area"], "cls.shapes.Base"), @"refused: `area` uses `self.x` on line 13, which Base does not have"); + assert_snapshot!(test.pull_up(&["helper"], "cls.shapes.Base"), @"refused: `helper` uses `self.plain` on line 19, which Base does not have; pull it up too"); + } + + /// Pulled up together, `helper` finds the `plain` it calls. + #[test] + fn members_pulled_up_together_satisfy_each_other() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.pull_up(&["helper", "plain", "LIMIT"], "cls.shapes.Base"), @" + --- /cls/shapes.py + class Base: + LIMIT = 3 + + def keep(self): + return 1 + + def plain(self): + return 2 + + def helper(self): + return self.plain() + 1 + + + class Child(Base): + def __init__(self, x): + self.x = x + + def area(self): + return self.x * self.LIMIT + + + class Other(Base): + pass + + + class GrandChild(Child): + def area(self): + return 0 + "); + } + + /// `Base().keep()` would no longer find `keep`. + #[test] + fn a_member_read_off_the_class_itself_is_not_pushed_down() { + let test = project(&SHAPES.replace("class Base:", "class Base:")); + assert_snapshot!(test.push_down(&["keep"]), @"refused: cls/user.py:3 uses `keep` on what may be a Base rather than a subclass"); + } + + /// `helper` calls `self.plain()` on what may be a `Child` rather than a `GrandChild`. + #[test] + fn a_member_the_class_s_own_methods_read_is_not_pushed_down() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.push_down(&["plain"]), @"refused: cls/shapes.py:19 uses `plain` on what may be a Child rather than a subclass"); + } + + #[test] + fn a_member_nothing_reads_on_the_class_is_pushed_down() { + let test = CursorTestBuilder::default() + .source( + "main.py", + r#"class Base: + LIMIT = 3 + + def keep(self): + return self.LIMIT + + +class Child(Base): + pass + + +class Other(Base): + def keep(self): + return 2 + + +def use(child: Child, other: Other): + return child.keep() + other.keep() +"#, + ) + .build(); + assert_snapshot!(test.push_down(&["keep"]), @" + --- /main.py + class Base: + LIMIT = 3 + + + class Child(Base): + def keep(self): + return self.LIMIT + + + class Other(Base): + def keep(self): + return 2 + + + def use(child: Child, other: Other): + return child.keep() + other.keep() + "); + } + + /// Pulled up where nothing above has the name, a method overrides nothing, and what has + /// the name below the superclass then overrides it: in basedpython it gains the modifier. + #[test] + fn override_follows_a_pull_up() { + let test = CursorTestBuilder::default() + .source( + "main.by", + r#"class Base: + pass + + +class Child(Base): + def describe(self) -> str: + return "child" + + +class Other(Base): + def describe(self) -> str: + return "other" + + +class Leaf(Child): + override def describe(self) -> str: + return "leaf" +"#, + ) + .build(); + assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @r#" + --- /main.by + class Base: + def describe(self) -> str: + return "child" + + + class Child(Base): + pass + + + class Other(Base): + override def describe(self) -> str: + return "other" + + + class Leaf(Child): + override def describe(self) -> str: + return "leaf" + "#); + } + + /// In python it gains `@override`, imported. + #[test] + fn override_follows_a_pull_up_in_python() { + let test = CursorTestBuilder::default() + .source( + "main.py", + r#"class Base: + pass + + +class Child(Base): + def describe(self) -> str: + return "child" + + +class Other(Base): + def describe(self) -> str: + return "other" +"#, + ) + .build(); + assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @r#" + --- /main.py + from typing import override + class Base: + def describe(self) -> str: + return "child" + + + class Child(Base): + pass + + + class Other(Base): + @override + def describe(self) -> str: + return "other" + "#); + } + + /// `Leaf` would inherit `Mid`'s `describe` instead of the one it pulls up. + #[test] + fn a_class_between_with_the_name_is_refused() { + let test = CursorTestBuilder::default() + .source( + "main.py", + r#"class Base: + pass + + +class Mid(Base): + def describe(self) -> str: + return "mid" + + +class Leaf(Mid): + def describe(self) -> str: + return "leaf" +"#, + ) + .build(); + assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @"refused: Leaf would inherit Mid's `describe` rather than its own"); + } +} diff --git a/crates/ty_ide/src/refactor/move_symbol.rs b/crates/ty_ide/src/refactor/move_symbol.rs index efca6b6f29..bf064db5d0 100644 --- a/crates/ty_ide/src/refactor/move_symbol.rs +++ b/crates/ty_ide/src/refactor/move_symbol.rs @@ -34,6 +34,7 @@ use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_db::source::source_text; use ruff_diagnostics::Edit; use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::token::TokenKind; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; use ruff_python_codegen::Stylist; @@ -45,7 +46,7 @@ use ty_project::Db; use ty_python_core::ProgramFile; use ty_python_core::definition::DefinitionKind; use ty_python_core::scope::FileScopeId; -use ty_python_core::semantic_index; +use ty_python_core::{SemanticIndex, semantic_index}; use ty_python_semantic::importer::{ImportAction, ImportRequest, MembersInScope}; use ty_python_semantic::types::Type; use ty_python_semantic::{HasType, SemanticModel}; @@ -186,11 +187,48 @@ pub fn move_symbol( return Err(format!("{target_module} has a `{name}` already")); } + // a `global` in the definition names the module it is in, which the move changes + if let Some(written) = global_written_by(db, index, declaration.statement.range()) { + return Err(format!( + "`{name}` assigns {source_module}'s `{written}` through `global`, which would name \ + {target_module}'s once moved" + )); + } + // What the definition reads from its module comes with it. let dependencies = dependencies(db, source, &parsed, declaration.statement, name); let mut target_imports: Vec = Vec::new(); let mut from_source: Vec = Vec::new(); + for builtin in builtins_read(db, source, declaration.statement) { + if target_binds(&builtin) { + return Err(format!( + "`{name}` uses the builtin `{builtin}`, which {target_module} binds to something \ + else" + )); + } + } for dependency in &dependencies { + let read = match dependency { + Dependency::Imported { request, .. } => &request.binds, + Dependency::Sibling(sibling) => sibling, + }; + if globals.symbol_id(read).is_some_and(|symbol| { + bindings_of_symbol( + db, + index, + &parsed, + FileScopeId::global(), + symbol, + source_type.is_stub(), + ) + .len() + > 1 + }) { + return Err(format!( + "`{name}` uses `{read}`, which {source_module} binds more than once, and an import \ + of it would keep only the value it had when imported" + )); + } match dependency { Dependency::Imported { request, .. } => { if target_binds(&request.binds) @@ -288,6 +326,15 @@ pub fn move_symbol( request, } = dependency && !used_elsewhere(&parsed, &request.binds, statement_range) + && !imported_from( + db, + source, + &parsed, + &model, + *binding_statement, + *alias_index, + &source_module, + ) && let Some(edit) = remove_alias( source_text, parsed.suite(), @@ -390,9 +437,70 @@ pub fn move_symbol( &mut moved, ); } + moved.edits = disjoint(db, moved.edits)?; Ok(moved) } +/// `edits` without two that overlap, which an editor would refuse. each removal of a statement +/// takes the blank lines around it, so two removed next to each other both claim the lines +/// between them: overlapping removals are made one. an insertion inside a removal goes where the +/// removal starts, which keeps it in the same place among what is left. any other overlap is +/// refused +fn disjoint(db: &dyn Db, edits: Vec) -> Result, String> { + let mut by_file: FxHashMap> = FxHashMap::default(); + let mut files: Vec = Vec::new(); + for edit in edits { + if !by_file.contains_key(&edit.file) { + files.push(edit.file); + } + by_file.entry(edit.file).or_default().push(edit.edit); + } + let mut disjoint = Vec::new(); + for file in files { + let edits = by_file.remove(&file).unwrap_or_default(); + let (removals, others): (Vec, Vec) = edits + .into_iter() + .partition(|edit| edit.content().is_none_or(str::is_empty) && !edit.range().is_empty()); + let mut ranges: Vec = removals.iter().map(Ranged::range).collect(); + ranges.sort_by_key(|range| (range.start(), range.end())); + let mut merged: Vec = Vec::new(); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start() < last.end() => *last = last.cover(range), + _ => merged.push(range), + } + } + let mut edits: Vec = merged.into_iter().map(Edit::range_deletion).collect(); + for edit in others { + let inside = edits.iter().find_map(|removal| { + (edit.range().is_empty() + && removal.start() < edit.start() + && edit.start() < removal.end()) + .then(|| removal.start()) + }); + edits.push(match inside { + Some(start) => { + Edit::insertion(edit.content().unwrap_or_default().to_string(), start) + } + None => edit, + }); + } + let mut ranges: Vec = edits.iter().map(Ranged::range).collect(); + ranges.sort_by_key(|range| (range.start(), range.end())); + if ranges + .windows(2) + .any(|pair| pair[0].end() > pair[1].start()) + { + return Err(format!( + "the move would make edits to {} that overlap", + file.path(db) + )); + } + disjoint.extend(edits.into_iter().map(|edit| FileEdit { file, edit })); + } + Ok(disjoint) +} + /// The file and offset of the top-level declaration named at `offset`: `offset` itself when it is /// in a declaration's name, or else where the name at `offset` is defined. fn declaration_site<'db>( @@ -474,16 +582,16 @@ impl<'a> Declaration<'a> { /// An import request whose strings it owns. #[derive(Debug, Clone, PartialEq, Eq)] -struct OwnedRequest { - module: String, - member: Option, - asname: Option, +pub(super) struct OwnedRequest { + pub(super) module: String, + pub(super) member: Option, + pub(super) asname: Option, /// The name the import binds. - binds: String, + pub(super) binds: String, } impl OwnedRequest { - fn request(&self) -> ImportRequest<'_> { + pub(super) fn request(&self) -> ImportRequest<'_> { match (&self.member, &self.asname) { (Some(member), Some(asname)) => { ImportRequest::import_from_as(&self.module, member, asname) @@ -497,7 +605,7 @@ impl OwnedRequest { /// A name a definition reads from the top level of its module. #[derive(Debug)] -enum Dependency { +pub(super) enum Dependency { /// A name one of the module's imports binds, as the import that binds it. Imported { /// The start of the statement that imports it, or `None` for a star import, which binds @@ -511,35 +619,19 @@ enum Dependency { } /// The names `statement` reads from the top level of `file`, other than `name`, which it defines. -fn dependencies( +pub(super) fn dependencies( db: &dyn Db, file: ProgramFile<'_>, parsed: &ParsedModuleRef, statement: &Stmt, name: &str, ) -> Vec { - struct Loads<'a> { - names: Vec<&'a ast::ExprName>, - } - impl<'a> SourceOrderVisitor<'a> for Loads<'a> { - fn visit_expr(&mut self, expr: &'a Expr) { - if let Expr::Name(name) = expr - && name.ctx.is_load() - { - self.names.push(name); - } - walk_expr(self, expr); - } - } - let mut loads = Loads { names: Vec::new() }; - loads.visit_stmt(statement); - let index = semantic_index(db, file); let globals = index.place_table(FileScopeId::global()); let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); let mut seen = FxHashSet::default(); let mut dependencies = Vec::new(); - for load in loads.names { + for load in loads(statement) { let id = load.id.as_str(); if id == name || !seen.insert(id) { continue; @@ -616,9 +708,141 @@ fn dependencies( dependencies } +/// the names `statement` reads +fn loads(statement: &Stmt) -> Vec<&ast::ExprName> { + struct Loads<'a> { + names: Vec<&'a ast::ExprName>, + } + impl<'a> SourceOrderVisitor<'a> for Loads<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + { + self.names.push(name); + } + walk_expr(self, expr); + } + } + let mut loads = Loads { names: Vec::new() }; + loads.visit_stmt(statement); + loads.names +} + +/// the names `statement` reads that no scope of `file` binds, the builtins among them +fn builtins_read(db: &dyn Db, file: ProgramFile<'_>, statement: &Stmt) -> Vec { + let index = semantic_index(db, file); + let mut read: Vec = Vec::new(); + for load in loads(statement) { + let id = load.id.as_str(); + if let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) + && resolving_scope(index, scope, id).is_none() + && !read.iter().any(|seen| seen == id) + { + read.push(id.to_string()); + } + } + read +} + +/// a name of the module's own that a `global` in a scope inside `range` assigns +fn global_written_by(db: &dyn Db, index: &SemanticIndex<'_>, range: TextRange) -> Option { + index + .use_def_map(FileScopeId::global()) + .definitions_with_usage() + .find_map(|(_, definition, _)| match definition.kind(db) { + DefinitionKind::NestedBindings(nested) => nested + .nested_declarations + .iter() + .any(|declaration| { + declaration.is_global() + && declaration.is_bound + && range.contains_range(declaration.range) + }) + .then(|| nested.name.to_string()), + _ => None, + }) +} + +/// whether a module of the project other than `file` reads the name the import starting at +/// `statement` binds, alias `alias_index`, from `file`'s module: imports it from there, reads it +/// as an attribute of the module, or reads it through a star import of the module +fn imported_from( + db: &dyn Db, + file: ProgramFile<'_>, + parsed: &ParsedModuleRef, + model: &SemanticModel<'_>, + statement: TextSize, + alias_index: usize, + module: &ModuleName, +) -> bool { + let Some(alias) = parsed + .suite() + .iter() + .find(|stmt| stmt.start() == statement) + .and_then(|stmt| match stmt { + Stmt::Import(import) => import.names.get(alias_index), + Stmt::ImportFrom(from) => from.names.get(alias_index), + _ => None, + }) + else { + // what cannot be looked up is kept + return true; + }; + let binding = alias.asname.as_ref().unwrap_or(&alias.name); + let Some(goto) = find_goto_target(model, parsed, binding.start()) else { + return true; + }; + let file_id = file.file(db); + let mut reads_by_file: FxHashMap> = FxHashMap::default(); + for reference in + references(db, file, &goto, ReferencesMode::RenameMultiFile).unwrap_or_default() + { + if reference.file() != file_id { + reads_by_file + .entry(reference.file()) + .or_default() + .push(reference.range()); + } + } + reads_by_file.into_iter().any(|(other, ranges)| { + let other = ProgramFile::new(db, other, file.program(db)); + let parsed = parsed_module(db, other.python_file(db)).load(db); + let model = SemanticModel::new(db, other); + let importing_file = ImportingFile::File(other.file(db), other.resolver_environment(db)); + let from_module = |from: &ast::StmtImportFrom| { + ModuleName::from_import_statement(db, importing_file, from) + .is_ok_and(|imported| &imported == module) + }; + let star_imports_module = parsed.suite().iter().any(|stmt| { + matches!(stmt, Stmt::ImportFrom(from) + if from.names.iter().any(|alias| alias.name.as_str() == "*") && from_module(from)) + }); + ranges.iter().any(|range| { + let covering = covering_node(parsed.syntax().into(), *range); + if let Some(from) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtImportFrom(from) => Some(from), + _ => None, + }) { + return from_module(from); + } + match covering + .ancestors() + .find(|node| !matches!(node, AnyNodeRef::Identifier(_))) + { + Some(AnyNodeRef::ExprAttribute(attribute)) => matches!( + attribute.value.inferred_type(&model), + Some(Type::ModuleLiteral(literal)) if literal.module(db).name(db) == module + ), + Some(AnyNodeRef::ExprName(_)) => star_imports_module, + _ => false, + } + }) + }) +} + /// Whether `name` is private to its module by python's convention: it starts with an underscore /// and is not a dunder. -fn is_private(name: &str) -> bool { +pub(super) fn is_private(name: &str) -> bool { name.starts_with('_') && !(name.starts_with("__") && name.ends_with("__")) } @@ -651,7 +875,7 @@ fn target_imports_same( } /// Whether a top-level import of `file` imports `module`, or something from it. -fn imports_module( +pub(super) fn imports_module( db: &dyn Db, file: ProgramFile<'_>, parsed: &ParsedModuleRef, @@ -711,16 +935,18 @@ fn used_elsewhere(parsed: &ParsedModuleRef, name: &str, except: TextRange) -> bo } /// The text a cut takes, and the range it removes from the module. -struct Cut { +pub(super) struct Cut { /// The definition with its decorators and the comments directly above it, ending in a line /// break. - text: String, + pub(super) text: String, + /// Where `text` starts in the module. + pub(super) start: TextSize, /// Its lines and the blank lines that separated it from the statement after it, or, when it is /// the last statement, from the one before. - removed: TextRange, + pub(super) removed: TextRange, } -fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { +pub(super) fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { let statement = &suite[index]; let floor = index .checked_sub(1) @@ -768,7 +994,8 @@ fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { } TextRange::new(start, removed_end) } else { - // the last statement takes the blank lines above it, and everything after it + // the last statement of its suite takes the blank lines above it, and at the end of the + // file the blank lines after it let mut removed_start = start; while removed_start > floor { let previous = source.line_start(removed_start - TextSize::from(1)); @@ -780,9 +1007,18 @@ fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { } removed_start = previous; } - TextRange::new(removed_start, TextSize::of(source)) + let removed_end = if source[end.to_usize()..].trim().is_empty() { + TextSize::of(source) + } else { + end + }; + TextRange::new(removed_start, removed_end) }; - Cut { text, removed } + Cut { + text, + start, + removed, + } } /// The text appended to the end of the target: the definition, set apart from what is above it @@ -961,7 +1197,7 @@ fn rewrite_references( continue; }; if from.names.len() == 1 { - if let Some(module_range) = from_module_range(text, from) { + if let Some(module_range) = from_module_range(&parsed, from) { moved.edits.push(FileEdit { file: file_id, edit: Edit::range_replacement(target_module.to_string(), module_range), @@ -989,23 +1225,46 @@ fn rewrite_references( actions.push(importer.import(request, &MembersInScope::empty(from.end()))); } else { // an import inside a function: the new one goes on the line after it - let indentation = - &text[TextRange::new(text.line_start(from.start()), from.start())]; + // when it is a line of its own, and after it on its line otherwise, + // which keeps it in the same block and before anything after it let statement = match &alias.asname { Some(asname) => { format!("from {target_module} import {name} as {asname}") } None => format!("from {target_module} import {name}"), }; - moved.edits.push(FileEdit { - file: file_id, - edit: Edit::insertion( + let tokens = parsed.tokens(); + let starts_line = tokens.before(from.start()).last().is_none_or(|token| { + matches!( + token.kind(), + TokenKind::Newline + | TokenKind::NonLogicalNewline + | TokenKind::Indent + | TokenKind::Dedent + ) + }); + let ends_line = tokens.after(from.end()).first().is_none_or(|token| { + matches!( + token.kind(), + TokenKind::Newline | TokenKind::Comment | TokenKind::EndOfFile + ) + }); + let edit = if starts_line && ends_line { + let indentation = + &text[TextRange::new(text.line_start(from.start()), from.start())]; + Edit::insertion( format!( "{indentation}{statement}{}", stylist.line_ending().as_str() ), text.full_line_end(from.end()), - ), + ) + } else { + Edit::insertion(format!("; {statement}"), from.end()) + }; + moved.edits.push(FileEdit { + file: file_id, + edit, }); } } @@ -1073,30 +1332,33 @@ fn rewrite_references( ); } -/// The range of the module a `from` import names, its leading dots included. -fn from_module_range(source: &str, from: &ast::StmtImportFrom) -> Option { - let start = from.start() + TextSize::try_from(source[from.range()].find("from")? + 4).ok()?; - let end = from.names.first()?.start(); - let between = &source[TextRange::new(start, end)]; - let keyword = between - .rfind(" import") - .or_else(|| between.rfind(" export"))?; - let module = &between[..keyword]; - let leading = module.len() - module.trim_start().len(); - let trailing = module.trim_end().len(); - Some(TextRange::new( - start + TextSize::try_from(leading).ok()?, - start + TextSize::try_from(trailing).ok()?, - )) +/// The range of the module a `from` import names, its leading dots included: from the token +/// after `from` to the end of the module's name, or of its dots when it has none. +fn from_module_range(parsed: &ParsedModuleRef, from: &ast::StmtImportFrom) -> Option { + let mut tokens = parsed.tokens().in_range(from.range()).iter(); + let keyword = tokens.next()?; + if keyword.kind() != TokenKind::From { + return None; + } + let first = tokens.next()?; + let end = match &from.module { + Some(module) => module.end(), + None => std::iter::once(first) + .chain(tokens) + .take_while(|token| matches!(token.kind(), TokenKind::Dot | TokenKind::Ellipsis)) + .last()? + .end(), + }; + Some(TextRange::new(first.start(), end)) } /// The one-based line `offset` is on. -fn line_number(source: &str, offset: TextSize) -> usize { +pub(super) fn line_number(source: &str, offset: TextSize) -> usize { source[..offset.to_usize()].matches('\n').count() + 1 } #[cfg(test)] -mod tests { +pub(super) mod tests { use std::fmt::Write as _; use insta::assert_snapshot; @@ -1162,6 +1424,61 @@ X = os.sep .build() } + /// Every file `moved` changes, as it is afterwards, and its warnings; or why it refused. + pub(in crate::refactor) fn rendered( + db: &ty_project::TestDb, + moved: Result, + ) -> String { + let moved = match moved { + Ok(moved) => moved, + Err(reason) => return format!("refused: {reason}"), + }; + let mut files: Vec<_> = moved.edits.iter().map(|edit| edit.file).collect(); + files.sort_by_key(|file| file.path(db).as_str().to_string()); + files.dedup(); + let mut out = String::new(); + for file in files { + let mut text = source_text(db, file).as_str().to_string(); + let mut edits: Vec<_> = moved + .edits + .iter() + .filter(|edit| edit.file == file) + .enumerate() + .collect(); + let mut ranges: Vec<_> = edits.iter().map(|(_, edit)| edit.edit.range()).collect(); + ranges.sort_by_key(|range| (range.start(), range.end())); + for pair in ranges.windows(2) { + assert!( + pair[0].end() <= pair[1].start(), + "edits to {} overlap, which an editor refuses: {:?} and {:?}", + file.path(db), + pair[0], + pair[1] + ); + } + // applied from the end, a removal before an insertion where it starts; of two + // insertions at one place, the one listed first ends up first + edits.sort_by_key(|(order, edit)| { + ( + std::cmp::Reverse(edit.edit.start()), + std::cmp::Reverse(edit.edit.end()), + std::cmp::Reverse(*order), + ) + }); + for (_, edit) in edits { + text.replace_range( + edit.edit.range().to_std_range(), + edit.edit.content().unwrap_or_default(), + ); + } + let _ = write!(out, "--- {}\n{text}", file.path(db)); + } + for warning in moved.warnings { + let _ = writeln!(out, "warning: {warning}"); + } + out + } + impl CursorTest { /// Every module the move to `mv/dst.py` changes, as it is afterwards, or why it refuses. fn move_to_dst(&self) -> String { @@ -1174,41 +1491,7 @@ X = os.sep target, ) }); - let moved = match moved { - Ok(moved) => moved, - Err(reason) => return format!("refused: {reason}"), - }; - let mut files: Vec<_> = moved.edits.iter().map(|edit| edit.file).collect(); - files.sort_by_key(|file| file.path(&self.db).as_str().to_string()); - files.dedup(); - let mut out = String::new(); - for file in files { - let mut text = source_text(&self.db, file).as_str().to_string(); - let mut edits: Vec<_> = moved - .edits - .iter() - .filter(|edit| edit.file == file) - .enumerate() - .collect(); - // applied from the end; of two at one place, the one listed first ends up first - edits.sort_by_key(|(order, edit)| { - ( - std::cmp::Reverse(edit.edit.start()), - std::cmp::Reverse(*order), - ) - }); - for (_, edit) in edits { - text.replace_range( - edit.edit.range().to_std_range(), - edit.edit.content().unwrap_or_default(), - ); - } - let _ = write!(out, "--- {}\n{text}", file.path(&self.db)); - } - for warning in moved.warnings { - let _ = writeln!(out, "warning: {warning}"); - } - out + rendered(&self.db, moved) } } @@ -1402,4 +1685,216 @@ X = os.sep .build(); assert_snapshot!(test.move_to_dst(), @"refused: mv.dst has a `f` already"); } + + /// a project of `mv/src.py`, `mv/dst.py` and the modules `others` names, by path and source + fn modules(src: &str, dst: &str, others: &[(&str, &str)]) -> CursorTest { + let mut builder = CursorTestBuilder::default(); + builder + .source("mv/__init__.py", "") + .source("mv/src.py", src) + .source("mv/dst.py", dst); + for (path, source) in others { + builder.source(*path, source); + } + builder.build() + } + + /// the import the definition takes with it and the definition itself are cut together, with + /// the blank lines between them, which both cuts would otherwise claim + #[test] + fn an_import_and_the_definition_after_it_are_cut_together() { + let test = modules( + "import math\n\n\ndef area(r): return math.pi*r*r\n", + "", + &[], + ); + assert_snapshot!(test.move_to_dst(), @" + --- /mv/dst.py + import math + + + def area(r): return math.pi*r*r + --- /mv/src.py + "); + } + + /// another module imports the name from the old module, so the import stays + #[test] + fn an_import_another_module_imports_from_the_old_one_is_kept() { + let test = modules( + &SRC.replace("def free(x)", "def free(x)"), + DST, + &[( + "mv/user.py", + "from mv.src import OrderedDict\n\nprint(OrderedDict())\n", + )], + ); + assert_snapshot!(test.move_to_dst(), @r#" + --- /mv/dst.py + """Destination.""" + + import os + from collections import OrderedDict + + X = os.sep + + + def free(x): + return OrderedDict(x=x) + --- /mv/src.py + import math + from collections import OrderedDict + + __all__ = ["RATE", "area", "helper"] + + RATE = 2 + _SECRET = 3 + + + def helper(x): + return x * RATE + + + @staticmethod + def area(r): + """Area of a circle.""" + return math.pi * r * r * RATE + + + def uses_private(x): + return x + _SECRET + "#); + } + + /// a module that imports the name from where the old module does needs nothing from the old + /// module, so the import goes + #[test] + fn an_import_another_module_makes_itself_is_dropped() { + let test = modules( + "from collections import OrderedDict\n\n\ndef free(x):\n return OrderedDict(x=x)\n", + "", + &[( + "mv/user.py", + "from collections import OrderedDict\n\nprint(OrderedDict())\n", + )], + ); + assert_snapshot!(test.move_to_dst(), @" + --- /mv/dst.py + from collections import OrderedDict + + + def free(x): + return OrderedDict(x=x) + --- /mv/src.py + "); + let star = modules( + "from collections import OrderedDict\n\n\ndef free(x):\n return OrderedDict(x=x)\n", + "", + &[( + "mv/user.py", + "from mv.src import *\n\nprint(OrderedDict())\n", + )], + ); + assert_snapshot!(star.move_to_dst(), @" + --- /mv/dst.py + from collections import OrderedDict + + + def free(x): + return OrderedDict(x=x) + --- /mv/src.py + from collections import OrderedDict + "); + } + + /// moved, `global COUNT` would name the target's `COUNT`, not the one the old module reads + #[test] + fn a_definition_that_assigns_a_global_of_its_module_is_refused() { + let test = modules( + "COUNT = 0\n\n\ndef bump():\n global COUNT\n COUNT += 1\n", + "", + &[], + ); + assert_snapshot!(test.move_to_dst(), @"refused: `bump` assigns mv.src's `COUNT` through `global`, which would name mv.dst's once moved"); + } + + /// an import of a name the old module rebinds would keep the value it had when imported + #[test] + fn a_definition_that_reads_a_name_its_module_rebinds_is_refused() { + let test = modules( + "COUNT = 0\n\n\ndef bump():\n global COUNT\n COUNT += 1\n\n\ndef show():\n return COUNT\n", + "", + &[], + ); + assert_snapshot!(test.move_to_dst(), @"refused: `show` uses `COUNT`, which mv.src binds more than once, and an import of it would keep only the value it had when imported"); + } + + /// the target's own `len` would be read in place of the builtin + #[test] + fn a_builtin_the_target_binds_is_refused() { + let test = modules( + "def size(x):\n return len(x)\n", + "len = 3\n", + &[], + ); + assert_snapshot!(test.move_to_dst(), @"refused: `size` uses the builtin `len`, which mv.dst binds to something else"); + } + + /// the module of an import is found by its tokens, so a comment that says `import` is not + /// taken for the keyword + #[test] + fn an_import_with_a_comment_is_pointed_at_the_new_module() { + let test = modules( + "def area(r):\n return r * r\n", + "", + &[( + "mv/user.py", + "from mv.src import ( # import area\n area,\n)\n\nprint(area(1))\n", + )], + ); + assert_snapshot!(test.move_to_dst(), @" + --- /mv/dst.py + def area(r): + return r * r + --- /mv/src.py + --- /mv/user.py + from mv.dst import ( # import area + area, + ) + + print(area(1)) + "); + } + + /// an import inside a function that does not start its line gets the new import after it, + /// on the same line + #[test] + fn an_import_that_shares_its_line_is_followed_on_that_line() { + let test = modules( + "def area(r):\n return r * r\n\n\ndef helper():\n return 1\n", + "", + &[( + "mv/user.py", + "def g():\n x = 1; from mv.src import area, helper\n return area(x) + helper()\n\n\ndef h():\n from mv.src import area, helper\n return area(helper())\n", + )], + ); + assert_snapshot!(test.move_to_dst(), @" + --- /mv/dst.py + def area(r): + return r * r + --- /mv/src.py + def helper(): + return 1 + --- /mv/user.py + def g(): + x = 1; from mv.src import helper; from mv.dst import area + return area(x) + helper() + + + def h(): + from mv.src import helper + from mv.dst import area + return area(helper()) + "); + } } diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index b8db5a4851..d0ab70415c 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -62,11 +62,11 @@ pub use types::extensions::{ }; pub use types::ide_support::{ ImplementationsFinder, ImportAliasResolution, OverridableMember, ResolvedDefinition, - TypeHierarchyClass, definitions_for_attribute, definitions_for_bin_op, + TypeHierarchyClass, class_bases, definitions_for_attribute, definitions_for_bin_op, definitions_for_case_name, definitions_for_context_sensitive_name, definitions_for_django_lookup_root, definitions_for_imported_symbol, definitions_for_name, - definitions_for_unary_op, map_stub_definition, type_hierarchy_prepare, type_hierarchy_subtypes, - type_hierarchy_supertypes, + definitions_for_unary_op, instance_member_names, is_within_classes, map_stub_definition, + type_hierarchy_prepare, type_hierarchy_subtypes, type_hierarchy_supertypes, }; pub use types::implicit_names::implicit_names; pub use types::reified_infer::{ diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 43aec42518..456cb63a07 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -46,8 +46,9 @@ use crate::types::signatures::{ }; use crate::types::{ BoundTypeVarInstance, CallDunderError, ClassBase, ClassLiteral, ClassType, KnownClass, - KnownFunction, KnownInstanceType, KnownUnion, PropertyAccessorRole, SpecialFormType, Type, - TypeContext, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, binding_type, + KnownFunction, KnownInstanceType, KnownUnion, PropertyAccessorRole, SpecialFormType, + SubclassOfType, Type, TypeContext, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, + binding_type, }; use crate::{Db, HasDefinition, HasType, ProgramEnvironment, SemanticModel}; use ruff_db::files::{File, FileRange}; @@ -2594,6 +2595,70 @@ fn direct_subtypes<'db>( subtypes } +/// The classes `class` names as its bases, as class literals, for a refactoring that asks what +/// a class inherits from above itself. Empty for a type that is no class. +pub fn class_bases<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: Type<'db>, +) -> Vec> { + let Some(class_literal) = extract_class_literal(db, env, class) else { + return Vec::new(); + }; + class_literal + .explicit_bases(db) + .iter() + .filter_map(|base| extract_class_literal(db, env, *base)) + .map(Type::from) + .collect() +} + +/// The names of the members an instance of `class` has, its own and inherited, attributes its +/// methods assign included. Empty for a type that is no class. +pub fn instance_member_names<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: Type<'db>, +) -> rustc_hash::FxHashSet { + let Some(class_literal) = extract_class_literal(db, env, class) else { + return rustc_hash::FxHashSet::default(); + }; + let instance = Type::instance(db, env, class_literal.default_specialization(db)); + crate::types::list_members::all_members(db, env, instance) + .into_iter() + .chain(crate::types::list_members::all_members(db, env, class)) + .map(|member| member.name) + .collect() +} + +/// Whether every value of `receiver` is one of `classes`, or below one of them: an instance of +/// one of them or of a subclass, or one of them or a subclass itself. A member read off such a +/// receiver is looked up from one of those classes down, never from above them. +/// +/// a gradual receiver may hold any value, so only a subtype of a class's instances, under any +/// specialization of it, is within that class +pub fn is_within_classes<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + receiver: Type<'db>, + classes: &[Type<'db>], +) -> bool { + if let Type::Union(union) = receiver { + return union + .elements(db) + .iter() + .all(|element| is_within_classes(db, env, *element, classes)); + } + classes + .iter() + .filter_map(|class| extract_class_literal(db, env, *class)) + .any(|class| { + let class = class.top_materialization(db); + receiver.is_subtype_of(db, env, Type::instance(db, env, class)) + || receiver.is_subtype_of(db, env, SubclassOfType::from(db, env, class)) + }) +} + /// Enumerates the reachable class definitions in `file`. fn reachable_class_literals_in_file<'db>( db: &'db dyn Db, @@ -4968,6 +5033,104 @@ mod tests { use ruff_text_size::Ranged; use ty_python_core::ProgramFile; + /// what `describe` says of the types of the expression statements in `source`'s last `def` + /// and of those at the end of the module + fn describe_statement_types( + source: &str, + describe: impl for<'db> Fn( + &'db crate::db::tests::TestDb, + &crate::ProgramEnvironment<'db>, + &[crate::types::Type<'db>], + &[crate::types::Type<'db>], + ) -> Vec, + ) -> anyhow::Result> { + let db = TestDbBuilder::new() + .with_file("/src/foo.py", source) + .build()?; + let file = system_path_to_file(&db, "/src/foo.py")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + let parsed = parsed_module(&db, file.python_file(&db)).load(&db); + let model = SemanticModel::new(&db, file); + let types = |suite: &[ast::Stmt]| { + suite + .iter() + .filter_map(ast::Stmt::as_expr_stmt) + .filter_map(|statement| crate::HasType::inferred_type(&*statement.value, &model)) + .collect::>() + }; + let function = parsed + .suite() + .iter() + .rev() + .find_map(ast::Stmt::as_function_def_stmt) + .context("expected a function")?; + let env = crate::ProgramEnvironment::from_file(file); + Ok(describe( + &db, + &env, + &types(&function.body), + &types(parsed.suite()), + )) + } + + #[test] + fn a_receiver_is_within_classes_only_when_every_value_is() -> anyhow::Result<()> { + let receivers = describe_statement_types( + r#" +from typing import Any + +class Shape: ... +class Circle(Shape): ... +class Box[T](Shape): ... + +def f( + circle: Circle, + box: Box[int], + circle_class: type[Circle], + shape: Shape, + gradual: Circle | Any, + gradual_class: type[Circle] | type[Any], + unknown, +): + circle + box + circle_class + shape + gradual + gradual_class + unknown + +Circle +Box +"#, + |db, env, receivers, classes| { + receivers + .iter() + .map(|receiver| { + format!( + "{}: {}", + receiver.display(db, env), + super::is_within_classes(db, env, *receiver, classes) + ) + }) + .collect() + }, + )?; + assert_eq!( + receivers, + [ + "Circle: true", + "Box[int]: true", + "type[Circle]: true", + "Shape: false", + "Circle | Any: false", + "type[Circle | Any]: false", + "unknown@f: false", + ] + ); + Ok(()) + } + /// what [`super::new_attribute_obstacle`] says of each class in `source`, by name fn new_attribute_obstacles(source: &str) -> anyhow::Result> { let db = TestDbBuilder::new() diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index ca5b978d53..4292b4c585 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -249,6 +249,19 @@ pub(super) fn request(req: server::Request) -> Task { requests::MoveSymbolRequestHandler::METHOD => background_document_request_task::< requests::MoveSymbolRequestHandler, >(req, BackgroundSchedule::Worker), + requests::PrepareMoveMembersRequestHandler::METHOD => background_document_request_task::< + requests::PrepareMoveMembersRequestHandler, + >( + req, BackgroundSchedule::Worker + ), + requests::PullMembersUpRequestHandler::METHOD => background_document_request_task::< + requests::PullMembersUpRequestHandler, + >(req, BackgroundSchedule::Worker), + requests::PushMembersDownRequestHandler::METHOD => background_document_request_task::< + requests::PushMembersDownRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 7ee8ecd521..726268841b 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -47,6 +47,7 @@ mod hover; mod injections; mod inlay_hints; mod module_name; +mod move_members; mod move_symbol; mod on_type_formatting; mod overridable_members; @@ -111,6 +112,9 @@ pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; +pub(super) use move_members::{ + PrepareMoveMembersRequestHandler, PullMembersUpRequestHandler, PushMembersDownRequestHandler, +}; pub(super) use move_symbol::{MoveSymbolRequestHandler, PrepareMoveSymbolRequestHandler}; pub(super) use on_type_formatting::OnTypeFormattingRequestHandler; pub(super) use overridable_members::OverridableMembersRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/move_members.rs b/crates/ty_server/src/server/api/requests/move_members.rs new file mode 100644 index 0000000000..ecb0c66db5 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/move_members.rs @@ -0,0 +1,289 @@ +//! `by/prepareMoveMembers`, `by/pullMembersUp` and `by/pushMembersDown`: moving a class's members +//! into one of its superclasses, or into its subclasses. see [`ty_ide::pull_members_up`] for what +//! is moved and what is refused + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ruff_db::files::FileRange; +use ty_ide::{HierarchyClass, MemberKind, movable_members, pull_members_up, push_members_down}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use super::move_symbol::MoveSymbolAnswer; +use crate::document::{FileRangeExt, PositionExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum PrepareMoveMembersRequest {} + +impl Request for PrepareMoveMembersRequest { + type Params = PrepareMoveMembersParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareMoveMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// a position in a class: its header or its body +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PrepareMoveMembersParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// the class, its members, and the project's classes they could move to +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MovableMembers { + class: ClassItem, + members: Vec, + /// the project's classes it inherits from, nearest first: where `by/pullMembersUp` can pull + superclasses: Vec, + /// the project's classes that name it as a base: where `by/pushMembersDown` pushes + subclasses: Vec, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClassItem { + name: String, + /// its module's dotted name and its own; `null` for a class in a file no module is + /// imported from + qualified_name: Option, + uri: Uri, + /// its name + range: Range, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MemberItem { + name: String, + /// `method`, `classmethod`, `staticmethod`, `property` or `attribute` + kind: String, + /// its name, in its first statement + range: Range, +} + +pub(crate) struct PrepareMoveMembersRequestHandler; + +impl RequestHandler for PrepareMoveMembersRequestHandler { + type RequestType = PrepareMoveMembersRequest; +} + +fn class_item( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + class: HierarchyClass, +) -> Option { + let location = FileRange::new(class.file, class.name_range) + .to_lsp_range(db, snapshot.encoding())? + .to_location()?; + Some(ClassItem { + name: class.name, + qualified_name: class.qualified_name, + uri: location.uri, + range: location.range, + }) +} + +impl BackgroundDocumentRequestHandler for PrepareMoveMembersRequestHandler { + fn document_uri(params: &PrepareMoveMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` outside a class of the project, when language services are off and for a + /// notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PrepareMoveMembersParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + let Some(movable) = movable_members(db, db.program_file(file), offset) else { + return Ok(None); + }; + let Some(class) = class_item(db, snapshot, movable.class.clone()) else { + return Ok(None); + }; + let members = movable + .members + .into_iter() + .filter_map(|member| { + let range = FileRange::new(movable.class.file, member.name_range) + .to_lsp_range(db, snapshot.encoding())? + .local_range(); + Some(MemberItem { + name: member.name, + kind: match member.kind { + MemberKind::Method => "method", + MemberKind::ClassMethod => "classmethod", + MemberKind::StaticMethod => "staticmethod", + MemberKind::Property => "property", + MemberKind::Attribute => "attribute", + } + .to_string(), + range, + }) + }) + .collect(); + Ok(Some(MovableMembers { + class, + members, + superclasses: movable + .superclasses + .into_iter() + .filter_map(|class| class_item(db, snapshot, class)) + .collect(), + subclasses: movable + .subclasses + .into_iter() + .filter_map(|class| class_item(db, snapshot, class)) + .collect(), + })) + } +} + +impl RetriableRequestHandler for PrepareMoveMembersRequestHandler {} + +/// the file and offset of a position in the snapshot's document, `None` where the request is not +/// answered +fn position( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + position: Position, +) -> Option<(ruff_db::files::File, ruff_text_size::TextSize)> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return None; + } + let file = snapshot.document().to_notebook_or_file(db)?; + if !file.source_type(db).is_py_file() { + return None; + } + let offset = position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding())?; + Some((file, offset)) +} + +pub(crate) enum PullMembersUpRequest {} + +impl Request for PullMembersUpRequest { + type Params = PullMembersUpParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pullMembersUp"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the class, the names of the members to pull up, and the superclass to pull them into +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PullMembersUpParams { + text_document: TextDocumentIdentifier, + position: Position, + members: Vec, + /// the superclass's `qualifiedName`, as `by/prepareMoveMembers` lists it + target: String, +} + +pub(crate) struct PullMembersUpRequestHandler; + +impl RequestHandler for PullMembersUpRequestHandler { + type RequestType = PullMembersUpRequest; +} + +impl BackgroundDocumentRequestHandler for PullMembersUpRequestHandler { + fn document_uri(params: &PullMembersUpParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PullMembersUpParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + pull_members_up( + db, + db.program_file(file), + offset, + ¶ms.members, + ¶ms.target, + ), + ))) + } +} + +impl RetriableRequestHandler for PullMembersUpRequestHandler { + /// the user is waiting on the move they asked for + const RETRY_ON_CANCELLATION: bool = true; +} + +pub(crate) enum PushMembersDownRequest {} + +impl Request for PushMembersDownRequest { + type Params = PushMembersDownParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pushMembersDown"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the class, and the names of the members to push down into its subclasses +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PushMembersDownParams { + text_document: TextDocumentIdentifier, + position: Position, + members: Vec, +} + +pub(crate) struct PushMembersDownRequestHandler; + +impl RequestHandler for PushMembersDownRequestHandler { + type RequestType = PushMembersDownRequest; +} + +impl BackgroundDocumentRequestHandler for PushMembersDownRequestHandler { + fn document_uri(params: &PushMembersDownParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PushMembersDownParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + push_members_down(db, db.program_file(file), offset, ¶ms.members), + ))) + } +} + +impl RetriableRequestHandler for PushMembersDownRequestHandler { + /// the user is waiting on the move they asked for + const RETRY_ON_CANCELLATION: bool = true; +} diff --git a/crates/ty_server/src/server/api/requests/move_symbol.rs b/crates/ty_server/src/server/api/requests/move_symbol.rs index 2e496efbed..d88e6d458d 100644 --- a/crates/ty_server/src/server/api/requests/move_symbol.rs +++ b/crates/ty_server/src/server/api/requests/move_symbol.rs @@ -14,7 +14,7 @@ use ruff_db::files::{FileRange, system_path_to_file}; use ruff_db::system::SystemPathBuf; use ruff_text_size::Ranged; use rustc_hash::FxHashMap; -use ty_ide::{MovableKind, movable_symbol, move_symbol}; +use ty_ide::{MovableKind, SymbolMove, movable_symbol, move_symbol}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{FileRangeExt, PositionExt}; @@ -197,12 +197,6 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { else { return Ok(None); }; - let refused = |reason: &str| { - Ok(Some(MoveSymbolAnswer { - refused: Some(reason.to_string()), - ..MoveSymbolAnswer::default() - })) - }; let Some(target) = params .target .to_file_path() @@ -210,12 +204,35 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { .and_then(|path| SystemPathBuf::from_path_buf(path).ok()) .and_then(|path| system_path_to_file(db, &path).ok()) else { - return refused("the target is not a file that exists"); + return Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + Err("the target is not a file that exists".to_string()), + ))); }; - let moved = match move_symbol(db, db.program_file(file), offset, target) { + Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + move_symbol(db, db.program_file(file), offset, target), + ))) + } +} + +impl MoveSymbolAnswer { + /// The answer to a move: its edits as a workspace edit and its warnings, or why it refused. + pub(super) fn of( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + moved: Result, + ) -> Self { + let refused = |reason: String| MoveSymbolAnswer { + refused: Some(reason), + ..MoveSymbolAnswer::default() + }; + let moved = match moved { Ok(moved) => moved, - Err(reason) => return refused(&reason), + Err(reason) => return refused(reason), }; let mut changes: FxHashMap> = FxHashMap::default(); for file_edit in moved.edits { @@ -223,14 +240,14 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { .to_lsp_range(db, snapshot.encoding()) .and_then(|range| range.to_location()) else { - return refused("a file the move changes cannot be edited"); + return refused("a file the move changes cannot be edited".to_string()); }; changes.entry(location.uri).or_default().push(TextEdit { range: location.range, new_text: file_edit.edit.content().unwrap_or_default().to_string(), }); } - Ok(Some(MoveSymbolAnswer { + MoveSymbolAnswer { edit: Some(WorkspaceEdit { changes: Some(changes.into_iter().collect()), document_changes: None, @@ -238,7 +255,7 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { }), refused: None, warnings: moved.warnings, - })) + } } } diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index 1eea0db91d..26632175df 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -157,3 +157,172 @@ fn a_refused_move_says_why() -> Result<()> { ); Ok(()) } + +/// the import the definition takes with it and the definition are removed as one edit: two +/// removals that both claim the blank lines between them would overlap, which a client rejects +#[test] +fn a_move_s_edits_to_a_file_do_not_overlap() -> Result<()> { + const CIRCLE: &str = "import math\n\n\ndef area(r): return math.pi*r*r\n"; + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes/__init__.py"), "")? + .with_file(SystemPath::new("src/shapes/circle.py"), CIRCLE)? + .with_file(SystemPath::new("src/shapes/geometry.py"), "")? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes/circle.py"), CIRCLE, 1); + let circle = server.file_uri(SystemPath::new("src/shapes/circle.py")); + let answer = server + .send_request_await::(json!({ + "textDocument": { "uri": circle }, + "position": { "line": 3, "character": 5 }, + "target": server.file_uri(SystemPath::new("src/shapes/geometry.py")), + })) + .expect("an answer"); + assert_eq!( + answer["edit"]["changes"][circle.as_str()], + json!([{ + "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 4, "character": 0 } }, + "newText": "", + }]), + "{answer:#}" + ); + Ok(()) +} + +/// `by/prepareMoveMembers`, in json throughout +enum PrepareMoveMembers {} + +impl Request for PrepareMoveMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareMoveMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/pullMembersUp`, in json throughout +enum PullMembersUp {} + +impl Request for PullMembersUp { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pullMembersUp"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/pushMembersDown`, in json throughout +enum PushMembersDown {} + +impl Request for PushMembersDown { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/pushMembersDown"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const SHAPES: &str = "\ +class Shape: + pass + + +class Square(Shape): + SIDES = 4 + + def describe(self) -> str: + return 'square' +"; + +fn shapes_server() -> Result { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.py"), SHAPES)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.py"), SHAPES, 1); + Ok(server) +} + +#[test] +fn a_class_lists_its_members_and_where_they_could_go() -> Result<()> { + let mut server = shapes_server()?; + let uri = server.file_uri(SystemPath::new("src/shapes.py")); + let answer = server.send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 7 }, + })); + assert_eq!( + answer, + Some(json!({ + "class": { + "name": "Square", + "qualifiedName": "shapes.Square", + "uri": uri, + "range": { "start": { "line": 4, "character": 6 }, "end": { "line": 4, "character": 12 } }, + }, + "members": [ + { + "name": "SIDES", + "kind": "attribute", + "range": { "start": { "line": 5, "character": 4 }, "end": { "line": 5, "character": 9 } }, + }, + { + "name": "describe", + "kind": "method", + "range": { "start": { "line": 7, "character": 8 }, "end": { "line": 7, "character": 16 } }, + }, + ], + "superclasses": [{ + "name": "Shape", + "qualifiedName": "shapes.Shape", + "uri": uri, + "range": { "start": { "line": 0, "character": 6 }, "end": { "line": 0, "character": 11 } }, + }], + "subclasses": [], + })) + ); + Ok(()) +} + +#[test] +fn a_pull_up_answers_its_edit_and_a_push_down_with_nowhere_to_go_is_refused() -> Result<()> { + let mut server = shapes_server()?; + let uri = server.file_uri(SystemPath::new("src/shapes.py")); + let pulled = server + .send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 7 }, + "members": ["describe"], + "target": "shapes.Shape", + })) + .expect("an answer"); + assert_eq!( + pulled["edit"]["changes"][uri.as_str()], + json!([ + { + // the blank line above it and the method, to the end of the file + "range": { "start": { "line": 6, "character": 0 }, "end": { "line": 9, "character": 0 } }, + "newText": "", + }, + { + "range": { "start": { "line": 1, "character": 0 }, "end": { "line": 2, "character": 0 } }, + "newText": " def describe(self) -> str:\n return 'square'\n", + }, + ]), + "{pulled:#}" + ); + + let pushed = server.send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 7 }, + "members": ["SIDES"], + })); + assert_eq!( + pushed, + Some(json!({ + "refused": "no class of the project names Square as a base, so there is nowhere to \ + push to", + "warnings": [], + })) + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 19d4df675c..ac768af376 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -627,6 +627,59 @@ the same and `target`, the URI of an existing `.py` or `.by` file of the project left as they were, each with its file and line, or `{refused, warnings}`, the reason nothing is moved. two edits at one position of a file apply in the order listed +### pulling members up and pushing them down + +a class's members — a method, with the accessors or overloads that share its name, +or a class attribute — move into one of the project's classes it inherits from +with `by/pullMembersUp`, or into the project's classes that name it as a base with +`by/pushMembersDown`, a copy into each that does not define the member itself. each +moves with its decorators and the comments above it, reindented for the body it +lands in: attributes before the methods, methods after them, in the order they +were written + +either is refused, with the reason, where the program would no longer mean what it +meant: + +- pulled up, a member that reads an attribute of `self` the superclass does not + have — which the superclass's other subclasses would miss — unless that is pulled + up with it; that calls `super()` from a class whose one base is not the + superclass; that a class below the superclass would then find in another class + than it does, by its method resolution order — another subclass that finds it + in a mixin or above the superclass, or the class itself in a class it finds + before the superclass; that reads, where the class body evaluates it — a value, + a default, a decorator — a name the class body binds that is not pulled with it, + or a name bound after the superclass is defined; or that the superclass has + already +- pushed down, a member something reads off the class itself, or off what may be an + instance of the class rather than of a subclass — another of the class's methods, + through `self`, included — or of a class nothing in the project subclasses +- a name-mangled `__member`, whose name depends on the class it is written in, a + method that reads `__class__`, which is the class it is written in, a member that + reads a name one class finds in a scope around it — a function it is defined in, + say — and the other finds elsewhere, and, between modules, a member that reads one + of its module's own names that the other module would have to import from a module + that imports it, or for any reason a moved + function is refused for between modules ([moving a definition](#moving-a-definition)): + a `global` write, a name its module binds more than once, a private name, an + attribute of its module object, a builtin or a name the other module binds to + something else. the imports a member reads go with it + +`override` follows the move. a method pulled up where nothing above has its name +overrides nothing there, so it loses the modifier, and the methods of that name +below the superclass now override it and gain one — `override def` in basedpython, +`@override` with its import in python. a method pushed down keeps it only where +something above still has the name, and a subclass's own method of that name loses +it once nothing above has the name + +`by/prepareMoveMembers` takes `textDocument` and `position` (anywhere in a class) +and answers `{class, members, superclasses, subclasses}`: the class and the classes +of the project above it, nearest first, and below it, each `{name, qualifiedName, uri, range}` with `range` its name; and its members, each `{name, kind, range}`, +`kind` one of `method`, `classmethod`, `staticmethod`, `property` and `attribute`; +`null` outside a class. `by/pullMembersUp` takes the same position, `members`, the +names to move, and `target`, a superclass's `qualifiedName`; `by/pushMembersDown` +takes the position and `members`. both answer as `by/moveSymbol` does, +`{edit, warnings}` or `{refused, warnings}` + ## unwrap and remove `by/unwrap` answers what an editor's *unwrap/remove* offers around a range, innermost From d2917ad5c58c851e7f295e8247bb1ae2d5f0bee2 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:03:36 +1000 Subject: [PATCH 036/110] change a function's parameters with by/changeSignature, after by/prepareChangeSignature lists them, rebinding every call in the project as python binds it and writing it again for the new list, renaming a renamed parameter in the body and in keywords, and refusing a change a call that unpacks, drops or reorders an argument with an effect, or lacks a value would not survive --- crates/ty_ide/src/lib.rs | 4 + crates/ty_ide/src/refactor.rs | 5 + .../ty_ide/src/refactor/change_signature.rs | 1259 +++++++++++++++++ crates/ty_server/src/server/api.rs | 11 + crates/ty_server/src/server/api/requests.rs | 4 + .../server/api/requests/change_signature.rs | 255 ++++ .../ty_server/tests/e2e/change_signature.rs | 120 ++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 48 + 9 files changed, 1707 insertions(+) create mode 100644 crates/ty_ide/src/refactor/change_signature.rs create mode 100644 crates/ty_server/src/server/api/requests/change_signature.rs create mode 100644 crates/ty_server/tests/e2e/change_signature.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index d56068d6e0..79cd3b24fd 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -114,6 +114,10 @@ pub use refactor::{ RefactorKind, RefactorOffer, SymbolMove, movable_members, movable_symbol, move_symbol, pull_members_up, push_members_down, refactor, refactors, }; +pub use refactor::{ + FunctionSignature, NewParameter, ParameterKind, SignatureParameter, change_signature, + function_signature, +}; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; pub use selection_range::selection_range; diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 5206ad57cd..b58451c06e 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -15,6 +15,7 @@ //! hands its edits back rather than throwing them away, and the caller tells it //! which refactorings to consider rather than being given all of them. +mod change_signature; mod class_members; mod data_class; mod evaluation; @@ -41,6 +42,10 @@ use ty_python_semantic::SemanticModel; use crate::FileEdit; +pub use change_signature::{ + FunctionSignature, NewParameter, ParameterKind, SignatureParameter, change_signature, + function_signature, +}; pub use class_members::{ ClassMember, HierarchyClass, MemberKind, MovableMembers, movable_members, pull_members_up, push_members_down, diff --git a/crates/ty_ide/src/refactor/change_signature.rs b/crates/ty_ide/src/refactor/change_signature.rs new file mode 100644 index 0000000000..d5066d4ac2 --- /dev/null +++ b/crates/ty_ide/src/refactor/change_signature.rs @@ -0,0 +1,1259 @@ +//! Changing a function's parameters, and every call of it in the project with them. +//! +//! ```py +//! def area(width, height=1): def area(height, width, *, scale=1.0): +//! return width * height ──► return width * height * scale +//! +//! area(2, 3) area(3, 2) +//! area(width=2) area(width=2, height=1) # when `height` loses its default +//! ``` +//! +//! The new parameter list is given in full: each parameter by its name, the parameter it was +//! (none for a new one), its kind, its default, and for a new one the value each call passes. +//! A renamed parameter is renamed where the body reads it and where calls pass it by keyword. +//! +//! Every call is rebound the way python binds it — its positional arguments to the positional +//! parameters, its keywords by name — and written again for the new list: by position while the +//! parameters still line up, by keyword after that, and keeping a keyword a keyword. The change is +//! refused, with the reason, where a call cannot be rewritten so that it means what it meant: +//! +//! - a call that unpacks `*args` or `**kwargs` into the function, whose arguments cannot be told +//! apart, unless the change leaves every existing call valid as it is; +//! - an argument the change would drop, or evaluate after one written after it, when evaluating it +//! could have an effect; +//! - a call with nothing to pass for a parameter without a default; +//! - the function used other than called — passed as a value, say — unless every existing call +//! stays valid; +//! - a removed parameter the body still reads, and a list python would not accept: a parameter +//! without a default after one with a default, two of a name, kinds out of order; +//! - a method another class overrides or that overrides another's, whose signatures would part. + +use ruff_db::files::File; +use ruff_db::parsed::parsed_module; +use ruff_db::source::source_text; +use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr}; +use ruff_python_stdlib::identifiers::is_identifier; +use ruff_python_stdlib::keyword::is_keyword; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashMap; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_semantic::types::Type; +use ty_python_semantic::{HasType, SemanticModel}; + +use super::evaluation::is_effect_free; +use super::move_symbol::{SymbolMove, line_number}; +use crate::code_action::FileEdit; +use crate::goto::find_goto_target; +use crate::method_kind::{MethodKind, resolved_method_kind}; +use crate::references::{ReferencesMode, references}; + +/// How a parameter is passed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum ParameterKind { + /// Before a `/`: only by position. + PositionalOnly, + PositionalOrKeyword, + /// `*args`. + Variadic, + /// After a `*` or `*args`: only by keyword. + KeywordOnly, + /// `**kwargs`. + VariadicKeyword, +} + +impl ParameterKind { + fn is_positional(self) -> bool { + matches!(self, Self::PositionalOnly | Self::PositionalOrKeyword) + } + + fn is_variadic(self) -> bool { + matches!(self, Self::Variadic | Self::VariadicKeyword) + } +} + +/// A parameter of a function as it is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SignatureParameter { + pub name: String, + pub kind: ParameterKind, + /// Its default, as the source writes it. + pub default: Option, + /// Its annotation, as the source writes it. + pub annotation: Option, +} + +/// A function whose signature [`change_signature`] can change. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FunctionSignature { + /// The file it is defined in, which need not be the one asked about. + pub file: File, + pub name: String, + pub name_range: TextRange, + /// Its parameters, without the `self` or `cls` a method is called with, which stays as it + /// is. + pub parameters: Vec, +} + +/// A parameter of the list [`change_signature`] writes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewParameter { + pub name: String, + /// The parameter it is, for one the function has; `None` for a new one. + pub old_name: Option, + pub kind: ParameterKind, + pub default: Option, + /// For a new parameter, what each call passes it; none leaves it to its default. + pub value: Option, +} + +/// The function defined or named at `offset`, and its parameters. +pub fn function_signature( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option { + let (file, offset) = definition_site(db, file, offset)?; + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let (function, receiver) = function_at(db, file, &parsed, offset)?; + Some(FunctionSignature { + file: file.file(db), + name: function.name.to_string(), + name_range: function.name.range(), + parameters: old_parameters(source.as_str(), function, receiver) + .into_iter() + .map(|old| old.parameter) + .collect(), + }) +} + +/// Gives the function defined or named at `offset` the parameters `parameters`, and rewrites every +/// call of it in the project to pass them. `Err` says why it cannot be changed. +pub fn change_signature( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + parameters: &[NewParameter], +) -> Result { + let (file, offset) = + definition_site(db, file, offset).ok_or_else(|| "there is no function here".to_string())?; + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let source = source.as_str(); + let (function, receiver) = function_at(db, file, &parsed, offset) + .ok_or_else(|| "there is no function here".to_string())?; + let name = function.name.as_str(); + let old = old_parameters(source, function, receiver); + let receiver_name = receiver.then(|| first_parameter_name(function)).flatten(); + + validate(name, &old, parameters, receiver_name.as_deref())?; + + if receiver { + // what it overrides in the project would part from it; a class of a library or the + // standard library, `object` above all, is not the project's to change + let overrides = crate::super_members(db, file, function.name.start()).unwrap_or_default(); + if let Some(overridden) = overrides + .iter() + .find(|overridden| db.project().files(db).contains(overridden.target.file())) + { + return Err(format!( + "`{name}` overrides {}.{name}, whose signature would no longer match", + overridden.superclass + )); + } + if let Some(implementations) = crate::goto_implementation(db, file, function.name.start()) + && implementations.value.into_iter().any(|target| { + target.file() != file.file(db) || target.focus_range() != function.name.range() + }) + { + return Err(format!( + "`{name}` is overridden, and the overrides' signatures would no longer match" + )); + } + } + + let mut moved = SymbolMove::default(); + let file_id = file.file(db); + + // the body: renamed parameters renamed where it reads them; removed ones must be unread + let model = SemanticModel::new(db, file); + for parameter in &old { + let new = parameters + .iter() + .find(|new| new.old_name.as_deref() == Some(parameter.parameter.name.as_str())); + if new.is_some_and(|new| new.name == parameter.parameter.name) { + continue; + } + let Some(goto) = find_goto_target(&model, &parsed, parameter.name_range.start()) else { + continue; + }; + let reads: Vec = references(db, file, &goto, ReferencesMode::Rename) + .unwrap_or_default() + .into_iter() + .filter(|reference| { + reference.file() == file_id + && reference.range() != parameter.name_range + && function.range().contains_range(reference.range()) + && !function + .parameters + .range() + .contains_range(reference.range()) + }) + .map(|reference| reference.range()) + .collect(); + match new { + Some(new) => moved.edits.extend(reads.into_iter().map(|range| FileEdit { + file: file_id, + edit: Edit::range_replacement(new.name.clone(), range), + })), + None => { + if let Some(read) = reads.first() { + return Err(format!( + "the body still reads `{}`, on line {}", + parameter.parameter.name, + line_number(source, read.start()) + )); + } + } + } + } + + // the definition + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::range_replacement( + parameter_list(source, function, receiver_name.as_deref(), &old, parameters), + function.parameters.range(), + ), + }); + + // the calls + let compatible = is_compatible(&old, parameters); + let goto = find_goto_target(&model, &parsed, function.name.start()) + .ok_or_else(|| format!("`{name}` cannot be searched for"))?; + let mut by_file: FxHashMap> = FxHashMap::default(); + for reference in + references(db, file, &goto, ReferencesMode::RenameMultiFile).unwrap_or_default() + { + if reference.file() == file_id && reference.range() == function.name.range() { + continue; + } + by_file + .entry(reference.file()) + .or_default() + .push(reference.range()); + } + let mut files: Vec = by_file.keys().copied().collect(); + files.sort_by_key(|file| file.path(db).as_str().to_string()); + for reference_file in files { + let program_file = ProgramFile::new(db, reference_file, file.program(db)); + let parsed = parsed_module(db, program_file.python_file(db)).load(db); + let text = source_text(db, reference_file); + let text = text.as_str(); + let model = SemanticModel::new(db, program_file); + let path = relative_path(db, reference_file); + for range in &by_file[&reference_file] { + let covering = covering_node(parsed.syntax().into(), *range); + let call = call_of(&covering); + let Some((call, bound)) = + call.map(|call| (call, is_bound_call(&model, call, receiver, function))) + else { + // an import of it, or a mention in its own `__all__`, needs nothing + if covering.ancestors().any(|node| { + matches!( + node, + AnyNodeRef::StmtImportFrom(_) | AnyNodeRef::StmtImport(_) + ) + }) { + continue; + } + if compatible { + continue; + } + return Err(format!( + "{path}:{} uses `{name}` other than by calling it, and its calls could not \ + be rewritten", + line_number(text, range.start()) + )); + }; + if compatible { + continue; + } + let rewritten = rewrite_call(&model, text, call, bound, &old, parameters) + .map_err(|reason| format!("{path}:{} {reason}", line_number(text, call.start())))?; + if let Some(rewritten) = rewritten { + moved.edits.push(FileEdit { + file: reference_file, + edit: Edit::range_replacement(rewritten, call.arguments.range()), + }); + } + } + } + Ok(moved) +} + +/// A parameter of the function as it is, with where its name is. +struct OldParameter { + parameter: SignatureParameter, + name_range: TextRange, + /// Its text without its default: its modifiers, name and annotation. + text: String, + /// The range of `text`'s name within it. + name_in_text: TextRange, +} + +fn old_parameters( + source: &str, + function: &ast::StmtFunctionDef, + receiver: bool, +) -> Vec { + let parameters = &function.parameters; + let mut old = Vec::new(); + let mut push = |parameter: &ast::Parameter, kind: ParameterKind, default: Option<&Expr>| { + let text = source[parameter.range()].to_string(); + old.push(OldParameter { + parameter: SignatureParameter { + name: parameter.name.to_string(), + kind, + default: default.map(|default| source[default.range()].to_string()), + annotation: parameter + .annotation + .as_ref() + .map(|annotation| source[annotation.range()].to_string()), + }, + name_range: parameter.name.range(), + name_in_text: parameter.name.range() - parameter.start(), + text, + }); + }; + let mut positional = parameters + .posonlyargs + .iter() + .map(|parameter| (parameter, ParameterKind::PositionalOnly)) + .chain( + parameters + .args + .iter() + .map(|parameter| (parameter, ParameterKind::PositionalOrKeyword)), + ); + if receiver { + positional.next(); + } + for (parameter, kind) in positional { + push(¶meter.parameter, kind, parameter.default.as_deref()); + } + if let Some(vararg) = ¶meters.vararg { + push(vararg, ParameterKind::Variadic, None); + } + for parameter in ¶meters.kwonlyargs { + push( + ¶meter.parameter, + ParameterKind::KeywordOnly, + parameter.default.as_deref(), + ); + } + if let Some(kwarg) = ¶meters.kwarg { + push(kwarg, ParameterKind::VariadicKeyword, None); + } + old +} + +fn first_parameter_name(function: &ast::StmtFunctionDef) -> Option { + function + .parameters + .posonlyargs + .iter() + .chain(&function.parameters.args) + .next() + .map(|parameter| parameter.parameter.name.to_string()) +} + +/// Checks that `parameters` is a list python accepts, made of the function's own parameters and +/// new ones. +fn validate( + name: &str, + old: &[OldParameter], + parameters: &[NewParameter], + receiver: Option<&str>, +) -> Result<(), String> { + let mut seen: Vec<&str> = Vec::new(); + let mut old_seen: Vec<&str> = Vec::new(); + let mut previous_kind = ParameterKind::PositionalOnly; + let mut positional_default: Option<&str> = None; + for parameter in parameters { + let new_name = parameter.name.as_str(); + if !is_identifier(new_name) || is_keyword(new_name) { + return Err(format!("`{new_name}` is not a name a parameter can have")); + } + if seen.contains(&new_name) || receiver == Some(new_name) { + return Err(format!( + "`{name}` would have two parameters named `{new_name}`" + )); + } + seen.push(new_name); + if parameter.kind < previous_kind + || (parameter.kind == previous_kind && parameter.kind.is_variadic()) + { + return Err(format!( + "`{new_name}` is out of order: python takes positional-only parameters, then the \ + others, `*args`, keyword-only ones and `**kwargs`, one of each `*` kind" + )); + } + previous_kind = parameter.kind; + if parameter.kind.is_variadic() + && (parameter.default.is_some() || parameter.value.is_some()) + { + return Err(format!( + "`{new_name}` takes what is left over, so it has no default" + )); + } + if parameter.kind.is_positional() { + match (¶meter.default, positional_default) { + (Some(_), _) => positional_default = Some(new_name), + (None, Some(with_default)) => { + return Err(format!( + "`{new_name}` has no default but follows `{with_default}`, which has one" + )); + } + (None, None) => {} + } + } + match ¶meter.old_name { + Some(old_name) => { + let Some(existing) = old + .iter() + .find(|existing| existing.parameter.name == *old_name) + else { + return Err(format!("`{name}` has no parameter `{old_name}`")); + }; + if old_seen.contains(&old_name.as_str()) { + return Err(format!("`{old_name}` is listed twice")); + } + old_seen.push(old_name); + if existing.parameter.kind.is_variadic() != parameter.kind.is_variadic() + || (existing.parameter.kind.is_variadic() + && existing.parameter.kind != parameter.kind) + { + return Err(format!( + "`{old_name}` cannot change between taking one argument and taking what \ + is left over" + )); + } + } + None => { + if parameter.kind.is_variadic() { + continue; + } + } + } + } + Ok(()) +} + +/// Whether every call of the function as it is stays valid, and means the same, against +/// `parameters`: the parameters it has keep their names, kinds, places and defaults, and those +/// added after them all have defaults. +fn is_compatible(old: &[OldParameter], parameters: &[NewParameter]) -> bool { + let mut new = parameters.iter().peekable(); + for existing in old { + // added keyword-only ones may come before `**kwargs` + while let Some(added) = new.peek() + && added.old_name.is_none() + && existing.parameter.kind == ParameterKind::VariadicKeyword + { + if added.default.is_none() && !added.kind.is_variadic() { + return false; + } + new.next(); + } + let Some(next) = new.next() else { + return false; + }; + if next.old_name.as_deref() != Some(existing.parameter.name.as_str()) + || next.name != existing.parameter.name + || next.kind != existing.parameter.kind + || (existing.parameter.default.is_some() && next.default.is_none()) + { + return false; + } + } + new.all(|added| { + added.old_name.is_none() + && (added.default.is_some() || added.kind.is_variadic()) + && !matches!(added.kind, ParameterKind::PositionalOnly) + }) +} + +/// The parenthesized parameter list `parameters` makes, the receiver first. +fn parameter_list( + source: &str, + function: &ast::StmtFunctionDef, + receiver: Option<&str>, + old: &[OldParameter], + parameters: &[NewParameter], +) -> String { + let mut parts: Vec = Vec::new(); + if receiver.is_some() + && let Some(first) = function + .parameters + .posonlyargs + .iter() + .chain(&function.parameters.args) + .next() + { + parts.push(source[first.range()].to_string()); + if !function.parameters.posonlyargs.is_empty() + && !parameters + .iter() + .any(|parameter| parameter.kind == ParameterKind::PositionalOnly) + { + parts.push("/".to_string()); + } + } + let has_variadic = parameters + .iter() + .any(|parameter| parameter.kind == ParameterKind::Variadic); + let mut previous: Option = None; + for parameter in parameters { + if previous == Some(ParameterKind::PositionalOnly) + && parameter.kind != ParameterKind::PositionalOnly + { + parts.push("/".to_string()); + } + if parameter.kind == ParameterKind::KeywordOnly + && !has_variadic + && previous.is_none_or(|previous| previous < ParameterKind::Variadic) + { + parts.push("*".to_string()); + } + previous = Some(parameter.kind); + let existing = parameter.old_name.as_ref().and_then(|old_name| { + old.iter() + .find(|existing| existing.parameter.name == *old_name) + }); + let mut text = match existing { + Some(existing) => { + let mut text = existing.text.clone(); + text.replace_range(existing.name_in_text.to_std_range(), ¶meter.name); + text + } + None => match parameter.kind { + ParameterKind::Variadic => format!("*{}", parameter.name), + ParameterKind::VariadicKeyword => format!("**{}", parameter.name), + _ => parameter.name.clone(), + }, + }; + if let Some(default) = ¶meter.default { + let annotated = + existing.is_some_and(|existing| existing.parameter.annotation.is_some()); + text.push_str(if annotated { " = " } else { "=" }); + text.push_str(default); + } + parts.push(text); + } + if previous == Some(ParameterKind::PositionalOnly) { + parts.push("/".to_string()); + } + format!("({})", parts.join(", ")) +} + +/// An argument of a call, as python binds it. +#[derive(Clone, Copy)] +struct Argument<'a> { + value: &'a Expr, + keyword: Option<&'a str>, +} + +/// The arguments of `call` rewritten for `parameters`, or `None` when they stay as they are. +fn rewrite_call( + model: &SemanticModel<'_>, + source: &str, + call: &ast::ExprCall, + bound: bool, + old: &[OldParameter], + parameters: &[NewParameter], +) -> Result, String> { + let arguments = &call.arguments; + if arguments + .args + .iter() + .any(|argument| matches!(argument, Expr::Starred(_))) + || arguments + .keywords + .iter() + .any(|keyword| keyword.arg.is_none()) + { + return Err("unpacks arguments into the call, which cannot be told apart".to_string()); + } + + // bind the call as python binds it to the function as it is + let mut positionals = arguments.args.iter(); + let receiver_argument = if bound { None } else { positionals.next() }; + let mut bound_to: Vec> = vec![None; old.len()]; + let mut extra_positional: Vec<&Expr> = Vec::new(); + let mut extra_keyword: Vec = Vec::new(); + let positional_slots: Vec = old + .iter() + .enumerate() + .filter(|(_, parameter)| parameter.parameter.kind.is_positional()) + .map(|(index, _)| index) + .collect(); + for (position, value) in positionals.enumerate() { + match positional_slots.get(position) { + Some(&slot) => { + bound_to[slot] = Some(Argument { + value, + keyword: None, + }); + } + None if old + .iter() + .any(|parameter| parameter.parameter.kind == ParameterKind::Variadic) => + { + extra_positional.push(value); + } + None => return Err("passes more arguments than the function takes".to_string()), + } + } + for keyword in &arguments.keywords { + let Some(name) = &keyword.arg else { + continue; + }; + let argument = Argument { + value: &keyword.value, + keyword: Some(name.as_str()), + }; + match old.iter().position(|parameter| { + parameter.parameter.name == name.as_str() + && matches!( + parameter.parameter.kind, + ParameterKind::PositionalOrKeyword | ParameterKind::KeywordOnly + ) + }) { + Some(slot) => bound_to[slot] = Some(argument), + None if old + .iter() + .any(|parameter| parameter.parameter.kind == ParameterKind::VariadicKeyword) => + { + extra_keyword.push(argument); + } + None => return Err(format!("passes `{name}`, which the function does not take")), + } + } + + // write them again for the new list + let mut by_position: Vec<(Option, String)> = Vec::new(); + let mut by_keyword: Vec<(Option, String)> = Vec::new(); + let mut positional_ok = true; + let mut used = vec![false; old.len()]; + for parameter in parameters { + match parameter.kind { + ParameterKind::Variadic => { + if !extra_positional.is_empty() { + if !positional_ok { + return Err(format!( + "passes extra arguments by position, which can no longer reach \ + `*{}`", + parameter.name + )); + } + by_position.extend( + extra_positional + .iter() + .map(|value| (Some(value.start()), source[value.range()].to_string())), + ); + extra_positional.clear(); + } + continue; + } + ParameterKind::VariadicKeyword => { + by_keyword.extend(extra_keyword.iter().map(|argument| { + ( + Some(argument.value.start()), + format!( + "{}={}", + argument.keyword.unwrap_or_default(), + &source[argument.value.range()] + ), + ) + })); + extra_keyword.clear(); + continue; + } + _ => {} + } + let slot = parameter.old_name.as_ref().and_then(|old_name| { + old.iter() + .position(|existing| existing.parameter.name == *old_name) + }); + if let Some(slot) = slot { + used[slot] = true; + } + let passed = slot.and_then(|slot| bound_to[slot]); + let (start, value, was_keyword) = match (passed, ¶meter.value) { + (Some(argument), _) => ( + Some(argument.value.start()), + source[argument.value.range()].to_string(), + argument.keyword.is_some(), + ), + (None, Some(value)) if slot.is_none() => (None, value.clone(), false), + (None, _) => { + if parameter.default.is_none() { + return Err(format!("has nothing to pass for `{}`", parameter.name)); + } + positional_ok = false; + continue; + } + }; + if positional_ok && parameter.kind.is_positional() && !was_keyword { + by_position.push((start, value)); + } else { + if parameter.kind == ParameterKind::PositionalOnly { + return Err(format!( + "would have to pass `{}` by keyword, which it only takes by position", + parameter.name + )); + } + positional_ok = false; + by_keyword.push((start, format!("{}={value}", parameter.name))); + } + } + if !extra_positional.is_empty() || !extra_keyword.is_empty() { + return Err("passes extra arguments the new parameters have nowhere to take".to_string()); + } + + // what is dropped, or moved after something written after it, must have no effect + for (slot, argument) in bound_to.iter().enumerate() { + if let Some(argument) = argument + && !used[slot] + && !is_effect_free(model, argument.value) + { + return Err(format!( + "passes `{}` for `{}`, which would no longer be evaluated", + &source[argument.value.range()], + old[slot].parameter.name + )); + } + } + by_keyword.sort_by_key(|(start, _)| start.unwrap_or(TextSize::from(u32::MAX))); + let order: Vec = by_position + .iter() + .chain(&by_keyword) + .filter_map(|(start, _)| *start) + .collect(); + for (earlier, later) in order.iter().zip(order.iter().skip(1)) { + if later < earlier { + let moved = arguments + .args + .iter() + .chain(arguments.keywords.iter().map(|keyword| &keyword.value)) + .find(|value| value.start() == *earlier); + if let Some(moved) = moved + && !is_effect_free(model, moved) + { + return Err(format!( + "would evaluate `{}` after an argument written after it", + &source[moved.range()] + )); + } + } + } + + let mut written: Vec = Vec::new(); + if let Some(receiver) = receiver_argument { + written.push(source[receiver.range()].to_string()); + } + written.extend(by_position.into_iter().map(|(_, text)| text)); + written.extend(by_keyword.into_iter().map(|(_, text)| text)); + let rewritten = format!("({})", written.join(", ")); + let original: Vec<&str> = arguments + .args + .iter() + .map(|value| &source[value.range()]) + .chain( + arguments + .keywords + .iter() + .map(|keyword| &source[keyword.range()]), + ) + .collect(); + if format!("({})", original.join(", ")) == rewritten { + return Ok(None); + } + Ok(Some(rewritten)) +} + +/// The call whose callee the reference at `covering` names: `f` of `f(...)` or of `x.f(...)`. +fn call_of<'a>( + covering: &ruff_python_ast::find_node::CoveringNode<'a>, +) -> Option<&'a ast::ExprCall> { + let mut ancestors = covering.ancestors(); + let mut callee = ancestors.next()?; + if matches!(callee, AnyNodeRef::Identifier(_)) { + callee = ancestors.next()?; + } + if !matches!( + callee, + AnyNodeRef::ExprName(_) | AnyNodeRef::ExprAttribute(_) + ) { + return None; + } + match ancestors.next()? { + AnyNodeRef::ExprCall(call) if call.func.range() == callee.range() => Some(call), + _ => None, + } +} + +/// Whether `call` passes the method's receiver itself, as a call through an instance does, rather +/// than as its first argument, as a call through the class does. +fn is_bound_call( + model: &SemanticModel<'_>, + call: &ast::ExprCall, + receiver: bool, + function: &ast::StmtFunctionDef, +) -> bool { + if !receiver { + return true; + } + let Expr::Attribute(attribute) = &*call.func else { + return false; + }; + let through_class = matches!( + attribute.value.inferred_type(model), + Some(Type::ClassLiteral(_) | Type::SubclassOf(_) | Type::GenericAlias(_)) + ); + !through_class || resolved_method_kind(model, function) == Some(MethodKind::Class) +} + +/// The function of `file` whose definition `offset` is in the name or the parameters of, and +/// whether it is a method that is passed its receiver. +fn function_at<'p>( + db: &dyn Db, + file: ProgramFile<'_>, + parsed: &'p ruff_db::parsed::ParsedModuleRef, + offset: TextSize, +) -> Option<(&'p ast::StmtFunctionDef, bool)> { + let covering = covering_node(parsed.syntax().into(), TextRange::empty(offset)); + let function = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtFunctionDef(function) => Some(function), + _ => None, + })?; + if !(function.name.range().contains_inclusive(offset) + || function.parameters.range().contains_inclusive(offset)) + { + return None; + } + // only a function written directly in a class body is a method + let receiver = is_in_class_body(parsed, function) + && resolved_method_kind(&SemanticModel::new(db, file), function)? != MethodKind::Static; + Some((function, receiver)) +} + +fn is_in_class_body( + parsed: &ruff_db::parsed::ParsedModuleRef, + function: &ast::StmtFunctionDef, +) -> bool { + let covering = covering_node(parsed.syntax().into(), function.range()); + covering + .ancestors() + .skip_while(|node| node.range() == function.range()) + .find(|node| node.is_statement()) + .is_some_and(|node| matches!(node, AnyNodeRef::StmtClassDef(_))) +} + +/// The file and offset of the function named at `offset`: `offset` itself when it is in a +/// definition's name or parameters, or else where the name at `offset` is defined. +fn definition_site<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + offset: TextSize, +) -> Option<(ProgramFile<'db>, TextSize)> { + let parsed = parsed_module(db, file.python_file(db)).load(db); + if function_at(db, file, &parsed, offset).is_some() { + return Some((file, offset)); + } + let targets = crate::goto_definition(db, file, offset)?; + let mut targets = targets.value.into_iter(); + let target = targets.next()?; + if targets.next().is_some() || !db.project().files(db).contains(target.file()) { + return None; + } + let target_file = ProgramFile::new(db, target.file(), file.program(db)); + let target_parsed = parsed_module(db, target_file.python_file(db)).load(db); + function_at( + db, + target_file, + &target_parsed, + target.focus_range().start(), + )?; + Some((target_file, target.focus_range().start())) +} + +/// `file`'s path relative to the project's root. +fn relative_path(db: &dyn Db, file: File) -> String { + let path = file.path(db); + let root = db.project().root(db); + path.as_system_path() + .and_then(|path| path.strip_prefix(root).ok()) + .map_or_else(|| path.to_string(), ToString::to_string) +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + + use super::{NewParameter, ParameterKind, change_signature, function_signature}; + use crate::refactor::move_symbol::tests::rendered; + use crate::tests::{CursorTest, CursorTestBuilder}; + + const GEO: &str = r#"def area(width, height=1, *, scale=1.0): + return width * height * scale + + +def forward(*args, **kwargs): + return area(*args, **kwargs) + + +print(area(2, 3)) +print(area(2, height=3, scale=2.0)) +print(area(width=2)) +"#; + + fn project(geo: &str) -> CursorTest { + CursorTestBuilder::default() + .source("sig/__init__.py", "") + .source("sig/geo.py", geo) + .source( + "sig/user.py", + "from sig.geo import area\n\narea(4, 5, scale=3)\n", + ) + .build() + } + + /// `name`, as the parameter `old` was, of `kind`, with `default`. + fn kept(name: &str, old: &str, kind: ParameterKind, default: Option<&str>) -> NewParameter { + NewParameter { + name: name.to_string(), + old_name: Some(old.to_string()), + kind, + default: default.map(ToString::to_string), + value: None, + } + } + + fn added( + name: &str, + kind: ParameterKind, + default: Option<&str>, + value: Option<&str>, + ) -> NewParameter { + NewParameter { + name: name.to_string(), + old_name: None, + kind, + default: default.map(ToString::to_string), + value: value.map(ToString::to_string), + } + } + + use ParameterKind::{KeywordOnly, PositionalOrKeyword}; + + impl CursorTest { + fn change(&self, parameters: &[NewParameter]) -> String { + let changed = salsa::attach(&self.db, || { + change_signature( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + parameters, + ) + }); + rendered(&self.db, changed) + } + } + + #[test] + fn the_signature_of_a_function_is_its_parameters() { + let test = project(GEO); + let signature = salsa::attach(&test.db, || { + function_signature( + &test.db, + test.program_file(test.cursor.file), + test.cursor.offset, + ) + }); + assert_snapshot!(format!("{signature:#?}"), @r#" + Some( + FunctionSignature { + file: file( + Id(101), + ), + name: "area", + name_range: 4..8, + parameters: [ + SignatureParameter { + name: "width", + kind: PositionalOrKeyword, + default: None, + annotation: None, + }, + SignatureParameter { + name: "height", + kind: PositionalOrKeyword, + default: Some( + "1", + ), + annotation: None, + }, + SignatureParameter { + name: "scale", + kind: KeywordOnly, + default: Some( + "1.0", + ), + annotation: None, + }, + ], + }, + ) + "#); + } + + /// a method made static by a decorator under another name is passed no receiver, so its + /// first parameter is one of the caller's + #[test] + fn a_static_method_under_another_name_lists_its_first_parameter() { + let test = CursorTestBuilder::default() + .source( + "main.py", + "static = staticmethod\n\n\nclass Shape:\n @static\n def area(width, height):\n return width * height\n", + ) + .build(); + let parameters = salsa::attach(&test.db, || { + function_signature( + &test.db, + test.program_file(test.cursor.file), + test.cursor.offset, + ) + }) + .map(|signature| { + signature + .parameters + .into_iter() + .map(|parameter| parameter.name) + .collect::>() + }); + assert_eq!( + parameters, + Some(vec!["width".to_string(), "height".to_string()]) + ); + } + + /// Without the call that unpacks into it, or the one that leaves `height` to the default it + /// loses, swapping the two parameters swaps what each call passes by position, and a call + /// that passed `height` by keyword passes both by keyword, in the order it wrote them. + #[test] + fn reordering_rewrites_each_call() { + let test = project( + &GEO.replace( + "def forward(*args, **kwargs):\n return area(*args, **kwargs)\n", + "", + ) + .replace("print(area(width=2))\n", ""), + ); + assert_snapshot!(test.change(&[ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @" + --- /sig/geo.py + def area(height, width, *, scale=1.0): + return width * height * scale + + + + + print(area(3, 2)) + print(area(width=2, height=3, scale=2.0)) + --- /sig/user.py + from sig.geo import area + + area(5, 4, scale=3) + "); + } + + /// `forward` unpacks `*args` into `area`, which cannot be rebound. + #[test] + fn a_call_that_unpacks_is_refused() { + let test = project(GEO); + assert_snapshot!(test.change(&[ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @"refused: sig/geo.py:6 unpacks arguments into the call, which cannot be told apart"); + } + + /// A parameter added with a default leaves every call valid, so none is touched, the one + /// that unpacks included. + #[test] + fn adding_a_parameter_with_a_default_changes_no_call() { + let test = project(GEO); + assert_snapshot!(test.change(&[ + kept("width", "width", PositionalOrKeyword, None), + kept("height", "height", PositionalOrKeyword, Some("1")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + added("depth", KeywordOnly, Some("1"), None), + ]), @" + --- /sig/geo.py + def area(width, height=1, *, scale=1.0, depth=1): + return width * height * scale + + + def forward(*args, **kwargs): + return area(*args, **kwargs) + + + print(area(2, 3)) + print(area(2, height=3, scale=2.0)) + print(area(width=2)) + "); + } + + /// A required parameter is passed the value given, by position while it can be. + #[test] + fn adding_a_required_parameter_passes_its_value() { + let test = project(&GEO.replace( + "def forward(*args, **kwargs):\n return area(*args, **kwargs)\n", + "", + )); + assert_snapshot!(test.change(&[ + kept("width", "width", PositionalOrKeyword, None), + added("depth", PositionalOrKeyword, None, Some("5")), + kept("height", "height", PositionalOrKeyword, Some("1")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @" + --- /sig/geo.py + def area(width, depth, height=1, *, scale=1.0): + return width * height * scale + + + + + print(area(2, 5, 3)) + print(area(2, 5, height=3, scale=2.0)) + print(area(width=2, depth=5)) + --- /sig/user.py + from sig.geo import area + + area(4, 5, 5, scale=3) + "); + } + + #[test] + fn a_parameter_without_a_default_after_one_with_is_refused() { + let test = project(GEO); + assert_snapshot!(test.change(&[ + kept("width", "width", PositionalOrKeyword, None), + kept("height", "height", PositionalOrKeyword, Some("1")), + added("depth", PositionalOrKeyword, None, Some("5")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @"refused: `depth` has no default but follows `height`, which has one"); + } + + #[test] + fn removing_a_parameter_the_body_reads_is_refused() { + let test = project(GEO); + assert_snapshot!(test.change(&[ + kept("width", "width", PositionalOrKeyword, None), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @"refused: the body still reads `height`, on line 2"); + } + + /// A rename renames the body's reads and the calls' keywords. + #[test] + fn renaming_renames_the_body_and_keywords() { + let test = project(&GEO.replace( + "def forward(*args, **kwargs):\n return area(*args, **kwargs)\n", + "", + )); + assert_snapshot!(test.change(&[ + kept("w", "width", PositionalOrKeyword, None), + kept("height", "height", PositionalOrKeyword, Some("1")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]), @" + --- /sig/geo.py + def area(w, height=1, *, scale=1.0): + return w * height * scale + + + + + print(area(2, 3)) + print(area(2, height=3, scale=2.0)) + print(area(w=2)) + "); + } + + /// Dropping an argument whose evaluation could have an effect would lose the effect. + #[test] + fn dropping_an_argument_with_an_effect_is_refused() { + let test = CursorTestBuilder::default() + .source( + "main.py", + "def f(a, b):\n return a\n\n\ndef g():\n return 1\n\n\nf(1, g())\n", + ) + .build(); + assert_snapshot!(test.change(&[kept("a", "a", PositionalOrKeyword, None)]), @"refused: main.py:9 passes `g()` for `b`, which would no longer be evaluated"); + let test = CursorTestBuilder::default() + .source( + "main.py", + "def f(a, b):\n return a\n\n\nf(1, 2)\n", + ) + .build(); + assert_snapshot!(test.change(&[kept("a", "a", PositionalOrKeyword, None)]), @" + --- /main.py + def f(a): + return a + + + f(1) + "); + } + + /// A method called through an instance is passed its receiver; through the class, its first + /// argument is the receiver. + #[test] + fn a_method_s_calls_keep_their_receiver() { + let test = CursorTestBuilder::default() + .source( + "main.py", + r#"class Shape: + def scaled(self, factor, offset=0): + return factor + offset + + +shape = Shape() +shape.scaled(2, 1) +Shape.scaled(shape, 2, offset=1) +"#, + ) + .build(); + assert_snapshot!(test.change(&[ + kept("offset", "offset", PositionalOrKeyword, None), + kept("factor", "factor", PositionalOrKeyword, None), + ]), @" + --- /main.py + class Shape: + def scaled(self, offset, factor): + return factor + offset + + + shape = Shape() + shape.scaled(1, 2) + Shape.scaled(shape, factor=2, offset=1) + "); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 4292b4c585..d77954c016 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -262,6 +262,17 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::PrepareChangeSignatureRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::Worker, + ) + } + requests::ChangeSignatureRequestHandler::METHOD => background_document_request_task::< + requests::ChangeSignatureRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 726268841b..ff67e8dbe2 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -16,6 +16,7 @@ mod alignment_groups; mod build_output; mod call_hierarchy_incoming_calls; mod call_hierarchy_outgoing_calls; +mod change_signature; mod check_workspace; mod code_action; mod code_action_resolve; @@ -81,6 +82,9 @@ pub(super) use alignment_groups::AlignmentGroupsRequestHandler; pub(super) use build_output::BuildOutputRequestHandler; pub(super) use call_hierarchy_incoming_calls::CallHierarchyIncomingCallsRequestHandler; pub(super) use call_hierarchy_outgoing_calls::CallHierarchyOutgoingCallsRequestHandler; +pub(super) use change_signature::{ + ChangeSignatureRequestHandler, PrepareChangeSignatureRequestHandler, +}; pub(super) use check_workspace::CheckWorkspaceRequestHandler; pub(crate) use check_workspace::SuspendedWorkspaceRequestKind; pub(super) use code_action::CodeActionRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/change_signature.rs b/crates/ty_server/src/server/api/requests/change_signature.rs new file mode 100644 index 0000000000..1e788dd8c4 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/change_signature.rs @@ -0,0 +1,255 @@ +//! `by/prepareChangeSignature` and `by/changeSignature`: a function's parameters, and changing +//! them with every call of the function in the project. see [`ty_ide::change_signature`] for how +//! calls are rewritten and what is refused + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ruff_db::files::FileRange; +use ty_ide::{NewParameter, ParameterKind, change_signature, function_signature}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use super::move_symbol::MoveSymbolAnswer; +use crate::document::{FileRangeExt, PositionExt}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +/// how a parameter is passed +#[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Kind { + /// before a `/` + PositionalOnly, + PositionalOrKeyword, + /// `*args` + Variadic, + /// after a `*` or `*args` + KeywordOnly, + /// `**kwargs` + VariadicKeyword, +} + +impl From for Kind { + fn from(kind: ParameterKind) -> Self { + match kind { + ParameterKind::PositionalOnly => Kind::PositionalOnly, + ParameterKind::PositionalOrKeyword => Kind::PositionalOrKeyword, + ParameterKind::Variadic => Kind::Variadic, + ParameterKind::KeywordOnly => Kind::KeywordOnly, + ParameterKind::VariadicKeyword => Kind::VariadicKeyword, + } + } +} + +impl From for ParameterKind { + fn from(kind: Kind) -> Self { + match kind { + Kind::PositionalOnly => ParameterKind::PositionalOnly, + Kind::PositionalOrKeyword => ParameterKind::PositionalOrKeyword, + Kind::Variadic => ParameterKind::Variadic, + Kind::KeywordOnly => ParameterKind::KeywordOnly, + Kind::VariadicKeyword => ParameterKind::VariadicKeyword, + } + } +} + +pub(crate) enum PrepareChangeSignatureRequest {} + +impl Request for PrepareChangeSignatureRequest { + type Params = PrepareChangeSignatureParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareChangeSignature"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// a position in a function's name or parameters where it is defined, or in a name that refers +/// to it +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct PrepareChangeSignatureParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// the function and its parameters +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Signature { + name: String, + /// the file it is defined in, which need not be the document asked about + uri: Uri, + /// its name, where it is defined + range: Range, + /// its parameters, without the `self` or `cls` of a method, which stays as it is + parameters: Vec, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SignatureParameter { + name: String, + kind: Kind, + /// as the source writes it + default: Option, + /// as the source writes it + annotation: Option, +} + +pub(crate) struct PrepareChangeSignatureRequestHandler; + +impl RequestHandler for PrepareChangeSignatureRequestHandler { + type RequestType = PrepareChangeSignatureRequest; +} + +impl BackgroundDocumentRequestHandler for PrepareChangeSignatureRequestHandler { + fn document_uri(params: &PrepareChangeSignatureParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` where no function of the project is named, when language services are off and for + /// a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: PrepareChangeSignatureParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + let Some(signature) = function_signature(db, db.program_file(file), offset) else { + return Ok(None); + }; + let Some(location) = FileRange::new(signature.file, signature.name_range) + .to_lsp_range(db, snapshot.encoding()) + .and_then(|range| range.to_location()) + else { + return Ok(None); + }; + Ok(Some(Signature { + name: signature.name, + uri: location.uri, + range: location.range, + parameters: signature + .parameters + .into_iter() + .map(|parameter| SignatureParameter { + name: parameter.name, + kind: parameter.kind.into(), + default: parameter.default, + annotation: parameter.annotation, + }) + .collect(), + })) + } +} + +impl RetriableRequestHandler for PrepareChangeSignatureRequestHandler {} + +pub(crate) enum ChangeSignatureRequest {} + +impl Request for ChangeSignatureRequest { + type Params = ChangeSignatureParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/changeSignature"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the function, as for `by/prepareChangeSignature`, and the whole of its new parameter list, in +/// order, without a method's `self` or `cls` +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ChangeSignatureParams { + text_document: TextDocumentIdentifier, + position: Position, + parameters: Vec, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ChangedParameter { + name: String, + /// the parameter it was; absent for a new one + #[serde(default)] + old_name: Option, + kind: Kind, + /// its default, written as source; absent for none + #[serde(default)] + default: Option, + /// for a new parameter, what each call passes it, written as source; absent leaves it to its + /// default + #[serde(default)] + value: Option, +} + +pub(crate) struct ChangeSignatureRequestHandler; + +impl RequestHandler for ChangeSignatureRequestHandler { + type RequestType = ChangeSignatureRequest; +} + +impl BackgroundDocumentRequestHandler for ChangeSignatureRequestHandler { + fn document_uri(params: &ChangeSignatureParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: ChangeSignatureParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + let parameters: Vec = params + .parameters + .into_iter() + .map(|parameter| NewParameter { + name: parameter.name, + old_name: parameter.old_name, + kind: parameter.kind.into(), + default: parameter.default, + value: parameter.value, + }) + .collect(); + Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + change_signature(db, db.program_file(file), offset, ¶meters), + ))) + } +} + +impl RetriableRequestHandler for ChangeSignatureRequestHandler { + /// the user is waiting on the change they asked for + const RETRY_ON_CANCELLATION: bool = true; +} + +/// the file and offset of a position in the snapshot's document, `None` where the request is not +/// answered +fn position( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + position: Position, +) -> Option<(ruff_db::files::File, ruff_text_size::TextSize)> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return None; + } + let file = snapshot.document().to_notebook_or_file(db)?; + if !file.source_type(db).is_py_file() { + return None; + } + let offset = position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding())?; + Some((file, offset)) +} diff --git a/crates/ty_server/tests/e2e/change_signature.rs b/crates/ty_server/tests/e2e/change_signature.rs new file mode 100644 index 0000000000..05df270505 --- /dev/null +++ b/crates/ty_server/tests/e2e/change_signature.rs @@ -0,0 +1,120 @@ +//! `by/prepareChangeSignature` and `by/changeSignature` over the wire +//! +//! how calls are rewritten, and when a change is refused, is `ty_ide`'s and its unit tests cover +//! it. what these cover is the contract a client reads: a function's parameters, and the +//! workspace edit or refusal a change answers + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use serde_json::json; + +use crate::{TestServer, TestServerBuilder}; + +/// `by/prepareChangeSignature`, in json throughout +enum PrepareChangeSignature {} + +impl Request for PrepareChangeSignature { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/prepareChangeSignature"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/changeSignature`, in json throughout +enum ChangeSignature {} + +impl Request for ChangeSignature { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/changeSignature"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const MAIN: &str = "\ +def area(width: float, height: float = 1) -> float: + return width * height + + +print(area(2, 3)) +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.py"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.py"), MAIN, 1); + Ok(server) +} + +#[test] +fn a_call_names_the_function_and_its_parameters() -> Result<()> { + let mut server = server()?; + let uri = server.file_uri(SystemPath::new("src/main.py")); + let answer = server.send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 7 }, + })); + assert_eq!( + answer, + Some(json!({ + "name": "area", + "uri": uri, + "range": { "start": { "line": 0, "character": 4 }, "end": { "line": 0, "character": 8 } }, + "parameters": [ + { "name": "width", "kind": "positionalOrKeyword", "default": null, "annotation": "float" }, + { "name": "height", "kind": "positionalOrKeyword", "default": "1", "annotation": "float" }, + ], + })) + ); + Ok(()) +} + +#[test] +fn a_change_answers_the_edit_to_the_definition_and_every_call() -> Result<()> { + let mut server = server()?; + let uri = server.file_uri(SystemPath::new("src/main.py")); + let answer = server + .send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 0, "character": 5 }, + "parameters": [ + { "name": "height", "oldName": "height", "kind": "positionalOrKeyword" }, + { "name": "width", "oldName": "width", "kind": "positionalOrKeyword" }, + { "name": "scale", "kind": "keywordOnly", "default": "1.0" }, + ], + })) + .expect("an answer"); + assert_eq!( + answer, + json!({ + "edit": { "changes": { uri.as_str(): [ + { + "range": { "start": { "line": 0, "character": 8 }, "end": { "line": 0, "character": 41 } }, + "newText": "(height: float, width: float, *, scale=1.0)", + }, + { + "range": { "start": { "line": 4, "character": 10 }, "end": { "line": 4, "character": 16 } }, + "newText": "(3, 2)", + }, + ] } }, + "warnings": [], + }) + ); + + let refused = server.send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 0, "character": 5 }, + "parameters": [{ "name": "width", "oldName": "width", "kind": "positionalOrKeyword" }], + })); + assert_eq!( + refused, + Some(json!({ + "refused": "the body still reads `height`, on line 2", + "warnings": [], + })) + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 1019b2a0e8..2c376d5b35 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -31,6 +31,7 @@ mod alignment_groups; mod build_output; mod build_trees; mod call_hierarchy; +mod change_signature; mod check_workspace; mod closed_documents; mod code_actions; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ac768af376..1e35912a23 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -680,6 +680,54 @@ names to move, and `target`, a superclass's `qualifiedName`; `by/pushMembersDown takes the position and `members`. both answer as `by/moveSymbol` does, `{edit, warnings}` or `{refused, warnings}` +### changing a signature + +`by/changeSignature` gives a function a new parameter list — parameters added, +removed, reordered, renamed, given or stripped of defaults, moved between kinds — +and rewrites every call of it in the project to pass them. each call is bound the +way python binds it, its positional arguments to the positional parameters and its +keywords by name, and written again for the new list: by position while the +parameters still line up, by keyword after that, a keyword staying a keyword, and +keywords in the order the call wrote them. a new parameter is passed the `value` +given, or left to its default. a renamed parameter is renamed where the body reads +it and where calls pass it by keyword, and a method's calls through an instance and +through its class keep their receiver. calls through a name an import gives the +function, `from m import area as a`, are rewritten too. where the new order would +evaluate an argument before one written before it and either could change what +the other does, the call passes them by keyword in the order it wrote them + +```py +def area(width, height=1, *, scale=1.0): ... # (height, width, *, scale=1.0) + +area(2, 3) # area(3, 2) +area(2, height=3, scale=2.0) # area(width=2, height=3, scale=2.0) +``` + +it is refused, with the file and line, where a call cannot be rewritten to mean what +it meant: a call that unpacks `*args` or `**kwargs` into the function, unless every +existing call stays valid as it is (a parameter added with a default after the others +leaves them all alone; one added before `**kwargs` does not, since it could take a +keyword); an argument the change would drop when evaluating it could have an +effect; an argument whose order of evaluation cannot be kept because it is only +taken by position; a keyword a call passes into `**kwargs` that a parameter of the +new list would take; a call with nothing to pass for a +parameter without a default; the function used other than called, unless every call +stays valid. and where the result would not be python, or not the same program: a +removed parameter the body still reads, a parameter without a default after one with +a default, two of one name, kinds out of order, or a method another class of the +project overrides or that overrides another's, whose signatures would part + +`by/prepareChangeSignature` takes `textDocument` and `position` — in a function's +name or parameters, or in a name that refers to it — and answers `{name, uri, range, parameters}`, its parameters each `{name, kind, default, annotation}` with `kind` +one of `positionalOnly`, `positionalOrKeyword`, `variadic` (`*args`), `keywordOnly` +and `variadicKeyword` (`**kwargs`), and `default` and `annotation` as the source +writes them; a method's `self` or `cls` is not listed and stays as it is. +`by/changeSignature` takes the same position and `parameters`, the whole new list +in order, each `{name, oldName?, kind, default?, value?}`: `oldName` the parameter it +was, absent for a new one; `default` written as source, absent for none; `value` what +each call passes a new one. it answers as `by/moveSymbol` does, `{edit, warnings}` or +`{refused, warnings}` + ## unwrap and remove `by/unwrap` answers what an editor's *unwrap/remove* offers around a range, innermost From 7d3fc1ac8d980f3160b12a160b95e139ed9cb697 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:11:14 +1000 Subject: [PATCH 037/110] extract a superclass with by/extractSuperclass, a new class above the class that takes its bases and the members named and becomes its one base, so the class finds every member where it did, refusing members that read what the new class would lack and classes whose bases python treats specially --- crates/ty_ide/src/lib.rs | 4 +- crates/ty_ide/src/refactor.rs | 4 +- crates/ty_ide/src/refactor/class_members.rs | 277 +++++++++++++++++- crates/ty_server/src/server/api.rs | 5 + crates/ty_server/src/server/api/requests.rs | 3 +- .../src/server/api/requests/move_members.rs | 64 +++- crates/ty_server/tests/e2e/move_symbol.rs | 43 +++ docs/basedpython/features/editor.md | 36 +++ 8 files changed, 429 insertions(+), 7 deletions(-) diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 79cd3b24fd..de8bba65b5 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -111,8 +111,8 @@ pub use override_members::{ pub use pasted_imports::{CopiedCode, CopiedImport, PastedImports, copied_imports, pasted_imports}; pub use refactor::{ ClassMember, HierarchyClass, MemberKind, MovableKind, MovableMembers, MovableSymbol, Refactor, - RefactorKind, RefactorOffer, SymbolMove, movable_members, movable_symbol, move_symbol, - pull_members_up, push_members_down, refactor, refactors, + RefactorKind, RefactorOffer, SymbolMove, extract_superclass, movable_members, movable_symbol, + move_symbol, pull_members_up, push_members_down, refactor, refactors, }; pub use refactor::{ FunctionSignature, NewParameter, ParameterKind, SignatureParameter, change_signature, diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index b58451c06e..795e0def2b 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -47,8 +47,8 @@ pub use change_signature::{ function_signature, }; pub use class_members::{ - ClassMember, HierarchyClass, MemberKind, MovableMembers, movable_members, pull_members_up, - push_members_down, + ClassMember, HierarchyClass, MemberKind, MovableMembers, extract_superclass, movable_members, + pull_members_up, push_members_down, }; pub use move_symbol::{MovableKind, MovableSymbol, SymbolMove, movable_symbol, move_symbol}; diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index 283653b2f7..1dd84f4ae1 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -535,6 +535,208 @@ pub fn push_members_down( Ok(moved) } +/// makes a new class `name`, directly above the class around `offset` in its module, that takes +/// the class's bases and its members `names`, and makes it the class's one base. the class's +/// method resolution order is what it was with the new class inserted right after it, so every +/// member is found where it was found, and `super()` in a moved method starts where it started. +/// `Err` says why it cannot be extracted. +pub fn extract_superclass( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, + names: &[String], + name: &str, +) -> Result { + let class = + ClassSite::at(db, file, offset).ok_or_else(|| "there is no class here".to_string())?; + let def = class + .def() + .ok_or_else(|| "there is no class here".to_string())?; + if !ruff_python_stdlib::identifiers::is_identifier(name) + || ruff_python_stdlib::keyword::is_keyword(name) + { + return Err(format!("`{name}` is not a name a class can have")); + } + if def.type_params.is_some() { + return Err(format!( + "{} is generic, and its type parameters would have to be shared", + class.name + )); + } + if all_classes(&class.parsed) + .iter() + .any(|other| other.name.as_str() == name) + || module_binds(&class, db, name) + { + return Err(format!( + "{} has a `{name}` already", + class.module(db).unwrap_or_default() + )); + } + let env = ProgramEnvironment::from_file(class.file); + let bases = class_bases(db, &env, class.ty); + // a base that is a special form rather than a class — `Protocol`, `NamedTuple`, `TypedDict`, + // `Generic[T]` — or an enum makes a subclass something other than what the base itself is, + // so the class would stop being what it is once it subclassed the new class instead + let written_bases = def + .arguments + .as_ref() + .map_or(0, |arguments| arguments.args.len()); + if bases.len() != written_bases + || bases.iter().any(|base| { + instance_member_names(db, &env, *base).contains(&Name::new_static("_member_map_")) + }) + { + return Err(format!( + "{} has a base whose subclasses python treats specially, which {name} would take \ + in its place", + class.name + )); + } + // what the new class has: everything the class's bases give it, and what moves into it + let inherited: FxHashSet = bases + .into_iter() + .flat_map(|base| instance_member_names(db, &env, base)) + .collect(); + let extracted: FxHashSet<&str> = names.iter().map(String::as_str).collect(); + let source = source_text(db, class.file.file(db)); + let source = source.as_str(); + + let mut texts: Vec<(usize, MemberKind, String)> = Vec::new(); + let mut removed: Vec = Vec::new(); + for member in names { + refuse_mangled(member)?; + let statements = member_statements(def, member); + if statements.is_empty() { + return Err(format!("{} has no member `{member}`", class.name)); + } + for &index in &statements { + let statement = &def.body[index]; + if !def.decorator_list.is_empty() && member_kind(statement) == MemberKind::Attribute { + return Err(format!( + "`{member}` is an attribute of a decorated class, which its decorator may read as a field" + )); + } + for (attribute, range) in receiver_attributes(statement) { + if extracted.contains(attribute.as_str()) || inherited.contains(&attribute) { + continue; + } + refuse_mangled(&attribute)?; + let is_member = def + .body + .iter() + .any(|statement| binds(statement, &attribute)); + return Err(format!( + "`{member}` uses `{}` on line {}, which {name} would not have{}", + &source[range], + line_number(source, range.start()), + if is_member { "; extract it too" } else { "" } + )); + } + for (sibling, _) in class_scope_reads(def, statement) { + if !extracted.contains(sibling.as_str()) && !inherited.contains(&sibling) { + return Err(format!( + "`{member}` reads {}'s `{sibling}`, which {name} would not have; \ + extract it too", + class.name + )); + } + } + texts.push(( + index, + member_kind(statement), + member_text(source, &def.body, index, false), + )); + removed.push(index); + } + } + texts.sort_by_key(|(index, _, _)| *index); + + let file_id = class.file.file(db); + let line_ending = Stylist::from_tokens(class.parsed.tokens(), source) + .line_ending() + .as_str() + .to_string(); + let mut moved = SymbolMove::default(); + moved + .edits + .extend(remove_members(file_id, source, def, &removed)); + + // the class takes the new one as its only base, which takes its bases + let base_text = def + .arguments + .as_ref() + .map(|arguments| { + let text = &source[arguments.range()]; + text[1..text.len() - 1].trim().to_string() + }) + .filter(|bases| !bases.is_empty()); + let new_bases = format!("({name})"); + moved.edits.push(FileEdit { + file: file_id, + edit: match &def.arguments { + Some(arguments) => Edit::range_replacement(new_bases, arguments.range()), + None => Edit::insertion(new_bases, def.name.end()), + }, + }); + + // the new class, above the class and the comments and decorators that go with it + let class_indent = { + let start = source.line_start(def.start()); + source[TextRange::new(start, def.start())].to_string() + }; + let body_indent = format!("{class_indent} "); + let from_indent = body_indentation(source, def); + let mut text = format!( + "{class_indent}class {name}{}:{line_ending}", + base_text + .map(|bases| format!("({bases})")) + .unwrap_or_default() + ); + let mut previous: Option = None; + for (_, kind, member) in &texts { + if previous.is_some_and(|previous| { + previous != MemberKind::Attribute || *kind != MemberKind::Attribute + }) { + text.push_str(&line_ending); + } + text.push_str(&reindent(member, &from_indent, &body_indent)); + previous = Some(*kind); + } + if texts.is_empty() { + text.push_str(&body_indent); + text.push_str("pass"); + text.push_str(&line_ending); + } + let blank_lines = if class_indent.is_empty() { 2 } else { 1 }; + text.push_str(&line_ending.repeat(blank_lines)); + let mut start = source.line_start(def.start()); + while start > TextSize::default() { + let previous = source.line_start(start - TextSize::from(1)); + if !source[TextRange::new(previous, start)] + .trim_start() + .starts_with('#') + { + break; + } + start = previous; + } + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::insertion(text, start), + }); + Ok(moved) +} + +/// whether the module of `class` binds `name` at its top level +fn module_binds(class: &ClassSite<'_>, db: &dyn Db, name: &str) -> bool { + let index = ty_python_core::semantic_index(db, class.file); + let globals = index.place_table(ty_python_core::scope::FileScopeId::global()); + globals + .symbol_id(name) + .is_some_and(|symbol| globals.symbol(symbol).is_bound()) +} + /// A class of the project, parsed. struct ClassSite<'db> { file: ProgramFile<'db>, @@ -1346,7 +1548,7 @@ fn attribute_receiver_type<'db>( mod tests { use insta::assert_snapshot; - use super::{movable_members, pull_members_up, push_members_down}; + use super::{extract_superclass, movable_members, pull_members_up, push_members_down}; use crate::refactor::move_symbol::tests::rendered; use crate::tests::{CursorTest, CursorTestBuilder}; @@ -1721,4 +1923,77 @@ class Leaf(Mid): .build(); assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @"refused: Leaf would inherit Mid's `describe` rather than its own"); } + impl CursorTest { + fn extract(&self, names: &[&str], name: &str) -> String { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let moved = salsa::attach(&self.db, || { + extract_superclass( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + &names, + name, + ) + }); + rendered(&self.db, moved) + } + } + + /// the new class takes the class's bases and sits right after it in its MRO, so what the + /// moved members find is what they found + #[test] + fn extracting_a_superclass_inserts_it_between_the_class_and_its_bases() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.extract(&["plain", "LIMIT"], "Shape"), @" + --- /cls/shapes.py + class Base: + def keep(self): + return 1 + + + class Shape(Base): + LIMIT = 3 + + def plain(self): + return 2 + + + class Child(Shape): + def __init__(self, x): + self.x = x + + def area(self): + return self.x * self.LIMIT + + def helper(self): + return self.plain() + 1 + + + class Other(Base): + pass + + + class GrandChild(Child): + def area(self): + return 0 + "); + } + + #[test] + fn extracting_what_reads_an_attribute_left_behind_is_refused() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.extract(&["area"], "Shape"), @"refused: `area` uses `self.x` on line 13, which Shape would not have"); + assert_snapshot!(test.extract(&["plain"], "Base"), @"refused: cls.shapes has a `Base` already"); + } + + #[test] + fn a_class_with_a_special_base_is_refused() { + let test = CursorTestBuilder::default() + .source( + "main.py", + "from typing import NamedTuple\n\n\nclass Point(NamedTuple):\n x: int\n y: int\n", + ) + .build(); + assert_snapshot!(test.extract(&["x"], "Base"), @"refused: Point has a base whose subclasses python treats specially, which Base would take in its place"); + } } diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index d77954c016..465cb1f568 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -273,6 +273,11 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::ExtractSuperclassRequestHandler::METHOD => background_document_request_task::< + requests::ExtractSuperclassRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index ff67e8dbe2..c8edf12e2f 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -117,7 +117,8 @@ pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; pub(super) use module_name::ModuleNameHandler; pub(super) use move_members::{ - PrepareMoveMembersRequestHandler, PullMembersUpRequestHandler, PushMembersDownRequestHandler, + ExtractSuperclassRequestHandler, PrepareMoveMembersRequestHandler, PullMembersUpRequestHandler, + PushMembersDownRequestHandler, }; pub(super) use move_symbol::{MoveSymbolRequestHandler, PrepareMoveSymbolRequestHandler}; pub(super) use on_type_formatting::OnTypeFormattingRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/move_members.rs b/crates/ty_server/src/server/api/requests/move_members.rs index ecb0c66db5..873cd77e64 100644 --- a/crates/ty_server/src/server/api/requests/move_members.rs +++ b/crates/ty_server/src/server/api/requests/move_members.rs @@ -8,7 +8,10 @@ use lsp_types::{ LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, }; use ruff_db::files::FileRange; -use ty_ide::{HierarchyClass, MemberKind, movable_members, pull_members_up, push_members_down}; +use ty_ide::{ + HierarchyClass, MemberKind, extract_superclass, movable_members, pull_members_up, + push_members_down, +}; use ty_project::{ProjectDatabase, SemanticDb as _}; use super::move_symbol::MoveSymbolAnswer; @@ -287,3 +290,62 @@ impl RetriableRequestHandler for PushMembersDownRequestHandler { /// the user is waiting on the move they asked for const RETRY_ON_CANCELLATION: bool = true; } + +pub(crate) enum ExtractSuperclassRequest {} + +impl Request for ExtractSuperclassRequest { + type Params = ExtractSuperclassParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/extractSuperclass"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// the class, the names of the members the new class takes, and the new class's name +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ExtractSuperclassParams { + text_document: TextDocumentIdentifier, + position: Position, + members: Vec, + name: String, +} + +pub(crate) struct ExtractSuperclassRequestHandler; + +impl RequestHandler for ExtractSuperclassRequestHandler { + type RequestType = ExtractSuperclassRequest; +} + +impl BackgroundDocumentRequestHandler for ExtractSuperclassRequestHandler { + fn document_uri(params: &ExtractSuperclassParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off and for a notebook or a template + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: ExtractSuperclassParams, + ) -> crate::server::Result> { + let Some((file, offset)) = position(db, snapshot, params.position) else { + return Ok(None); + }; + Ok(Some(MoveSymbolAnswer::of( + db, + snapshot, + extract_superclass( + db, + db.program_file(file), + offset, + ¶ms.members, + ¶ms.name, + ), + ))) + } +} + +impl RetriableRequestHandler for ExtractSuperclassRequestHandler { + /// the user is waiting on the refactoring they asked for + const RETRY_ON_CANCELLATION: bool = true; +} diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index 26632175df..3644fe112a 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -326,3 +326,46 @@ fn a_pull_up_answers_its_edit_and_a_push_down_with_nowhere_to_go_is_refused() -> ); Ok(()) } + +/// `by/extractSuperclass`, in json throughout +enum ExtractSuperclass {} + +impl Request for ExtractSuperclass { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/extractSuperclass"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[test] +fn an_extracted_superclass_takes_the_class_s_bases_and_becomes_its_base() -> Result<()> { + let mut server = shapes_server()?; + let uri = server.file_uri(SystemPath::new("src/shapes.py")); + let answer = server + .send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 7 }, + "members": ["SIDES"], + "name": "Polygon", + })) + .expect("an answer"); + assert_eq!( + answer["edit"]["changes"][uri.as_str()], + json!([ + { + "range": { "start": { "line": 5, "character": 0 }, "end": { "line": 7, "character": 0 } }, + "newText": "", + }, + { + "range": { "start": { "line": 4, "character": 12 }, "end": { "line": 4, "character": 19 } }, + "newText": "(Polygon)", + }, + { + "range": { "start": { "line": 4, "character": 0 }, "end": { "line": 4, "character": 0 } }, + "newText": "class Polygon(Shape):\n SIDES = 4\n\n\n", + }, + ]), + "{answer:#}" + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 1e35912a23..42a6dd3eab 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -680,6 +680,42 @@ names to move, and `target`, a superclass's `qualifiedName`; `by/pushMembersDown takes the position and `members`. both answer as `by/moveSymbol` does, `{edit, warnings}` or `{refused, warnings}` +### extracting a superclass + +`by/extractSuperclass` makes a new class, directly above a class in its module, +that takes the class's bases and the members named, and makes the new class the +class's one base: + +```py +class Square(Shape): + SIDES = 4 + + def area(self): ... +``` + +with `SIDES` extracted into `Polygon`, becomes + +```py +class Polygon(Shape): + SIDES = 4 + + +class Square(Polygon): + def area(self): ... +``` + +the new class sits right after the class in its method resolution order, so a moved +member is found where it was found and `super()` in a moved method starts where it +started. it is refused, with the reason, for a member that reads an attribute of +`self` the new class would not have (one the class's own methods assign, or a member +left behind), a name-mangled member, a method that reads `__class__`, which would +be the new class, an attribute of a decorated class (a data +class's field, say), a generic class, a class whose base python treats specially — +`Protocol`, `NamedTuple`, `TypedDict`, `Generic`, an enum — and a name the module +binds already. it takes `textDocument`, `position` (anywhere in the class), +`members`, the names to move, as `by/prepareMoveMembers` lists them, and `name`, +and answers as `by/moveSymbol` does + ### changing a signature `by/changeSignature` gives a function a new parameter list — parameters added, From 7bd76624ed8ea30268b7a2cf67e5aafe0e0e8226 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:21:09 +1000 Subject: [PATCH 038/110] offer inverting an if with an else, De Morgan's law, converting a string's quotes and turning a loop that fills an empty list, set or dict into a comprehension as refactor.rewrite code actions, flipping a comparison only where its operands make the negation mean the same and refusing a comprehension that would unbind the loop variable or reorder an effect --- crates/ty_ide/src/refactor.rs | 21 + crates/ty_ide/src/refactor/intentions.rs | 947 ++++++++++++++++++ .../e2e__initialize__initialization.snap | 6 +- ...ialize__initialization_with_workspace.snap | 6 +- docs/basedpython/features/editor.md | 56 +- 5 files changed, 1021 insertions(+), 15 deletions(-) create mode 100644 crates/ty_ide/src/refactor/intentions.rs diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 795e0def2b..af90ef4860 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -24,6 +24,7 @@ mod extract_variable; mod flow; mod hazards; mod inline_variable; +mod intentions; mod move_symbol; mod names; mod return_annotation; @@ -66,6 +67,10 @@ pub enum RefactorKind { ConvertFromDataClass, SplitList, JoinList, + InvertIf, + DeMorgan, + FlipQuotes, + LoopToComprehension, } impl RefactorKind { @@ -81,6 +86,10 @@ impl RefactorKind { RefactorKind::ConvertFromDataClass, RefactorKind::SplitList, RefactorKind::JoinList, + RefactorKind::InvertIf, + RefactorKind::DeMorgan, + RefactorKind::FlipQuotes, + RefactorKind::LoopToComprehension, ]; /// The LSP code action kind the refactoring is offered as. @@ -97,6 +106,10 @@ impl RefactorKind { RefactorKind::ConvertFromDataClass => "refactor.rewrite.fromDataClass", RefactorKind::SplitList => "refactor.rewrite.splitList", RefactorKind::JoinList => "refactor.rewrite.joinList", + RefactorKind::InvertIf => "refactor.rewrite.invertIf", + RefactorKind::DeMorgan => "refactor.rewrite.deMorgan", + RefactorKind::FlipQuotes => "refactor.rewrite.flipQuotes", + RefactorKind::LoopToComprehension => "refactor.rewrite.loopToComprehension", } } @@ -115,6 +128,10 @@ impl RefactorKind { RefactorKind::ConvertFromDataClass => "convert-from-data-class", RefactorKind::SplitList => "split-list", RefactorKind::JoinList => "join-list", + RefactorKind::InvertIf => "invert-if", + RefactorKind::DeMorgan => "de-morgan", + RefactorKind::FlipQuotes => "flip-quotes", + RefactorKind::LoopToComprehension => "loop-to-comprehension", } } @@ -241,6 +258,10 @@ impl<'db> RefactorContext<'db> { } RefactorKind::SplitList => split_join::plan(self, range, split_join::Direction::Split), RefactorKind::JoinList => split_join::plan(self, range, split_join::Direction::Join), + RefactorKind::InvertIf => intentions::invert_if(self, range), + RefactorKind::DeMorgan => intentions::de_morgan(self, range), + RefactorKind::FlipQuotes => intentions::flip_quotes(self, range), + RefactorKind::LoopToComprehension => intentions::loop_to_comprehension(self, range), } } } diff --git a/crates/ty_ide/src/refactor/intentions.rs b/crates/ty_ide/src/refactor/intentions.rs new file mode 100644 index 0000000000..a22853fcb1 --- /dev/null +++ b/crates/ty_ide/src/refactor/intentions.rs @@ -0,0 +1,947 @@ +//! Small rewrites of one construct that leave what it means as it was: inverting an `if`, +//! De Morgan's law, a string's quotes, and a loop that builds a collection as a comprehension. +//! +//! Each is offered only where the checker can show the rewrite means the same. A comparison is +//! negated by flipping its operator only where the operands are python's own integers, strings or +//! bytes — `not x < 1` is not `x >= 1` for a float that is `nan` — and is written `not (...)` +//! otherwise. De Morgan's law changes what an `and` or an `or` evaluates to, which is one of its +//! operands, into a `bool`, so it is offered where only the truth of the result is read. A loop +//! becomes a comprehension only where nothing could tell the difference: the loop's variable is +//! not read after it, which a comprehension does not leave bound, and the collection is not read +//! while it is built. + +use ruff_diagnostics::Edit; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_ast::{self as ast, AnyNodeRef, BoolOp, CmpOp, Expr, Stmt, StringFlags, UnaryOp}; +use ruff_source_file::LineRanges; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use ty_python_semantic::types::ide_support::{BuiltinScalar, builtin_scalar_kinds}; + +use ty_python_core::semantic_index; + +use super::evaluation::is_effect_free; +use super::flow::{reaching_definitions, resolving_scope}; +use super::{Plan, RefactorContext, Refusal}; + +/// Invert an `if` with an `else`: negate its condition and swap its two bodies. +pub(super) fn invert_if(context: &RefactorContext<'_>, range: TextRange) -> Result { + let source = context.source(); + let covering = covering_node(context.parsed.syntax().into(), range); + let Some(statement) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtIf(statement) => Some(statement), + _ => None, + }) else { + return Err(Refusal::NotApplicable); + }; + // asked about on the header: the `if` and its condition + if range.start() > statement.test.end() { + return Err(Refusal::NotApplicable); + } + let title = "Invert `if` condition"; + let [clause] = statement.elif_else_clauses.as_slice() else { + return Err(Refusal::refused( + title, + if statement.elif_else_clauses.is_empty() { + "it has no `else` to swap its body with" + } else { + "it has an `elif`, which would have to be nested to be inverted" + }, + )); + }; + if clause.test.is_some() { + return Err(Refusal::refused( + title, + "it has an `elif`, which would have to be nested to be inverted", + )); + } + if statement.pattern.is_some() || clause.pattern.is_some() { + return Err(Refusal::NotApplicable); + } + let (Some(first), Some(last)) = (statement.body.first(), statement.body.last()) else { + return Err(Refusal::NotApplicable); + }; + let (Some(else_first), Some(else_last)) = (clause.body.first(), clause.body.last()) else { + return Err(Refusal::NotApplicable); + }; + let header_end = source.full_line_end(statement.test.end()); + let else_header_end = source.full_line_end(clause.start()); + if first.start() < header_end || else_first.start() < else_header_end { + return Err(Refusal::refused( + title, + "a body is written on the line of its header", + )); + } + let body = TextRange::new(header_end, source.full_line_end(last.end())); + let else_body = TextRange::new(else_header_end, source.full_line_end(else_last.end())); + let condition = negated(context, &statement.test).text; + Ok(Plan { + title: title.to_string(), + edits: vec![ + Edit::range_replacement(condition, statement.test.range()), + Edit::range_replacement(with_line_end(&source[else_body]), body), + Edit::range_replacement(with_line_end(&source[body]), else_body), + ], + }) +} + +fn with_line_end(text: &str) -> String { + if text.ends_with(['\n', '\r']) { + text.to_string() + } else { + format!("{text}\n") + } +} + +/// Rewrite the innermost `and` or `or` around the range by De Morgan's law: `a and b` as +/// `not (not a or not b)`, and `not (a or b)` as `not a and not b`. +pub(super) fn de_morgan(context: &RefactorContext<'_>, range: TextRange) -> Result { + let covering = covering_node(context.parsed.syntax().into(), range); + let ancestors: Vec = covering.ancestors().collect(); + let Some(position) = ancestors + .iter() + .position(|node| matches!(node, AnyNodeRef::ExprBoolOp(_))) + else { + return Err(Refusal::NotApplicable); + }; + let AnyNodeRef::ExprBoolOp(bool_op) = ancestors[position] else { + return Err(Refusal::NotApplicable); + }; + let title = "Apply De Morgan's law".to_string(); + let joined = match bool_op.op { + BoolOp::And => "or", + BoolOp::Or => "and", + }; + + // `not (a or b)` becomes `not a and not b`, which is a `bool` either way + if let Some(AnyNodeRef::ExprUnaryOp(not)) = ancestors.get(position + 1) + && not.op == UnaryOp::Not + { + let text = join(context, bool_op, joined); + return Ok(Plan { + title, + edits: vec![Edit::range_replacement(text, not.range())], + }); + } + + // `a and b` evaluates to `a` or `b`, which `not (...)` makes a `bool`: only where the + // truth of it is all that is read, or where it is a `bool` already + let is_bool = builtin_scalar_kinds(&context.model, &Expr::BoolOp(bool_op.clone())) + .is_some_and(|kinds| kinds.iter().all(|kind| *kind == BuiltinScalar::Bool)); + if !is_bool && !only_truth_is_read(&ancestors[position..]) { + return Err(Refusal::refused( + title, + "its value is used, and the rewrite would make it a `bool`", + )); + } + let inner = join(context, bool_op, joined); + Ok(Plan { + title, + edits: vec![Edit::range_replacement( + format!("not ({inner})"), + bool_op.range(), + )], + }) +} + +/// The operands of `bool_op`, each negated, joined by `joined`. +fn join(context: &RefactorContext<'_>, bool_op: &ast::ExprBoolOp, joined: &str) -> String { + bool_op + .values + .iter() + .map(|value| { + let negated = negated(context, value); + // an `or` among the operands of an `and` keeps its own parentheses + if negated.is_or && joined == "and" { + format!("({})", negated.text) + } else { + negated.text + } + }) + .collect::>() + .join(&format!(" {joined} ")) +} + +/// Whether what `ancestors[0]` evaluates to is only read for its truth: it is the condition of +/// an `if`, a `while`, an `assert`, a conditional expression or a comprehension's `if`, the +/// operand of `not`, or an operand of an `and` or an `or` that is itself only read for its truth. +fn only_truth_is_read(ancestors: &[AnyNodeRef]) -> bool { + let Some((node, rest)) = ancestors.split_first() else { + return false; + }; + let Some(parent) = rest.first() else { + return false; + }; + let range = node.range(); + match parent { + AnyNodeRef::StmtIf(statement) => statement.test.range() == range, + AnyNodeRef::ElifElseClause(clause) => clause + .test + .as_ref() + .is_some_and(|test| test.range() == range), + AnyNodeRef::StmtWhile(statement) => statement.test.range() == range, + AnyNodeRef::StmtAssert(statement) => statement.test.range() == range, + AnyNodeRef::ExprIf(expression) => expression.test.range() == range, + AnyNodeRef::Comprehension(comprehension) => { + comprehension.ifs.iter().any(|test| test.range() == range) + } + AnyNodeRef::ExprUnaryOp(unary) => unary.op == UnaryOp::Not, + AnyNodeRef::ExprBoolOp(_) => only_truth_is_read(rest), + _ => false, + } +} + +/// An expression negated, and whether the negation is an `or` at its top. +struct Negated { + text: String, + is_or: bool, +} + +/// `expr` negated for its truth: `not` taken off, a comparison's operator flipped where that +/// means the same, an `and` and an `or` swapped by De Morgan's law, and `not (...)` otherwise. +fn negated(context: &RefactorContext<'_>, expr: &Expr) -> Negated { + let source = context.source(); + let text = |expr: &Expr| source[expr.range()].to_string(); + match expr { + Expr::UnaryOp(unary) if unary.op == UnaryOp::Not => Negated { + is_or: matches!(&*unary.operand, Expr::BoolOp(inner) if inner.op == BoolOp::Or), + text: if unary.operand.is_named_expr() { + format!("({})", text(&unary.operand)) + } else { + text(&unary.operand) + }, + }, + Expr::Compare(compare) + if let [left, right] = &*compare.operands + && let Some(flipped) = flipped(context, compare) => + { + Negated { + text: format!("{} {} {}", text(left), flipped.as_str(), text(right)), + is_or: false, + } + } + Expr::BoolOp(bool_op) => { + let joined = match bool_op.op { + BoolOp::And => "or", + BoolOp::Or => "and", + }; + Negated { + text: join(context, bool_op, joined), + is_or: joined == "or", + } + } + Expr::Name(_) | Expr::Attribute(_) | Expr::Call(_) | Expr::Subscript(_) => Negated { + text: format!("not {}", text(expr)), + is_or: false, + }, + _ => Negated { + text: format!("not ({})", text(expr)), + is_or: false, + }, + } +} + +/// The operator that negates `compare`'s, where swapping it means the same. +fn flipped(context: &RefactorContext<'_>, compare: &ast::ExprCompare) -> Option { + const INTEGERS: &[BuiltinScalar] = &[BuiltinScalar::Bool, BuiltinScalar::Int]; + const NUMBERS: &[BuiltinScalar] = &[ + BuiltinScalar::Bool, + BuiltinScalar::Int, + BuiltinScalar::Float, + BuiltinScalar::Complex, + ]; + let ([op], [left, right]) = (&*compare.ops, &*compare.operands) else { + return None; + }; + let op = *op; + match op { + // python defines these as the negations of each other + CmpOp::Is => return Some(CmpOp::IsNot), + CmpOp::IsNot => return Some(CmpOp::Is), + CmpOp::In => return Some(CmpOp::NotIn), + CmpOp::NotIn => return Some(CmpOp::In), + _ => {} + } + let left = builtin_scalar_kinds(&context.model, left)?; + let right = builtin_scalar_kinds(&context.model, right)?; + let within = |kinds: &[BuiltinScalar], allowed: &[BuiltinScalar]| { + kinds.iter().all(|kind| allowed.contains(kind)) + }; + let same_family = |allowed: &[BuiltinScalar]| within(&left, allowed) && within(&right, allowed); + let totally_ordered = same_family(INTEGERS) + || same_family(&[BuiltinScalar::Str]) + || same_family(&[BuiltinScalar::Bytes]); + match op { + // `==` and `!=` of python's own scalars are each other's negation, `nan` included + CmpOp::Eq | CmpOp::NotEq if totally_ordered || same_family(NUMBERS) => { + Some(if op == CmpOp::Eq { + CmpOp::NotEq + } else { + CmpOp::Eq + }) + } + CmpOp::Lt if totally_ordered => Some(CmpOp::GtE), + CmpOp::LtE if totally_ordered => Some(CmpOp::Gt), + CmpOp::Gt if totally_ordered => Some(CmpOp::LtE), + CmpOp::GtE if totally_ordered => Some(CmpOp::Lt), + _ => None, + } +} + +/// Write a string literal in the other quotes: `'it'` as `"it"`, escaping what the new quote +/// would end and unescaping the old one. +pub(super) fn flip_quotes( + context: &RefactorContext<'_>, + range: TextRange, +) -> Result { + let source = context.source(); + let covering = covering_node(context.parsed.syntax().into(), range); + let Some((literal_range, flags)) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StringLiteral(literal) => { + Some((literal.range(), literal.flags.as_any_string_flags())) + } + AnyNodeRef::BytesLiteral(literal) => { + Some((literal.range(), literal.flags.as_any_string_flags())) + } + _ => None, + }) else { + return Err(Refusal::NotApplicable); + }; + if flags.is_triple_quoted() { + return Err(Refusal::NotApplicable); + } + let old = flags.quote_str(); + let (new, title) = if old == "'" { + ("\"", "Convert to a double-quoted string") + } else { + ("'", "Convert to a single-quoted string") + }; + let text = &source[literal_range]; + let prefix_len = flags.prefix().text_len().to_usize(); + let body = &text[prefix_len + 1..text.len() - 1]; + let raw = flags.prefix().is_raw(); + let mut converted = String::with_capacity(body.len() + 2); + let mut characters = body.chars(); + while let Some(character) = characters.next() { + match character { + '\\' => match characters.next() { + // the old quote needs no escape any more + Some(next) if !raw && next.to_string() == old => converted.push(next), + Some(next) => { + converted.push('\\'); + converted.push(next); + } + None => converted.push('\\'), + }, + character if character.to_string() == new => { + if raw { + return Err(Refusal::refused( + title, + format!("a raw string cannot escape the {new} it holds"), + )); + } + converted.push('\\'); + converted.push(character); + } + character => converted.push(character), + } + } + Ok(Plan { + title: title.to_string(), + edits: vec![Edit::range_replacement( + format!("{}{new}{converted}{new}", &text[..prefix_len]), + literal_range, + )], + }) +} + +/// Rewrite a loop that only adds to a collection made empty right above it as a comprehension: +/// +/// ```py +/// result = [] result = [item * 2 for item in items if item > 0] +/// for item in items: +/// if item > 0: +/// result.append(item * 2) +/// ``` +/// +/// for a list's `append`, a set's `add` and a dict's item assignment. +pub(super) fn loop_to_comprehension( + context: &RefactorContext<'_>, + range: TextRange, +) -> Result { + let source = context.source(); + let covering = covering_node(context.parsed.syntax().into(), range); + let Some(for_loop) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::StmtFor(statement) => Some(statement), + _ => None, + }) else { + return Err(Refusal::NotApplicable); + }; + // asked about on the header + if range.start() > for_loop.iter.end() || for_loop.is_async || for_loop.pattern.is_some() { + return Err(Refusal::NotApplicable); + } + let Some((suite, index)) = find_suite(context.parsed.suite(), for_loop) else { + return Err(Refusal::NotApplicable); + }; + let Some(Stmt::Assign(initial)) = index + .checked_sub(1) + .and_then(|previous| suite.get(previous)) + else { + return Err(Refusal::NotApplicable); + }; + let [Expr::Name(collection)] = initial.targets.as_slice() else { + return Err(Refusal::NotApplicable); + }; + let name = collection.id.as_str(); + + // the body: one addition, under at most one `if` + let (condition, addition) = match for_loop.body.as_slice() { + [Stmt::If(statement)] if statement.elif_else_clauses.is_empty() => { + match statement.body.as_slice() { + [addition] => (Some(&*statement.test), addition), + _ => return Err(Refusal::NotApplicable), + } + } + [addition] => (None, addition), + _ => return Err(Refusal::NotApplicable), + }; + let (open, close, element) = match (&*initial.value, addition) { + (Expr::List(list), Stmt::Expr(expression)) if list.elts.is_empty() => { + let element = added(&expression.value, name, "append").ok_or(Refusal::NotApplicable)?; + ("[", "]", source[element.range()].to_string()) + } + (Expr::Call(call), Stmt::Expr(expression)) + if matches!(&*call.func, Expr::Name(func) if func.id.as_str() == "set") + && call.arguments.is_empty() => + { + let element = added(&expression.value, name, "add").ok_or(Refusal::NotApplicable)?; + ("{", "}", source[element.range()].to_string()) + } + (Expr::Dict(dict), Stmt::Assign(assign)) if dict.is_empty() => { + let [Expr::Subscript(target)] = assign.targets.as_slice() else { + return Err(Refusal::NotApplicable); + }; + if !matches!(&*target.value, Expr::Name(value) if value.id.as_str() == name) { + return Err(Refusal::NotApplicable); + } + // the loop evaluates the value before the key, and a comprehension the key first + if !is_effect_free(&context.model, &target.slice) + && !is_effect_free(&context.model, &assign.value) + { + return Err(Refusal::refused( + "Convert loop to dict comprehension", + "a comprehension evaluates the key before the value, which the loop evaluates \ + after it", + )); + } + ( + "{", + "}", + format!( + "{}: {}", + &source[target.slice.range()], + &source[assign.value.range()] + ), + ) + } + _ => return Err(Refusal::NotApplicable), + }; + let kind = match open { + "[" => "list", + _ if element.contains(": ") && matches!(&*initial.value, Expr::Dict(_)) => "dict", + _ => "set", + }; + let title = format!("Convert loop to {kind} comprehension"); + + if !for_loop.orelse.is_empty() { + return Err(Refusal::refused( + title, + "its `else` runs after the loop, which a comprehension has no place for", + )); + } + // `set` is python's own unless the program binds the name + if let Expr::Call(call) = &*initial.value + && let Expr::Name(func) = &*call.func + { + let index = semantic_index(context.db, context.file); + let shadowed = index + .try_expression_scope_id(&ast::ExprRef::Name(func)) + .is_none_or(|scope| resolving_scope(index, scope, func.id.as_str()).is_some()); + if shadowed { + return Err(Refusal::refused(title, "`set` is not python's own here")); + } + } + // read while it is built, it would be read before it is bound + let mut collection_reads = Reads::new(name); + collection_reads.visit_expr(&for_loop.iter); + collection_reads.visit_body(&for_loop.body); + if collection_reads.count > 1 { + return Err(Refusal::refused( + title, + format!("the loop reads `{name}` while it is built"), + )); + } + // a comprehension does not leave its variable bound after it + let mut bound = Vec::new(); + names_in(&for_loop.target, &mut bound); + let after = TextRange::new(for_loop.end(), scope_end(context, for_loop)); + for variable in &bound { + if reads_binding_after(context, variable, after, for_loop.target.range()) { + return Err(Refusal::refused( + title, + format!( + "`{variable}` is read after the loop, which a comprehension leaves unbound" + ), + )); + } + } + // a class body's names are not visible inside a comprehension + if in_class_body(context, for_loop) { + return Err(Refusal::refused( + title, + "a comprehension in a class body cannot see the class's names", + )); + } + + let mut comprehension = format!( + "{open}{element} for {} in {}", + &source[for_loop.target.range()], + &source[for_loop.iter.range()] + ); + if let Some(condition) = condition { + comprehension.push_str(" if "); + comprehension.push_str(&source[condition.range()]); + } + comprehension.push_str(close); + Ok(Plan { + title, + edits: vec![ + Edit::range_replacement(comprehension, initial.value.range()), + Edit::range_deletion(TextRange::new( + source.full_line_end(initial.end()), + source.full_line_end(for_loop.end()), + )), + ], + }) +} + +/// The element `name.method(element)` adds. +fn added<'a>(expr: &'a Expr, name: &str, method: &str) -> Option<&'a Expr> { + let Expr::Call(call) = expr else { + return None; + }; + let Expr::Attribute(attribute) = &*call.func else { + return None; + }; + if attribute.attr.as_str() != method + || !matches!(&*attribute.value, Expr::Name(value) if value.id.as_str() == name) + { + return None; + } + match (&*call.arguments.args, call.arguments.keywords.is_empty()) { + ([element], true) if !element.is_starred_expr() => Some(element), + _ => None, + } +} + +/// The suite `for_loop` is written in, and its index there. +fn find_suite<'a>(body: &'a [Stmt], for_loop: &ast::StmtFor) -> Option<(&'a [Stmt], usize)> { + for (index, statement) in body.iter().enumerate() { + if statement.range() == for_loop.range() { + return Some((body, index)); + } + if statement.range().contains_range(for_loop.range()) { + for nested in super::text::nested_suites(statement) { + if let Some(found) = find_suite(nested, for_loop) { + return Some(found); + } + } + } + } + None +} + +/// Counts the loads of a name. +struct Reads<'a> { + name: &'a str, + count: usize, +} + +impl<'a> Reads<'a> { + fn new(name: &'a str) -> Self { + Self { name, count: 0 } + } +} + +impl<'b> SourceOrderVisitor<'b> for Reads<'_> { + fn visit_expr(&mut self, expr: &'b Expr) { + if let Expr::Name(name) = expr + && name.id.as_str() == self.name + { + self.count += 1; + } + walk_expr(self, expr); + } +} + +fn names_in(target: &Expr, names: &mut Vec) { + match target { + Expr::Name(name) => names.push(name.id.to_string()), + Expr::Tuple(tuple) => tuple + .elts + .iter() + .for_each(|element| names_in(element, names)), + Expr::List(list) => list + .elts + .iter() + .for_each(|element| names_in(element, names)), + Expr::Starred(starred) => names_in(&starred.value, names), + _ => {} + } +} + +/// Where the scope the loop is in ends: its function, or the module. +fn scope_end(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> TextSize { + let covering = covering_node(context.parsed.syntax().into(), for_loop.range()); + covering + .ancestors() + .find_map(|node| match node { + AnyNodeRef::StmtFunctionDef(function) => Some(function.end()), + _ => None, + }) + .unwrap_or_else(|| context.parsed.syntax().end()) +} + +/// Whether a read of `name` in `range` may read what the loop bound to it, the binding in +/// `target`: a read the loop's binding reaches, or a read in a nested scope, which could run at +/// any time. +fn reads_binding_after( + context: &RefactorContext<'_>, + name: &str, + range: TextRange, + target: TextRange, +) -> bool { + struct Loads<'a> { + name: &'a str, + range: TextRange, + found: Vec<&'a ast::ExprName>, + } + impl<'a> SourceOrderVisitor<'a> for Loads<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.id.as_str() == self.name + && name.ctx.is_load() + && self.range.contains_range(name.range()) + { + self.found.push(name); + } + walk_expr(self, expr); + } + } + let mut loads = Loads { + name, + range, + found: Vec::new(), + }; + loads.visit_body(context.parsed.suite()); + let index = semantic_index(context.db, context.file); + let Some(loop_scope) = covering_node(context.parsed.syntax().into(), target) + .ancestors() + .find_map(|node| match node { + AnyNodeRef::StmtFor(for_loop) => { + index.try_expression_scope_id(&ast::ExprRef::from(&*for_loop.iter)) + } + _ => None, + }) + else { + return !loads.found.is_empty(); + }; + loads.found.into_iter().any(|load| { + let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { + return true; + }; + if scope != loop_scope { + return true; + } + let reaching = reaching_definitions(context.db, index, scope, load); + reaching.maybe_unbound && reaching.definitions.is_empty() + || reaching.definitions.iter().any(|definition| { + target.contains_range(definition.focus_range(context.db, &context.parsed).range()) + }) + }) +} + +fn in_class_body(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> bool { + let covering = covering_node(context.parsed.syntax().into(), for_loop.range()); + covering + .ancestors() + .skip_while(|node| node.range() == for_loop.range()) + .find(|node| { + matches!( + node, + AnyNodeRef::StmtClassDef(_) | AnyNodeRef::StmtFunctionDef(_) + ) + }) + .is_some_and(|node| matches!(node, AnyNodeRef::StmtClassDef(_))) +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + + use crate::refactor::RefactorKind; + use crate::refactor::test_support::RefactorTest; + + #[test] + fn invert_if_flips_a_comparison_of_integers() { + let test = RefactorTest::basedpython( + r#" + def f(a: final int, b: bool): + if a > 1 and not b: + print("yes") + else: + print("no") + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @r#" + Invert `if` condition + --- + + def f(a: final int, b: bool): + if a <= 1 or b: + print("no") + else: + print("yes") + "#); + } + + /// an `int` may be an instance of a subclass, whose `>` is its own and need not be the + /// negation of its `<=` + #[test] + fn invert_if_keeps_not_for_an_int_that_may_be_a_subclass() { + let test = RefactorTest::python( + r#" + def f(a: int): + if a > 1: + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(a: int): + if not (a > 1): + return 2 + else: + return 1 + "); + } + + /// `not x > 1` is not `x <= 1` for a float that is `nan`. + #[test] + fn invert_if_keeps_not_for_a_float() { + let test = RefactorTest::python( + r#" + def f(x: float): + if x > 1: + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(x: float): + if not (x > 1): + return 2 + else: + return 1 + "); + } + + #[test] + fn invert_if_without_else_is_refused() { + let test = RefactorTest::python( + r#" + def f(a): + if a: + return 1 + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @"refused: Invert `if` condition (it has no `else` to swap its body with)"); + } + + #[test] + fn de_morgan_in_a_condition() { + let test = RefactorTest::python( + r#" + def f(a, b): + if not (a or b): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b): + if not a and not b: + return 1 + "); + let test = RefactorTest::python( + r#" + def f(a, b): + if a and not b: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b): + if not (not a or b): + return 2 + "); + } + + /// `a and b` is `a` or `b`, which the rewrite would make a `bool`. + #[test] + fn de_morgan_on_a_value_is_refused() { + let test = RefactorTest::python( + r#" + def f(a: str, b: str): + return a and b + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @"refused: Apply De Morgan's law (its value is used, and the rewrite would make it a `bool`)"); + } + + #[test] + fn flip_quotes_escapes_and_unescapes() { + let test = RefactorTest::python("x = 'single'\n"); + assert_snapshot!(test.apply(RefactorKind::FlipQuotes), @r#" + Convert to a double-quoted string + --- + x = "single" + "#); + let test = RefactorTest::python("x = \"it's\"\n"); + assert_snapshot!(test.apply(RefactorKind::FlipQuotes), @r" + Convert to a single-quoted string + --- + x = 'it\'s' + "); + let test = RefactorTest::python("x = 'say \\'hi\\' \"now\"'\n"); + assert_snapshot!(test.apply(RefactorKind::FlipQuotes), @r#" + Convert to a double-quoted string + --- + x = "say 'hi' \"now\"" + "#); + let test = RefactorTest::python("x = r'it\"s'\n"); + assert_snapshot!(test.apply(RefactorKind::FlipQuotes), @r#"refused: Convert to a double-quoted string (a raw string cannot escape the " it holds)"#); + } + + #[test] + fn a_loop_that_builds_a_list_becomes_a_comprehension() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + if item > 0: + result.append(item * 2) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to list comprehension + --- + + def f(items: list[int]): + result = [item * 2 for item in items if item > 0] + return result + "); + let test = RefactorTest::python( + r#" + def f(items: list[int]): + squares = {} + for item in items: + squares[item] = item * item + return squares + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to dict comprehension + --- + + def f(items: list[int]): + squares = {item: item * item for item in items} + return squares + "); + } + + /// A comprehension leaves `item` unbound, which the loop left bound to the last item. + #[test] + fn a_loop_whose_variable_is_read_after_it_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + result.append(item) + return result, item + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (`item` is read after the loop, which a comprehension leaves unbound)"); + } + + #[test] + fn a_loop_that_reads_what_it_builds_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + seen = [] + for item in items: + if item not in seen: + seen.append(item) + return seen + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (the loop reads `seen` while it is built)"); + } + /// A later loop binds `item` afresh before reading it, so the first loop's `item` is not + /// read after it. + #[test] + fn a_variable_rebound_before_it_is_read_again_does_not_refuse() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + result.append(item) + names = [] + for item in items: + names.append(str(item)) + return result, names + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to list comprehension + --- + + def f(items: list[int]): + result = [item for item in items] + names = [] + for item in items: + names.append(str(item)) + return result, names + "); + } +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index d900370732..69d1fc49d2 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -65,7 +65,11 @@ expression: initialization_result "refactor.rewrite.toDataClass", "refactor.rewrite.fromDataClass", "refactor.rewrite.splitList", - "refactor.rewrite.joinList" + "refactor.rewrite.joinList", + "refactor.rewrite.invertIf", + "refactor.rewrite.deMorgan", + "refactor.rewrite.flipQuotes", + "refactor.rewrite.loopToComprehension" ], "resolveProvider": true }, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index d900370732..69d1fc49d2 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -65,7 +65,11 @@ expression: initialization_result "refactor.rewrite.toDataClass", "refactor.rewrite.fromDataClass", "refactor.rewrite.splitList", - "refactor.rewrite.joinList" + "refactor.rewrite.joinList", + "refactor.rewrite.invertIf", + "refactor.rewrite.deMorgan", + "refactor.rewrite.flipQuotes", + "refactor.rewrite.loopToComprehension" ], "resolveProvider": true }, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 42a6dd3eab..2b68c64e3a 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -458,19 +458,23 @@ a refactoring either keeps the program meaning what it meant or is refused, and refusal says why. an editor that asks for refactorings explicitly shows the reason; one that asks while the caret moves is only offered what applies -| refactoring | kind | offered on | -| ----------------------- | ----------------------------------- | ------------------------------------ | -| inline variable | `refactor.inline.variable` | a variable assigned once | -| extract variable | `refactor.extract.variable` | a selected expression | -| introduce constant | `refactor.extract.constant` | a selected expression of literals | -| introduce parameter | `refactor.extract.parameter` | an expression of literals in a `def` | -| introduce attribute | `refactor.extract.field` | a selected expression in a method | -| extract function | `refactor.extract.function` | selected statements | -| add return annotation | `refactor.rewrite.returnAnnotation` | a `def` header with no `->` | -| convert to data class | `refactor.rewrite.toDataClass` | a class header | -| convert from data class | `refactor.rewrite.fromDataClass` | a `data class` header | -| put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | -| put on one line | `refactor.rewrite.joinList` | a list spread over several lines | +| refactoring | kind | offered on | +| ----------------------- | -------------------------------------- | ------------------------------------ | +| inline variable | `refactor.inline.variable` | a variable assigned once | +| extract variable | `refactor.extract.variable` | a selected expression | +| introduce constant | `refactor.extract.constant` | a selected expression of literals | +| introduce parameter | `refactor.extract.parameter` | an expression of literals in a `def` | +| introduce attribute | `refactor.extract.field` | a selected expression in a method | +| extract function | `refactor.extract.function` | selected statements | +| add return annotation | `refactor.rewrite.returnAnnotation` | a `def` header with no `->` | +| convert to data class | `refactor.rewrite.toDataClass` | a class header | +| convert from data class | `refactor.rewrite.fromDataClass` | a `data class` header | +| put on separate lines | `refactor.rewrite.splitList` | a list not yet one element a line | +| put on one line | `refactor.rewrite.joinList` | a list spread over several lines | +| invert `if` condition | `refactor.rewrite.invertIf` | an `if` with an `else` | +| apply De Morgan's law | `refactor.rewrite.deMorgan` | an `and` or an `or` | +| convert quotes | `refactor.rewrite.flipQuotes` | a string or bytes literal | +| loop to comprehension | `refactor.rewrite.loopToComprehension` | a loop filling an empty collection | inlining is offered on the variable's assignment and on any of its reads alike, and replaces every read either way @@ -562,6 +566,32 @@ decided by working out the rewrite, so the actions a `textDocument/codeAction` reply offers have each been computed to answer it. send `only` to narrow that — a request that does not ask for `refactor` kinds does no refactoring work at all +inverting an `if` negates its condition and swaps its body with its `else`'s. a +negation takes off a `not`, swaps `and` and `or` by De Morgan's law, and flips a +comparison's operator only where that means the same: `is`, `is not`, `in` and +`not in` always, `==` and `!=` between python's own numbers, strings and bytes, and +the orderings between its integers, strings or bytes — `not x > 1` is not +`x <= 1` for a float that is `nan`, so a float's comparison is written +`not (x > 1)`. inverting an `if` and De Morgan's law are refused where a comment +in the condition would be lost. De Morgan's law rewrites `not (a or b)` as `not a and not b`, and +`a and b` as `not (not a or not b)` where only the truth of it is read — an `if`, a +`while`, an `assert` or a `not` — or where it is a `bool` already, since `a and b` +otherwise evaluates to `a` or `b`. converting a string's quotes escapes the new +quote where it appears and unescapes the old one, and is refused for a raw string +holding the new quote, which cannot be escaped. a loop that only adds to a +collection made empty right above it — a list's `append`, a set's `add`, a dict's +item assignment, under at most one `if` — becomes a comprehension, refused where +the loop's variable is read anywhere outside the loop that the loop's binding may +reach — above it on a later round, in a function, or as `del` (a comprehension +leaves it unbound) — or is declared `global` or `nonlocal`, where the loop reads +what it builds, where a function that reads the collection may be called while the +loop builds it, where it has an `else`, in a class body, where it adds what a +`yield` evaluates to or assigns its variable or its iterable with a walrus, which +python does not allow in a comprehension, where it has a comment the comprehension +would not keep, where it calls `super()` without arguments before python 3.12, and +for a dict whose key and value could both have effects, which a comprehension +evaluates in the other order + putting a list on separate lines, or on one, rewrites the innermost list with brackets of its own around the caret — a call's arguments, a definition's parameters, a display's elements — and only the text between its elements. an From 7d2bd0fb1e0ecb0630fec726fbd3277206c9165c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:31:33 +1000 Subject: [PATCH 039/110] name in by/documentUsageCounts, with implementations, each subclass of a class and each member overriding a method that it counts, by the class it is and its qualified name, shapes.base.Circle, with where it is declared, in the order textDocument/implementation answers them, so an editor can say which classes they are without asking for each count --- crates/ty_ide/src/declaration.rs | 85 ++++++++++++++----- crates/ty_ide/src/goto.rs | 59 +++++++------ crates/ty_ide/src/goto_implementation.rs | 68 +++++++++++---- crates/ty_ide/src/hover.rs | 6 +- crates/ty_ide/src/lib.rs | 4 +- crates/ty_ide/src/usage_counts.rs | 76 ++++++++++++----- .../src/types/ide_support.rs | 2 +- .../api/requests/document_usage_counts.rs | 45 +++++++++- crates/ty_server/tests/e2e/usage_counts.rs | 79 ++++++++++++++++- docs/basedpython/features/editor.md | 23 +++++ 10 files changed, 354 insertions(+), 93 deletions(-) diff --git a/crates/ty_ide/src/declaration.rs b/crates/ty_ide/src/declaration.rs index bd6a22e157..5fcb4caac7 100644 --- a/crates/ty_ide/src/declaration.rs +++ b/crates/ty_ide/src/declaration.rs @@ -1,5 +1,5 @@ -//! What a declaration is: its kind, its name, and the dotted name it is reached by from outside -//! its module. +//! what a declaration is: its kind, its name, and the dotted name it is reached by from outside +//! its module //! //! ```py //! # shapes/base.py @@ -11,9 +11,9 @@ //! ``` //! //! `Circle` is the class `shapes.base.Circle`, `area` the method `shapes.base.Circle.area`, and -//! `radius` the attribute `shapes.base.Circle.radius`, although it is assigned in `__init__`. A +//! `radius` the attribute `shapes.base.Circle.radius`, although it is assigned in `__init__`. a //! name local to a function, a parameter among them, has no qualified name, and nor does a type -//! parameter: nothing outside the function or the generic reaches it. +//! parameter: nothing outside the function or the generic reaches it use ruff_db::files::FileRange; use ruff_db::parsed::parsed_module; @@ -29,18 +29,18 @@ use ty_python_semantic::{HasDefinition, ImportAliasResolution, ResolvedDefinitio use crate::goto::{Definitions, GotoTarget, find_goto_target}; use crate::{Db, NavigationTarget, RangedValue}; -/// What kind of thing a declaration declares. +/// what kind of thing a declaration declares #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DeclaredKind { Module, Class, Function, - /// A `def` in a class body. + /// a `def` in a class body Method, - /// A `def` in a class body that its decorators make a property. + /// a `def` in a class body that its decorators make a property Property, - /// A name a class body binds that is not a `def` or a class, or an attribute a method assigns - /// on `self`. + /// a name a class body binds that is not a `def` or a class, or an attribute a method assigns + /// on `self` Attribute, Variable, Parameter, @@ -49,7 +49,7 @@ pub enum DeclaredKind { } impl DeclaredKind { - /// The kind as a reader names it. + /// the kind as a reader names it pub const fn as_str(self) -> &'static str { match self { DeclaredKind::Module => "module", @@ -66,15 +66,15 @@ impl DeclaredKind { } } -/// A declaration, described. +/// a declaration, described #[derive(Debug, Clone, PartialEq, Eq)] pub struct Declaration { pub kind: DeclaredKind, - /// The declared name. + /// the declared name pub name: String, - /// The module's dotted name, then each class the declaration is made in, then the name: + /// the module's dotted name, then each class the declaration is made in, then the name: /// `shapes.base.Circle.area`. `None` for a name local to a function, and for a declaration - /// in a file no module is imported from. + /// in a file no module is imported from pub qualified_name: Option, } @@ -141,8 +141,8 @@ pub(crate) fn single_declaration(db: &dyn Db, declared: &Definitions<'_>) -> Opt .then_some(first) } -/// What `resolved` declares. `None` for what is not a declaration of a name: an import that was -/// not followed to what it imports, or a range in a file that is not a definition. +/// what `resolved` declares. `None` for what is not a declaration of a name: an import that was +/// not followed to what it imports, or a range in a file that is not a definition pub(crate) fn declaration(db: &dyn Db, resolved: &ResolvedDefinition<'_>) -> Option { match resolved { ResolvedDefinition::Module(file) => { @@ -227,8 +227,55 @@ fn definition_declaration(db: &dyn Db, definition: Definition<'_>) -> Option, +} + +/// the class `resolved` declares, or the class it is a member of: `Circle` for the class +/// `Circle`, for its method `area` and for the `radius` its `__init__` assigns on `self` alike. +/// `None` for anything else, such as a module's function +pub(crate) fn declaring_class(db: &dyn Db, resolved: &ResolvedDefinition<'_>) -> Option { + let ResolvedDefinition::Definition(definition) = resolved else { + return None; + }; + let definition = *definition; + if matches!(definition.kind(db), DefinitionKind::Class(_)) { + let declaration = definition_declaration(db, definition)?; + return Some(ClassName { + name: declaration.name, + qualified_name: declaration.qualified_name, + }); + } + let file = definition.program_file(db); + let index = semantic_index(db, file); + let scope = definition.file_scope(db); + if let PlaceExprRef::Member(member) = index.place_table(scope).place(definition.place(db)) { + member.as_instance_attribute()?; + let class = index.class_definition_of_method(scope)?; + return declaring_class(db, &ResolvedDefinition::Definition(class)); + } + if index.scope(scope).kind() != ScopeKind::Class { + return None; + } + let module = parsed_module(db, file.python_file(db)).load(db); + let name = scope.to_scope_id(db, file).name(db, &module).to_string(); + let qualified_name = containers(db, file, scope).and_then(|containers| { + let module = module_name(db, file)?; + Some(format!("{module}.{}", containers.join("."))) + }); + Some(ClassName { + name, + qualified_name, + }) +} + +/// the names of the classes `scope` is nested in, outermost first, `scope` itself included when +/// it is a class's. `None` when a function encloses it, which makes what it declares local fn containers(db: &dyn Db, file: ProgramFile<'_>, scope: FileScopeId) -> Option> { let index = semantic_index(db, file); let module = parsed_module(db, file.python_file(db)).load(db); @@ -252,7 +299,7 @@ fn containers(db: &dyn Db, file: ProgramFile<'_>, scope: FileScopeId) -> Option< Some(containers) } -/// The dotted name `file` is imported by. +/// the dotted name `file` is imported by fn module_name(db: &dyn Db, file: ProgramFile<'_>) -> Option { file_to_module(db, file.resolver_file(db)).map(|module| module.name(db).to_string()) } diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index 1c1ffd70be..46b16709ba 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -279,6 +279,37 @@ fn is_declared_by_object(model: &SemanticModel<'_>, definition: Definition<'_>) == object } +/// where an editor goes for `definition`: the name it declares, within the whole of it +pub(crate) fn navigation_target( + db: &dyn ty_python_semantic::Db, + definition: &ResolvedDefinition<'_>, +) -> NavigationTarget { + match definition { + ResolvedDefinition::Definition(definition) => { + let module = ruff_db::parsed::parsed_module(db, definition.python_file(db)).load(db); + + let focus_range = definition.focus_range(db, &module); + let full_range = definition.full_range(db, &module); + + NavigationTarget::create( + db, + focus_range.file(), + focus_range.range(), + full_range.range(), + ) + } + ResolvedDefinition::Module(file) => { + NavigationTarget::new(file.file(db), TextRange::default()) + } + ResolvedDefinition::FileWithRange(file_range) => NavigationTarget::create( + db, + file_range.file(), + file_range.range(), + file_range.range(), + ), + } +} + /// The resolved definitions for a `GotoTarget` #[derive(Debug)] pub(crate) struct Definitions<'db>(Vec>); @@ -461,32 +492,8 @@ impl<'db> Definitions<'db> { db: &'db dyn ty_python_semantic::Db, ) -> crate::NavigationTargets { self.0 - .into_iter() - .map(|definition| match definition { - ResolvedDefinition::Definition(definition) => { - let module = - ruff_db::parsed::parsed_module(db, definition.python_file(db)).load(db); - - let focus_range = definition.focus_range(db, &module); - let full_range = definition.full_range(db, &module); - - NavigationTarget::create( - db, - focus_range.file(), - focus_range.range(), - full_range.range(), - ) - } - ResolvedDefinition::Module(file) => { - NavigationTarget::new(file.file(db), TextRange::default()) - } - ResolvedDefinition::FileWithRange(file_range) => NavigationTarget::create( - db, - file_range.file(), - file_range.range(), - file_range.range(), - ), - }) + .iter() + .map(|definition| navigation_target(db, definition)) .collect() } diff --git a/crates/ty_ide/src/goto_implementation.rs b/crates/ty_ide/src/goto_implementation.rs index ae63e274a7..e78857af84 100644 --- a/crates/ty_ide/src/goto_implementation.rs +++ b/crates/ty_ide/src/goto_implementation.rs @@ -55,7 +55,8 @@ //! customize what happens when an attribute is read, assigned, or deleted (descriptors) are not //! interpreted as properties. -use crate::goto::{Definitions, GotoTarget, find_goto_target}; +use crate::declaration::{ClassName, declaring_class}; +use crate::goto::{Definitions, GotoTarget, find_goto_target, navigation_target}; use crate::{Db, NavigationTarget, NavigationTargets, RangedValue}; use rayon::prelude::*; use ruff_db::files::{File, FileRange}; @@ -123,8 +124,18 @@ pub fn goto_implementation( }) } +/// an implementation [`implementations_at_each`] finds, and the class it is or is declared in +#[derive(Debug, Clone)] +pub(crate) struct Implementation { + pub(crate) target: NavigationTarget, + /// the subclass that is the implementation, or the class whose member overrides. `None` for + /// an implementation that is neither + pub(crate) class: Option, +} + /// What [`goto_implementation`] answers at each of `offsets` in `file`, found with one pass over -/// the project's files for all of them rather than one pass each. +/// the project's files for all of them rather than one pass each, with the class each +/// implementation is or is declared in /// /// Each file is scanned once, for every target in turn; the same scan of the same files as /// [`goto_implementation`] makes for each target, so each finds the same implementations, in the @@ -133,7 +144,7 @@ pub(crate) fn implementations_at_each( db: &dyn Db, file: ProgramFile<'_>, offsets: &[TextSize], -) -> Vec> { +) -> Vec>> { let module = parsed_module(db, file.python_file(db)).load(db); let model = SemanticModel::new(db, file); let finders: Vec<_> = offsets @@ -167,32 +178,38 @@ pub(crate) fn implementations_at_each( .enumerate() .filter_map(|(index, finder)| { let definitions = finder.as_ref()?.implementations_for_file(db, file)?; - let targets = definitions_to_implementation_targets(db, definitions); - (!targets.is_empty()).then_some((index, targets)) + let found = definitions_to_implementations(db, definitions); + (!found.is_empty()).then_some((index, found)) }) .collect::>() }) .collect::>(); - let mut implementation_targets: Vec>> = finders + let mut implementations: Vec>> = finders .into_iter() .map(|finder| { - finder.map(|finder| { - definitions_to_implementation_targets(db, finder.into_initial_definitions()) - }) + finder + .map(|finder| definitions_to_implementations(db, finder.into_initial_definitions())) }) .collect(); - for (index, targets) in batches.into_iter().flatten() { - if let Some(found) = &mut implementation_targets[index] { - found.extend(targets); + for (index, found) in batches.into_iter().flatten() { + if let Some(implementations) = &mut implementations[index] { + implementations.extend(found); } } - implementation_targets + implementations .into_iter() - .map(|targets| { - targets - .filter(|targets| !targets.is_empty()) - .map(|targets| targets.into_iter().collect()) + .map(|found| { + let mut found = found?; + // as `NavigationTargets` holds them: each target once, by file and then by offset + found.sort_by_key(|implementation| { + ( + implementation.target.file(), + implementation.target.focus_range().start(), + ) + }); + found.dedup_by(|a, b| a.target == b.target); + (!found.is_empty()).then_some(found) }) .collect() } @@ -256,6 +273,23 @@ fn definitions_to_implementation_targets( .unwrap_or_default() } +/// [`definitions_to_implementation_targets`], with the class each is or is declared in +fn definitions_to_implementations( + db: &dyn Db, + definitions: Vec, +) -> Vec { + let Some(definitions) = Definitions::new(definitions).map_stubs_for_implementation(db) else { + return Vec::new(); + }; + definitions + .iter() + .map(|definition| Implementation { + target: navigation_target(db, definition), + class: declaring_class(db, definition), + }) + .collect() +} + #[cfg(test)] mod tests { use crate::goto_implementation; diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index 2e826eeb93..d1809b34c1 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -335,7 +335,7 @@ struct DecoratedFunctionHover<'db> { } /// `None` for a `def` without decorators, whose type is the function it writes, and for a use of -/// a property, which reads the property's value rather than naming the `def`. +/// a property, which reads the property's value rather than naming the `def` fn decorated_function_hover<'db>( db: &'db dyn Db, goto_target: &GotoTarget<'_>, @@ -631,9 +631,9 @@ impl<'a, 'db> IntoIterator for &'a Hover<'db> { #[derive(Debug, Clone)] pub enum HoverContent<'db> { - /// What the hovered name declares: its kind, and its qualified name when it has one. + /// what the hovered name declares: its kind, and its qualified name when it has one Declaration(Declaration), - /// A decorated `def`: its decorators, as written, over the function it writes. + /// a decorated `def`: its decorators, as written, over the function it writes DecoratedFunction { decorators: Vec, function: Type<'db>, diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index de8bba65b5..655fb9f5ac 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -72,7 +72,7 @@ pub use completion::{ CompletionInsertTextFormat, CompletionKind, CompletionSettings, completion, }; pub use data_flow::{Finding, FindingKind, data_flow_at}; -pub use declaration::{Declaration, DeclarationAt, DeclaredKind, declaration_at}; +pub use declaration::{ClassName, Declaration, DeclarationAt, DeclaredKind, declaration_at}; pub use django_template::{ DisplayTemplateHover, DjangoChecker, DjangoCodeLens, DjangoLensAction, DjangoLensTarget, PreparedTemplateRename, TemplateCompletion, TemplateEdit, TemplateHover, TemplateInlayHint, @@ -139,7 +139,7 @@ pub use type_hierarchy::{ }; pub use unwrap::{UnwrapOption, unwrap_options}; pub use usage_counts::{ - DeclarationKind, DeclarationUsages, UsageCountKinds, document_usage_counts, + DeclarationKind, DeclarationUsages, Implementor, UsageCountKinds, document_usage_counts, }; pub use workspace_symbols::{WorkspaceSymbolInfo, workspace_symbols}; diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 79651afeec..1642f67c24 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -29,10 +29,11 @@ use ruff_text_size::TextRange; use ty_python_core::ProgramFile; use ty_python_semantic::types::ide_support::{class_member_definitions, overridden_members}; +use crate::declaration::ClassName; use crate::find_references::find_references_to_each; use crate::goto_implementation::implementations_at_each; use crate::symbols::symbols_for_file; -use crate::{Db, SymbolKind}; +use crate::{Db, NavigationTarget, SymbolKind}; /// which counts to work out: each is a search of the workspace #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -72,6 +73,20 @@ pub struct DeclarationUsages { /// whether a method is abstract, an `@abstractmethod` or a protocol member /// with no body, so that what overrides it implements it pub is_abstract: bool, + /// what `implementations` counts, in the order `textDocument/implementation` + /// answers them: each subclass of a class, or each member overriding a + /// method, with the class it is in. empty when not asked for + pub implementors: Vec, +} + +/// a subclass of a counted class, or a member overriding a counted method +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Implementor { + /// the subclass, or the class whose member overrides the method. `None` for + /// one in no class + pub class: Option, + /// the subclass or the overriding member, where a click on it goes + pub target: NavigationTarget, } /// the declarations of `file` with the counts `kinds` asks for, in source order @@ -108,16 +123,20 @@ pub fn document_usage_counts( let found = implementations_at_each(db, file, &offsets); for (index, implementations) in counted.into_iter().zip(found) { let declaration = &mut declarations[index]; - declaration.implementations = Some(implementations.map_or(0, |found| { - found - .iter() - .filter(|target| { - // the declaration is its own first implementation - target.file() != source_file - || target.focus_range() != declaration.name_range - }) - .count() - })); + declaration.implementors = implementations + .unwrap_or_default() + .into_iter() + // the declaration is its own first implementation + .filter(|implementation| { + implementation.target.file() != source_file + || implementation.target.focus_range() != declaration.name_range + }) + .map(|implementation| Implementor { + class: implementation.class, + target: implementation.target, + }) + .collect(); + declaration.implementations = Some(declaration.implementors.len()); } } @@ -158,6 +177,7 @@ fn counted_declarations(db: &dyn Db, file: ProgramFile<'_>) -> Vec` for each implementor, and + /// `(abstract)` for an abstract method fn usage_counts(&self) -> String { let file = self.cursor.file; let source = source_text(&self.db, file); @@ -255,6 +276,21 @@ mod tests { if let Some(implementations) = declaration.implementations { let _ = write!(line, " impls={implementations}"); } + for implementor in &declaration.implementors { + let file = implementor.target.file(); + let target_source = source_text(&self.db, file); + let _ = write!( + line, + " <{} {}:{}>", + implementor.class.as_ref().map_or("?", |class| class + .qualified_name + .as_deref() + .unwrap_or(&class.name)), + file.path(&self.db).as_str(), + LineIndex::from_source_text(&target_source) + .line_index(implementor.target.focus_range().start()) + ); + } if declaration.is_abstract { line.push_str(" (abstract)"); } @@ -294,11 +330,11 @@ make(Puppy()) ", ); assert_snapshot!(test.usage_counts(), @" - class Animal 4 refs=2 impls=2 - method speak 6 refs=1 impls=2 (abstract) + class Animal 4 refs=2 impls=2 + method speak 6 refs=1 impls=2 (abstract) method name 8 refs=1 impls=0 - class Dog 11 refs=2 impls=1 - method speak 12 refs=0 impls=1 + class Dog 11 refs=2 impls=1 + method speak 12 refs=0 impls=1 class Puppy 15 refs=1 impls=0 method speak 16 refs=0 impls=0 function make 19 refs=2 @@ -332,8 +368,8 @@ print(Shape().area(), Square().area()) ) .build(); assert_snapshot!(test.usage_counts(), @" - class Shape 2 refs=2 impls=1 - method area 3 refs=1 impls=1 + class Shape 2 refs=2 impls=1 + method area 3 refs=1 impls=1 function unused 6 refs=0 "); } @@ -361,8 +397,8 @@ class Circle(Shape): ); assert_snapshot!(test.usage_counts(), @" function logged 2 refs=1 - class Shape 7 refs=1 impls=1 - method area 8 refs=0 impls=1 + class Shape 7 refs=1 impls=1 + method area 8 refs=0 impls=1 class Circle 11 refs=0 impls=0 method area 13 refs=0 impls=0 "); diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 456cb63a07..fcc9d0be18 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -5592,7 +5592,7 @@ cast(*args) } } -/// basedpython: a `def` as its source writes it and as its decorators leave it. +/// basedpython: a `def` as its source writes it and as its decorators leave it #[derive(Debug, Clone, Copy)] pub struct DecoratedFunction<'db> { /// the function the `def` makes, before any decorator applies to it diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs index b2436b8c04..58d54fea62 100644 --- a/crates/ty_server/src/server/api/requests/document_usage_counts.rs +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -33,7 +33,7 @@ use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, TextDocument use ty_ide::{DeclarationKind, UsageCountKinds, document_usage_counts}; use ty_project::{ProjectDatabase, SemanticDb as _}; -use crate::document::ToRangeExt; +use crate::document::{ToLink, ToRangeExt}; use crate::server::api::traits::{ BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, }; @@ -102,6 +102,31 @@ pub(crate) struct DeclarationUsages { /// method with implementations counted and found #[serde(rename = "abstract")] is_abstract: bool, + /// what `implementations` counts, in the order `textDocument/implementation` + /// answers them, so that a client can name them without asking. there when + /// `implementations` is + #[serde(skip_serializing_if = "Option::is_none")] + implementors: Option>, +} + +/// a subclass of a class, or a member overriding a method +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Implementor { + /// the subclass, or the class whose member overrides the method: `Circle`. + /// absent for an implementation in no class + #[serde(skip_serializing_if = "Option::is_none")] + class_name: Option, + /// the dotted name of that class, `shapes.base.Circle`. absent for a class + /// local to a function, or in a file no module is imported from + #[serde(skip_serializing_if = "Option::is_none")] + qualified_name: Option, + /// the file the subclass or the overriding member is declared in + uri: Uri, + /// the whole subclass or member, its decorators included + range: Range, + /// its name, which is where to go + selection_range: Range, } pub(crate) struct DocumentUsageCountsRequestHandler; @@ -166,6 +191,24 @@ impl BackgroundDocumentRequestHandler for DocumentUsageCountsRequestHandler { DeclarationKind::Function => Kind::Function, DeclarationKind::Method => Kind::Method, }, + implementors: declaration.implementations.map(|_| { + declaration + .implementors + .iter() + .filter_map(|implementor| { + let link = implementor.target.to_link(db, None, encoding)?; + let class = implementor.class.as_ref(); + Some(Implementor { + class_name: class.map(|class| class.name.clone()), + qualified_name: class + .and_then(|class| class.qualified_name.clone()), + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + }) + }) + .collect() + }), references: declaration.references, implementations: declaration.implementations, is_abstract: declaration.is_abstract, diff --git a/crates/ty_server/tests/e2e/usage_counts.rs b/crates/ty_server/tests/e2e/usage_counts.rs index ffeefc5178..3d84c1e49b 100644 --- a/crates/ty_server/tests/e2e/usage_counts.rs +++ b/crates/ty_server/tests/e2e/usage_counts.rs @@ -221,15 +221,86 @@ fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result None => Vec::new(), }; // less the declaration itself, which is the first of its own implementations - let others = implementations - .iter() + let others: Vec<_> = implementations + .into_iter() .filter(|(target, start)| *target != uri || *start != at.position) - .count(); + .collect(); assert_eq!( declaration["implementations"].as_u64(), - Some(others as u64), + Some(others.len() as u64), "{declaration:#}" ); + // and the implementors are those places, in that order + let implementors: Vec<_> = declaration["implementors"] + .as_array() + .expect("implementors with implementations") + .iter() + .map(|implementor| { + ( + serde_json::from_value::(implementor["uri"].clone()) + .expect("a uri"), + serde_json::from_value::( + implementor["selectionRange"]["start"].clone(), + ) + .expect("a position"), + ) + }) + .collect(); + assert_eq!(implementors, others, "{declaration:#}"); + } + Ok(()) +} + +/// each subclass of a class, and each member overriding a method, is named by its class, so that +/// a client can say which classes they are without asking where each count goes +#[test] +fn implementors_are_named_by_their_class() -> Result<()> { + let mut server = server()?; + + let declarations = ask(&mut server, &["implementations"]); + let implementors: Vec<_> = declarations + .iter() + .map(|declaration| { + ( + declaration["name"].as_str().unwrap_or_default().to_string(), + declaration.get("implementors").map(|implementors| { + implementors + .as_array() + .expect("a list") + .iter() + .map(|implementor| { + format!( + "{} {} {}:{}", + implementor["className"].as_str().unwrap_or_default(), + implementor["qualifiedName"].as_str().unwrap_or_default(), + implementor["selectionRange"]["start"]["line"], + implementor["range"]["start"]["line"], + ) + }) + .collect::>() + }), + ) + }) + .collect(); + assert_eq!( + implementors, + [ + ( + "Shape".to_string(), + Some(vec!["Square main.Square 2:2".to_string()]) + ), + ( + "area".to_string(), + Some(vec!["Square main.Square 3:3".to_string()]) + ), + ("describe".to_string(), Some(vec![])), + ("total".to_string(), None), + ] + ); + + // asked for references only, no implementors + for declaration in ask(&mut server, &["references"]) { + assert_eq!(declaration.get("implementors"), None, "{declaration:#}"); } Ok(()) } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 2b68c64e3a..5dcfcd5b06 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1200,6 +1200,29 @@ sends that request from the name. a method with implementations is marked `abstract` when it is an `@abstractmethod` or a protocol member with no body, so that what overrides it implements it. a count not named is not worked out +with `implementations` comes `implementors`, the places it counts, in the +order `textDocument/implementation` answers them, so that an editor can say +"is overridden in Circle, Square" without asking. each is a subclass of the +class, or a member overriding the method, with the `uri`, `range` and +`selectionRange` it is declared at and the class it is in: `className`, the +subclass itself or the class whose member overrides, and `qualifiedName`, that +class's dotted name, `shapes.base.Circle`, absent for a class local to a +function + +```json +{ + "name": "area", "kind": "method", "implementations": 1, "abstract": true, + "implementors": [ + { + "className": "Circle", "qualifiedName": "shapes.base.Circle", + "uri": "file:///src/shapes/base.by", + "range": { "start": { "line": 32, "character": 4 }, "end": { "line": 34, "character": 41 } }, + "selectionRange": { "start": { "line": 33, "character": 8 }, "end": { "line": 33, "character": 12 } } + } + ] +} +``` + each count is a search of the workspace, made for all the declarations of the document in one pass: each file is read once, and searched for each declaration whose name it holds. the searches run one at a time, on a thread of their own, From 9d2cbc478f1ec539899316fda133b00cbc34f5d1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:35:16 +1000 Subject: [PATCH 040/110] count a dunder method in by/documentUsageCounts for the methods overriding it, as PyCharm marks a base's __init__ overridden in Circle and Square, leaving out only its references, which the language makes rather than its name, and a module's dunder function --- crates/ty_ide/src/usage_counts.rs | 68 ++++++++++++++----- .../api/requests/document_usage_counts.rs | 5 +- docs/basedpython/features/editor.md | 7 +- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 1642f67c24..3ced11449e 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -16,10 +16,11 @@ //! //! the declarations counted are the document's outline ([`crate::document_symbols()`]) //! at the depth a reader scans a module at: its classes and functions, and the -//! methods and properties of those classes. a dunder is left out, being called -//! by the language rather than by name, and so is an `extension` block, which -//! names a class it does not declare. a nested function or a class inside a -//! class is not counted +//! methods and properties of those classes. a dunder method is counted for its +//! overrides alone, as PyCharm marks `__init__` overridden: its usages are not +//! counted, since the language calls it rather than its name. a module's dunder +//! function is left out, and so is an `extension` block, which names a class it +//! does not declare. a nested function or a class inside a class is not counted use ruff_db::parsed::parsed_module; use ruff_python_ast::AnyNodeRef; @@ -65,7 +66,7 @@ pub struct DeclarationUsages { /// its name where it is declared pub name_range: TextRange, /// how many places use it, the declaration itself not among them. `None` - /// when not asked for + /// when not asked for, and for a dunder method pub references: Option, /// how many classes subclass a class, directly or not, and how many members /// of those override a method. `None` when not asked for, and for a function @@ -103,13 +104,17 @@ pub fn document_usage_counts( let mut declarations = declarations; if kinds.references { - let offsets: Vec<_> = declarations + // the language calls a dunder, not its name + let (offsets, counted): (Vec<_>, Vec<_>) = declarations .iter() - .map(|declaration| declaration.name_range.start()) - .collect(); + .enumerate() + .filter(|(_, declaration)| !is_dunder(&declaration.name)) + .map(|(index, declaration)| (declaration.name_range.start(), index)) + .unzip(); let found = find_references_to_each(db, file, &offsets, false); - for (declaration, references) in declarations.iter_mut().zip(found) { - declaration.references = Some(references.map_or(0, |references| references.len())); + for (index, references) in counted.into_iter().zip(found) { + declarations[index].references = + Some(references.map_or(0, |references| references.len())); } } if kinds.implementations { @@ -168,7 +173,8 @@ fn counted_declarations(db: &dyn Db, file: ProgramFile<'_>) -> Vec class A: - def __init__(self) -> None: ... - def f(self) -> None: def inner() -> None: ... inner() @@ -420,13 +424,45 @@ class A: extension list: def second(self) -> int: return 0 + +def __getattr__(name: str) -> int: + return 0 ", ) .build(); assert_snapshot!(test.usage_counts(), @" class A 2 refs=0 impls=0 - method f 5 refs=0 impls=0 - method second 10 refs=0 impls=0 + method f 3 refs=0 impls=0 + method second 8 refs=0 impls=0 + "); + } + + /// A dunder method is counted for the methods overriding it, as PyCharm marks a base's + /// `__init__` overridden, but not for its usages: the language calls it, not its name. + #[test] + fn a_dunder_method_is_counted_for_its_overrides_alone() { + let test = cursor_test( + " +class Shape: + def __init__(self, name: str) -> None: + self.name = name + + def __eq__(self, other: object) -> bool: + return False + +class Circle(Shape): + def __init__(self) -> None: + super().__init__('circle') + +Circle() == Shape('shape') +", + ); + assert_snapshot!(test.usage_counts(), @" + class Shape 2 refs=2 impls=1 + method __init__ 3 impls=1 + method __eq__ 6 impls=0 + class Circle 9 refs=1 impls=0 + method __init__ 10 impls=0 "); } } diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs index 58d54fea62..e7be58e322 100644 --- a/crates/ty_server/src/server/api/requests/document_usage_counts.rs +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -89,7 +89,8 @@ pub(crate) struct DeclarationUsages { /// answer for it selection_range: Range, /// how many places use it, as `textDocument/references` answers without the - /// declaration. there when asked for + /// declaration. there when asked for, except on a dunder method, which the + /// language calls rather than its name #[serde(skip_serializing_if = "Option::is_none")] references: Option, /// how many classes subclass a class, directly or not, and how many of their @@ -147,7 +148,7 @@ impl BackgroundDocumentRequestHandler for DocumentUsageCountsRequestHandler { /// `null` when language services are off, or for a document whose declarations are not /// counted here: a notebook, whose ranges are per cell, or a template. otherwise the /// declarations in source order: the document's classes and functions, and the methods and - /// properties of those classes, less dunders and `extension` blocks + /// properties of those classes, less `extension` blocks and a module's dunder functions fn run_with_snapshot( db: &ProjectDatabase, snapshot: &DocumentSnapshot, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 5dcfcd5b06..1006921fee 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1183,8 +1183,11 @@ each of its declarations and how many classes subclass it or members override it: what an editor draws above each one as "3 usages" or "2 overrides". the declarations are the document's outline at the depth a reader scans a module at — its classes and functions, and the methods and properties of those -classes. a dunder is left out, being called by the language rather than by -name, and so is an `extension` block, which names a class it does not declare +classes. a dunder method is there for what overrides it, as a base's `__init__` +is marked overridden in PyCharm, but its usages are not counted, the language +calling it rather than its name: it has no `references`, even when they are +asked for. a module's dunder function is left out, and so is an `extension` +block, which names a class it does not declare each declaration comes with its `name`, its `kind` (`class`, `function` or `method`), its whole `range` from its first decorator, its `selectionRange` on From 850d5bfcbf55656e5235b351e6a1025a7798a048 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:45:16 +1000 Subject: [PATCH 041/110] answer by/references, textDocument/references with what each reference does with the name, import, isinstance, typeHint, superclass, write, read or declaration, the groups PyCharm's python sorts usages into, for an editor's find usages to group them by --- crates/ty_ide/src/lib.rs | 2 + crates/ty_ide/src/usage_kinds.rs | 262 ++++++++++++++++++ .../src/types/ide_support.rs | 9 + crates/ty_server/src/server/api.rs | 5 + crates/ty_server/src/server/api/requests.rs | 4 +- .../api/requests/classified_references.rs | 131 +++++++++ .../src/server/api/requests/references.rs | 52 ++-- .../tests/e2e/classified_references.rs | 118 ++++++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 31 +++ 10 files changed, 589 insertions(+), 26 deletions(-) create mode 100644 crates/ty_ide/src/usage_kinds.rs create mode 100644 crates/ty_server/src/server/api/requests/classified_references.rs create mode 100644 crates/ty_server/tests/e2e/classified_references.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 655fb9f5ac..e96f4decef 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -57,6 +57,7 @@ mod test_links; mod type_hierarchy; mod unwrap; mod usage_counts; +mod usage_kinds; mod workspace_symbols; pub use add_dependency::{AddDependency, DependencyTarget}; @@ -141,6 +142,7 @@ pub use unwrap::{UnwrapOption, unwrap_options}; pub use usage_counts::{ DeclarationKind, DeclarationUsages, Implementor, UsageCountKinds, document_usage_counts, }; +pub use usage_kinds::{UsageKind, usage_kinds}; pub use workspace_symbols::{WorkspaceSymbolInfo, workspace_symbols}; use ruff_db::{ diff --git a/crates/ty_ide/src/usage_kinds.rs b/crates/ty_ide/src/usage_kinds.rs new file mode 100644 index 0000000000..cb6ee13819 --- /dev/null +++ b/crates/ty_ide/src/usage_kinds.rs @@ -0,0 +1,262 @@ +//! What each reference a search finds does with the name: what an editor's *find usages* +//! groups them by. +//! +//! ```py +//! from shapes.base import Shape # import +//! +//! class Circle(Shape): ... # superclass +//! +//! def total(shapes: list[Shape]): ... # type hint +//! +//! if isinstance(item, Shape): ... # isinstance +//! +//! Shape = make_shape() # write +//! print(Shape) # read +//! ``` +//! +//! These are the groups PyCharm's Python sorts usages into, in the order it tries them: a +//! reference inside an import statement is an import whatever else it is, then the class list of +//! an `isinstance` call, then a type expression, then a class's bases. What is none of those is a +//! read or a write of the name, or the declaration itself. + +use ruff_db::parsed::parsed_module; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::{self as ast, AnyNodeRef}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_core::ProgramFile; +use ty_python_semantic::SemanticModel; +use ty_python_semantic::types::ide_support::{ + CallArgumentForm, call_argument_forms, calls_isinstance, +}; + +use crate::django_template::is_django_template_path; +use crate::{Db, ReferenceKind, ReferenceTarget}; + +/// What a reference does with the name it refers to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum UsageKind { + /// The name where it is declared: a `def`'s or a `class`'s name, or a first assignment. + Declaration, + /// In an `import` or a `from ... import` statement. + Import, + /// Among the classes an `isinstance` call tests against: its second argument, or an element + /// of a tuple there. + Isinstance, + /// In a type expression: an annotation, a return or `raises` annotation, a type alias's + /// value, a type parameter's bound or default, or an argument a call takes as a type, such + /// as `cast`'s first. + TypeHint, + /// In the parentheses of a class's header, among its bases or its keywords. + Superclass, + /// Assigning, deleting or otherwise binding the name. + Write, + /// Reading the name's value, which calling it is. + Read, +} + +/// What each of `targets` does with the name it refers to, in the same order. `program` is the +/// program the search that found them ran in. +pub fn usage_kinds( + db: &dyn Db, + program: ProgramFile<'_>, + targets: &[ReferenceTarget], +) -> Vec { + let program = program.program(db); + targets + .iter() + .map(|target| usage_kind(db, ProgramFile::new(db, target.file(), program), target)) + .collect() +} + +fn usage_kind(db: &dyn Db, file: ProgramFile<'_>, target: &ReferenceTarget) -> UsageKind { + // a django template's names are only read, and its text is no python to classify + let template = target + .file() + .path(db) + .as_system_path() + .is_some_and(is_django_template_path); + (!template) + .then(|| syntactic_kind(db, file, target.range())) + .flatten() + .unwrap_or(match target.kind() { + ReferenceKind::Read => UsageKind::Read, + ReferenceKind::Write => UsageKind::Write, + ReferenceKind::Other => UsageKind::Declaration, + }) +} + +/// The group the syntax around `range` puts a reference in, `None` when it is only a read or a +/// write. +fn syntactic_kind(db: &dyn Db, file: ProgramFile<'_>, range: TextRange) -> Option { + let module = parsed_module(db, file.python_file(db)).load(db); + let covering = covering_node(module.syntax().into(), range); + let ancestors: Vec> = covering.ancestors().collect(); + + if ancestors.iter().any(|node| { + matches!( + node, + AnyNodeRef::StmtImport(_) | AnyNodeRef::StmtImportFrom(_) + ) + }) { + return Some(UsageKind::Import); + } + + let model = SemanticModel::new(db, file); + let mut kind = None; + // from the innermost node out, so that the nearest of nested groups wins: a class in a + // `cast` in a base class list is in a type expression + for node in &ancestors { + kind = match node { + AnyNodeRef::ExprCall(call) => call_argument_kind(&model, call, range), + AnyNodeRef::Parameter(parameter) => { + holds(parameter.annotation.as_deref(), range).then_some(UsageKind::TypeHint) + } + AnyNodeRef::StmtFunctionDef(function) => { + let returns = function + .returns + .as_deref() + .filter(|_| !function.is_asserts_return); + (holds(returns, range) || holds(function.raises.as_deref(), range)) + .then_some(UsageKind::TypeHint) + } + AnyNodeRef::StmtAnnAssign(assignment) => { + holds(Some(&*assignment.annotation), range).then_some(UsageKind::TypeHint) + } + AnyNodeRef::StmtTypeAlias(alias) => { + holds(Some(&*alias.value), range).then_some(UsageKind::TypeHint) + } + AnyNodeRef::TypeParamTypeVar(type_param) => (holds(type_param.bound.as_deref(), range) + || holds(type_param.default.as_deref(), range)) + .then_some(UsageKind::TypeHint), + AnyNodeRef::TypeParamParamSpec(type_param) => { + holds(type_param.default.as_deref(), range).then_some(UsageKind::TypeHint) + } + AnyNodeRef::TypeParamTypeVarTuple(type_param) => { + holds(type_param.default.as_deref(), range).then_some(UsageKind::TypeHint) + } + AnyNodeRef::StmtClassDef(class) => class + .arguments + .as_deref() + .is_some_and(|arguments| arguments.range().contains_range(range)) + .then_some(UsageKind::Superclass), + _ => None, + }; + // a statement's groups are all inside it + if kind.is_some() || node.is_statement() { + break; + } + } + kind +} + +/// The group a reference at `range`, inside one of `call`'s arguments or its callee, is in. +fn call_argument_kind( + model: &SemanticModel<'_>, + call: &ast::ExprCall, + range: TextRange, +) -> Option { + let (index, argument) = call + .arguments + .iter_source_order() + .enumerate() + .find(|(_, argument)| argument.range().contains_range(range))?; + if index == 1 && matches!(argument, ast::ArgOrKeyword::Arg(_)) && calls_isinstance(model, call) + { + return Some(UsageKind::Isinstance); + } + (call_argument_forms(model, call).get(index) == Some(&CallArgumentForm::Type)) + .then_some(UsageKind::TypeHint) +} + +fn holds(expression: Option<&ast::Expr>, range: TextRange) -> bool { + expression.is_some_and(|expression| expression.range().contains_range(range)) +} + +#[cfg(test)] +mod tests { + use crate::tests::CursorTest; + use crate::{find_references, usage_kinds}; + use insta::assert_snapshot; + use ruff_db::source::{line_index, source_text}; + + impl CursorTest { + /// each reference to the name at the cursor, the declaration included: its line, its + /// line's text, and what it does with the name + fn usage_kinds(&self) -> String { + let file = self.program_file(self.cursor.file); + let Some(targets) = find_references(&self.db, file, self.cursor.offset, true) else { + return "no references".to_string(); + }; + let kinds = usage_kinds(&self.db, file, &targets); + targets + .iter() + .zip(kinds) + .map(|(target, kind)| { + let source = source_text(&self.db, target.file()); + let index = line_index(&self.db, target.file()); + let line = index.line_index(target.range().start()); + let text = &source[index.line_range(line, &source)]; + format!("{line} {:?}: {}", kind, text.trim()) + }) + .collect::>() + .join("\n") + } + } + + #[test] + fn each_reference_is_grouped_by_what_it_does() { + let test = CursorTest::builder() + .source( + "shapes.py", + " +class Shape: ... +", + ) + .source( + "main.py", + " +from typing import cast +from shapes import Shape +import shapes + +class Circle(Shape): ... + +class Square(shapes.Shape, metaclass=type): ... + +def total(items: list[Shape], other: 'Shape') -> Shape | None: + for item in items: + if isinstance(item, (Shape, int)): + return cast(Shape, item) + print(Shape, isinstance(Shape, type)) + return Shape() + +type Shapes = list[Shape] + +def first[T: Shape](items: list[T]) -> T: + return items[0] + +Shape = Circle +del Shape +", + ) + .build(); + assert_snapshot!(test.usage_kinds(), @" + 2 Declaration: class Shape: ... + 3 Import: from shapes import Shape + 6 Superclass: class Circle(Shape): ... + 8 Superclass: class Square(shapes.Shape, metaclass=type): ... + 10 TypeHint: def total(items: list[Shape], other: 'Shape') -> Shape | None: + 10 TypeHint: def total(items: list[Shape], other: 'Shape') -> Shape | None: + 10 TypeHint: def total(items: list[Shape], other: 'Shape') -> Shape | None: + 12 Isinstance: if isinstance(item, (Shape, int)): + 13 TypeHint: return cast(Shape, item) + 14 Read: print(Shape, isinstance(Shape, type)) + 14 Read: print(Shape, isinstance(Shape, type)) + 15 Read: return Shape() + 17 TypeHint: type Shapes = list[Shape] + 19 TypeHint: def first[T: Shape](items: list[T]) -> T: + 22 Write: Shape = Circle + 23 Write: del Shape + "); + } +} diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index fcc9d0be18..60f0b9fa10 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -1828,6 +1828,15 @@ fn argument_form_from_binding<'db>( CallArgumentForm::Unknown } +/// Whether `call` calls python's `isinstance`, by whatever name it is reached. +pub fn calls_isinstance(model: &SemanticModel<'_>, call: &ast::ExprCall) -> bool { + matches!( + call.func.inferred_type(model), + Some(Type::FunctionLiteral(function)) + if function.is_known(model.db(), KnownFunction::IsInstance) + ) +} + /// Returns the form of each call-site argument in source order. /// /// `CallArgumentForm::Unknown` indicates that an argument is unmatched or its form cannot be diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 465cb1f568..998fa990b1 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -86,6 +86,11 @@ pub(super) fn request(req: server::Request) -> Task { requests::ReferencesRequestHandler::METHOD => background_document_request_task::< requests::ReferencesRequestHandler, >(req, BackgroundSchedule::Worker), + requests::ClassifiedReferencesRequestHandler::METHOD => background_document_request_task::< + requests::ClassifiedReferencesRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::DocumentHighlightRequestHandler::METHOD => background_document_request_task::< requests::DocumentHighlightRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index c8edf12e2f..adf40925e4 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -18,6 +18,7 @@ mod call_hierarchy_incoming_calls; mod call_hierarchy_outgoing_calls; mod change_signature; mod check_workspace; +mod classified_references; mod code_action; mod code_action_resolve; mod code_lens; @@ -57,7 +58,7 @@ mod pasted_imports; mod prepare_call_hierarchy; mod prepare_rename; mod prepare_type_hierarchy; -mod references; +pub(super) mod references; mod rename; mod run_modules; mod selection_range; @@ -87,6 +88,7 @@ pub(super) use change_signature::{ }; pub(super) use check_workspace::CheckWorkspaceRequestHandler; pub(crate) use check_workspace::SuspendedWorkspaceRequestKind; +pub(super) use classified_references::ClassifiedReferencesRequestHandler; pub(super) use code_action::CodeActionRequestHandler; pub(super) use code_action_resolve::CodeActionResolveRequestHandler; pub(super) use code_lens::CodeLensRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/classified_references.rs b/crates/ty_server/src/server/api/requests/classified_references.rs new file mode 100644 index 0000000000..312799fdad --- /dev/null +++ b/crates/ty_server/src/server/api/requests/classified_references.rs @@ -0,0 +1,131 @@ +//! `by/references`: `textDocument/references`, each reference with what it does with the name +//! +//! what an editor's *find usages* groups usages by: `import`, `isinstance`, `typeHint`, +//! `superclass`, `write`, `read` and `declaration`, the groups PyCharm's Python sorts into. a +//! `Location` has nowhere to say it, so this is a request of its own, answering exactly the +//! references `textDocument/references` does, in the same order. see [`ty_ide::usage_kinds`] for +//! what each kind is + +use std::borrow::Cow; + +use lsp_types::{LspRequestMethod, MessageDirection, Range, ReferenceParams, Request, Uri}; +use ty_ide::{UsageKind, usage_kinds}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::ToLink; +use crate::server::api::requests::references::references_at; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum ClassifiedReferencesRequest {} + +impl Request for ClassifiedReferencesRequest { + type Params = ReferenceParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/references"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// one reference, where `textDocument/references` would put it, and what it does with the name +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClassifiedReference { + uri: Uri, + range: Range, + kind: Kind, +} + +/// what a reference does with the name, the first of these that holds +#[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Kind { + /// in an `import` or `from ... import` statement + Import, + /// among the classes an `isinstance` call tests against + Isinstance, + /// in a type expression: an annotation, a type alias's value, a type parameter's bound, or + /// an argument a call takes as a type, `cast`'s first + TypeHint, + /// in the parentheses of a class's header + Superclass, + /// binding the name: an assignment, a `del`, a `for` target + Write, + /// reading the name, calling it among the ways + Read, + /// the name where it is declared, with `includeDeclaration` + Declaration, +} + +impl From for Kind { + fn from(kind: UsageKind) -> Self { + match kind { + UsageKind::Import => Kind::Import, + UsageKind::Isinstance => Kind::Isinstance, + UsageKind::TypeHint => Kind::TypeHint, + UsageKind::Superclass => Kind::Superclass, + UsageKind::Write => Kind::Write, + UsageKind::Read => Kind::Read, + UsageKind::Declaration => Kind::Declaration, + } + } +} + +pub(crate) struct ClassifiedReferencesRequestHandler; + +impl RequestHandler for ClassifiedReferencesRequestHandler { + type RequestType = ClassifiedReferencesRequest; +} + +impl BackgroundDocumentRequestHandler for ClassifiedReferencesRequestHandler { + fn document_uri(params: &ReferenceParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document_position_params.text_document.uri) + } + + /// `null` where `textDocument/references` answers `null`: when language services are off, + /// and where there is no name to search for + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: ReferenceParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(targets) = references_at( + db, + snapshot, + params.text_document_position_params.position, + params.context.include_declaration, + ) else { + return Ok(None); + }; + let kinds = usage_kinds(db, db.program_file(file), &targets); + + Ok(Some( + targets + .into_iter() + .zip(kinds) + .filter_map(|(target, kind)| { + let location = target.to_location(db, snapshot.encoding())?; + Some(ClassifiedReference { + uri: location.uri, + range: location.range, + kind: kind.into(), + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for ClassifiedReferencesRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/references.rs b/crates/ty_server/src/server/api/requests/references.rs index 4bce3753ad..299a55dbe5 100644 --- a/crates/ty_server/src/server/api/requests/references.rs +++ b/crates/ty_server/src/server/api/requests/references.rs @@ -1,8 +1,8 @@ use std::borrow::Cow; use lsp_types::ReferencesRequest; -use lsp_types::{Location, ReferenceParams, Uri}; -use ty_ide::{django_references, find_references}; +use lsp_types::{Location, Position, ReferenceParams, Uri}; +use ty_ide::{ReferenceTarget, django_references, find_references}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{PositionExt, ToLink}; @@ -36,34 +36,15 @@ impl BackgroundDocumentRequestHandler for ReferencesRequestHandler { return Ok(None); } - let Some(file) = snapshot.document().to_notebook_or_file(db) else { - return Ok(None); - }; - - let Some(offset) = params.text_document_position_params.position.to_text_size( + let Some(references_result) = references_at( db, - file, - snapshot.document().uri(), - snapshot.encoding(), + snapshot, + params.text_document_position_params.position, + params.context.include_declaration, ) else { return Ok(None); }; - let include_declaration = params.context.include_declaration; - let template = snapshot.is_django_template(); - - // a python symbol's references are found exactly as they were; the django - // names a module writes as plain strings are what is left over once it - // declines - let found = (!template) - .then(|| find_references(db, db.program_file(file), offset, include_declaration)) - .flatten() - .or_else(|| django_references(db, file, offset, include_declaration, template)); - - let Some(references_result) = found else { - return Ok(None); - }; - let locations: Vec<_> = references_result .into_iter() .filter_map(|target| target.to_location(db, snapshot.encoding())) @@ -74,3 +55,24 @@ impl BackgroundDocumentRequestHandler for ReferencesRequestHandler { } impl RetriableRequestHandler for ReferencesRequestHandler {} + +/// what `textDocument/references` finds from `position` in the snapshot's document, `None` where +/// there is no name to search for. `by/references` answers the same references +pub(crate) fn references_at( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + position: Position, + include_declaration: bool, +) -> Option> { + let file = snapshot.document().to_notebook_or_file(db)?; + let offset = position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding())?; + let template = snapshot.is_django_template(); + + // a python symbol's references are found exactly as they were; the django + // names a module writes as plain strings are what is left over once it + // declines + (!template) + .then(|| find_references(db, db.program_file(file), offset, include_declaration)) + .flatten() + .or_else(|| django_references(db, file, offset, include_declaration, template)) +} diff --git a/crates/ty_server/tests/e2e/classified_references.rs b/crates/ty_server/tests/e2e/classified_references.rs new file mode 100644 index 0000000000..74c62db2b5 --- /dev/null +++ b/crates/ty_server/tests/e2e/classified_references.rs @@ -0,0 +1,118 @@ +//! `by/references` over the wire +//! +//! what each kind is, is `ty_ide`'s and its unit tests cover it. what these cover is the contract +//! a client reads: the references `textDocument/references` answers, in its order, each with its +//! kind + +use anyhow::Result; +use lsp_types::{ + LspRequestMethod, MessageDirection, PartialResultParams, Position, ReferenceContext, + ReferenceParams, ReferencesRequest, Request, TextDocumentIdentifier, + TextDocumentPositionParams, WorkDoneProgressParams, +}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::TestServerBuilder; + +/// the request as a client sends it, in json throughout +enum ClassifiedReferences {} + +impl Request for ClassifiedReferences { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/references"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const SHAPES: &str = "\ +class Shape: ... +"; + +const MAIN: &str = "\ +from shapes import Shape + +class Circle(Shape): ... + +def first(shapes: list[Shape]) -> Shape: + if isinstance(shapes[0], Shape): + return shapes[0] + return Shape() + +Shape = Circle +"; + +#[test] +fn each_reference_comes_with_what_it_does() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.by"), SHAPES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.by"), SHAPES, 1); + let uri = server.file_uri(SystemPath::new("src/shapes.by")); + let position = Position::new(0, 7); + + let classified = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": position, + "context": { "includeDeclaration": true }, + })) + .expect("references of a class"); + let classified = classified.as_array().expect("a list"); + let kinds: Vec<_> = classified + .iter() + .map(|reference| { + format!( + "{} {}", + reference["range"]["start"]["line"], + reference["kind"].as_str().unwrap_or_default() + ) + }) + .collect(); + assert_eq!( + kinds, + [ + "0 declaration", + "0 import", + "2 superclass", + "4 typeHint", + "4 typeHint", + "5 isinstance", + "7 read", + "9 write", + ] + ); + + // the same places `textDocument/references` goes to, in its order + let locations = server + .send_request_await::(ReferenceParams { + text_document_position_params: TextDocumentPositionParams { + text_document: TextDocumentIdentifier { uri }, + position, + }, + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + context: ReferenceContext { + include_declaration: true, + }, + }) + .unwrap_or_default(); + let places: Vec<_> = classified + .iter() + .map( + |reference| serde_json::json!({ "uri": reference["uri"], "range": reference["range"] }), + ) + .collect(); + assert_eq!( + places, + serde_json::to_value(&locations)? + .as_array() + .cloned() + .unwrap_or_default() + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 2c376d5b35..f9d0f1fe5e 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -33,6 +33,7 @@ mod build_trees; mod call_hierarchy; mod change_signature; mod check_workspace; +mod classified_references; mod closed_documents; mod code_actions; mod commands; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 1006921fee..7659d68592 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1071,6 +1071,37 @@ name. a called class is the class, not the `__init__` the call runs, and a decorated `def` is the `def`. the answer is `null` where there is no name, and for a name whose definitions declare different things +## what each usage does + +an editor's *find usages* groups the usages of a name by what each does with +it, as PyCharm's Python groups them: the imports of a class apart from its uses +as a base and in annotations. a `Location` has nowhere to say that, so +`by/references` takes what `textDocument/references` takes, a position and +`context.includeDeclaration`, and answers the same references in the same +order, each with its `kind`: + +```json +[ + { "uri": "file:///src/shapes/util.by", "range": { "start": { "line": 0, "character": 31 }, "end": { "line": 0, "character": 36 } }, "kind": "import" }, + { "uri": "file:///src/shapes/util.by", "range": { "start": { "line": 3, "character": 25 }, "end": { "line": 3, "character": 30 } }, "kind": "typeHint" } +] +``` + +| kind | the reference is | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `import` | in an `import` or `from ... import` statement | +| `isinstance` | among the classes an `isinstance` call tests against: its second argument, or in a tuple there | +| `typeHint` | in a type expression: an annotation, a return or `raises` annotation, a type alias's value, a type parameter's bound or default, or an argument a call takes as a type, such as `cast`'s first | +| `superclass` | in the parentheses of a class's header, a base or a keyword such as `metaclass=` | +| `write` | binding the name: an assignment, an augmented assignment, a `del`, a `for` or `with` target | +| `read` | reading it, which calling it is | +| `declaration` | the name where it is declared, there with `includeDeclaration` | + +a reference is the first of these that holds, from the top, so an import is an +`import` although it binds the name, and of nested groups the nearest wins: a +class in `cast`'s first argument in a base list is a `typeHint`. the answer is +`null` where `textDocument/references` answers `null` + ## type info an editor's *type info* shows the type of an expression, and which expression From a2b17f2b30fb69073b30f3067d1d24ea5b1e4b63 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:55:21 +1000 Subject: [PATCH 042/110] answer by/documentInheritedMembers, the members each class of a document inherits as PyCharm's structure view lists them, every member a class of its method resolution order declares, attributes assigned on self among them, from the first to declare the name and none the class declares, each as its superclass's outline has it with where it is --- crates/ty_ide/src/inherited_members.rs | 265 ++++++++++++++++++ crates/ty_ide/src/lib.rs | 2 + crates/ty_python_semantic/src/lib.rs | 6 +- .../src/types/ide_support.rs | 33 +++ crates/ty_server/src/server/api.rs | 6 + crates/ty_server/src/server/api/requests.rs | 2 + .../requests/document_inherited_members.rs | 160 +++++++++++ .../ty_server/tests/e2e/inherited_members.rs | 121 ++++++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 34 +++ 10 files changed, 627 insertions(+), 3 deletions(-) create mode 100644 crates/ty_ide/src/inherited_members.rs create mode 100644 crates/ty_server/src/server/api/requests/document_inherited_members.rs create mode 100644 crates/ty_server/tests/e2e/inherited_members.rs diff --git a/crates/ty_ide/src/inherited_members.rs b/crates/ty_ide/src/inherited_members.rs new file mode 100644 index 0000000000..2cd3ee102f --- /dev/null +++ b/crates/ty_ide/src/inherited_members.rs @@ -0,0 +1,265 @@ +//! the members each class of a document inherits: what an editor's structure view lists under a +//! class when it shows inherited members +//! +//! ```py +//! class Shape: +//! name: str +//! +//! def __init__(self, name: str) -> None: +//! self.name = name +//! +//! def area(self) -> float: ... +//! +//! def describe(self) -> str: ... +//! +//! class Circle(Shape): +//! def __init__(self, radius: float) -> None: +//! self.radius = radius +//! +//! def area(self) -> float: ... +//! ``` +//! +//! `Circle` inherits `name` and `describe` from `Shape`, and the members of `object` that neither +//! declares: the members each class of its method resolution order declares, as that class's +//! outline lists them, attributes a method assigns on `self` among them. a name is listed once, +//! from the first class in that order to declare it, and not at all when the class declares it +//! itself, as PyCharm's structure view lists them + +use ruff_db::files::File; +use ruff_db::parsed::parsed_module; +use ruff_python_ast::AnyNodeRef; +use ruff_python_ast::find_node::covering_node; +use ruff_python_ast::name::Name; +use ruff_text_size::TextRange; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_python_core::ProgramFile; +use ty_python_semantic::{HasDefinition, SemanticModel, class_definition_mro}; + +use crate::symbols::symbols_for_file; +use crate::{Db, HierarchicalSymbols, SymbolId, SymbolKind, symbol_detail}; + +/// the members a class of the document inherits +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClassInheritedMembers { + /// the class's name where the document declares it + pub name_range: TextRange, + pub members: Vec, +} + +/// a member a class inherits, as the outline of the superclass that declares it lists it +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InheritedMember { + pub name: String, + pub kind: SymbolKind, + /// what the outline shows beside the name: a method's parameters, a nested class's bases + pub detail: Option, + /// the superclass that declares it + pub superclass: Name, + /// the module the superclass is declared in, when a module name reaches it + pub superclass_module: Option, + pub file: File, + /// the whole declaration + pub full_range: TextRange, + /// its name + pub name_range: TextRange, +} + +/// the inherited members of each class `file`'s outline holds, nested classes among them, in +/// source order. a class inheriting nothing but what it declares itself is there with no members +pub fn document_inherited_members( + db: &dyn Db, + file: ProgramFile<'_>, +) -> Vec { + let symbols = symbols_for_file(db, file).to_hierarchical(); + let module = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + let program = file.program(db); + let mut outlines: FxHashMap = FxHashMap::default(); + + let mut classes = Vec::new(); + let mut pending: Vec = symbols.iter().map(|(id, _)| id).collect(); + pending.reverse(); + while let Some(id) = pending.pop() { + let Some(symbol) = symbols.get(id) else { + continue; + }; + let mut children: Vec = symbols.children(id).map(|(child, _)| child).collect(); + children.reverse(); + pending.extend(children); + if symbol.kind != SymbolKind::Class { + continue; + } + let Some(class) = covering_node(module.syntax().into(), symbol.name_range) + .ancestors() + .find_map(|node| match node { + AnyNodeRef::StmtClassDef(class) if class.name.range == symbol.name_range => { + Some(class) + } + _ => None, + }) + else { + continue; + }; + + // what the class declares itself hides what it would inherit under the same name + let mut seen: FxHashSet = symbols + .children(id) + .map(|(_, member)| member.name.to_string()) + .collect(); + let mut members = Vec::new(); + for superclass in class_definition_mro(db, class.definition(&model)) { + let superclass_file = superclass.file.file(db); + let superclass_program_file = ProgramFile::new(db, superclass_file, program); + let flat = symbols_for_file(db, superclass_program_file); + let Some(superclass_id) = flat.iter().find_map(|(id, candidate)| { + (candidate.kind == SymbolKind::Class + && candidate.name_range == superclass.selection_range) + .then_some(id) + }) else { + continue; + }; + let outline = outlines + .entry(superclass_file) + .or_insert_with(|| flat.to_hierarchical()); + let superclass_module = ty_module_resolver::file_to_module(db, superclass.file) + .map(|module| module.name(db).to_string()); + for (_, member) in outline.children(superclass_id) { + if !seen.insert(member.name.to_string()) { + continue; + } + members.push(InheritedMember { + name: member.name.to_string(), + kind: member.kind, + detail: symbol_detail(db, superclass_program_file, &member), + superclass: superclass.name.clone(), + superclass_module: superclass_module.clone(), + file: superclass_file, + full_range: member.full_range, + name_range: member.name_range, + }); + } + } + classes.push(ClassInheritedMembers { + name_range: symbol.name_range, + members, + }); + } + classes +} + +#[cfg(test)] +mod tests { + use crate::tests::CursorTest; + use crate::{SymbolKind, document_inherited_members}; + use insta::assert_snapshot; + use ruff_db::source::{line_index, source_text}; + + impl CursorTest { + /// for each class, its name, then one line per inherited member: its kind, its name, its + /// detail, and the superclass and module it is declared in. `object`'s members are counted + /// rather than listed + fn inherited_members(&self) -> String { + let file = self.program_file(self.cursor.file); + let source = source_text(&self.db, self.cursor.file); + let index = line_index(&self.db, self.cursor.file); + let mut out = Vec::new(); + for class in document_inherited_members(&self.db, file) { + out.push(format!( + "{} (line {})", + &source[class.name_range], + index.line_index(class.name_range.start()) + )); + let mut from_object = 0; + for member in &class.members { + if member.superclass.as_str() == "object" { + from_object += 1; + continue; + } + let kind = match member.kind { + SymbolKind::Method => "method", + SymbolKind::Constructor => "constructor", + SymbolKind::Property => "property", + SymbolKind::Field => "field", + SymbolKind::Class => "class", + _ => "other", + }; + out.push(format!( + " {kind} {}{} <- {} in {}", + member.name, + member.detail.as_deref().unwrap_or_default(), + member.superclass, + member.superclass_module.as_deref().unwrap_or("?"), + )); + } + out.push(format!(" and {from_object} from object")); + } + out.join("\n") + } + } + + #[test] + fn each_class_lists_what_it_inherits_and_does_not_declare() { + let test = CursorTest::builder() + .source( + "shapes.py", + " +class Shape: + name: str + sides = 0 + + def __init__(self, name: str) -> None: + self.name = name + self.label = name.title() + + def area(self) -> float: + return 0.0 + + @property + def describe(self) -> str: + return self.name + + class Meta: ... +", + ) + .source( + "main.py", + " +from shapes import Shape + +class Circle(Shape): + def __init__(self, radius: float) -> None: + super().__init__('circle') + self.radius = radius + + def area(self) -> float: + return self.radius + +class Unit(Circle): + sides = 1 + + class Inner: ... +", + ) + .build(); + assert_snapshot!(test.inherited_members(), @" + Circle (line 4) + field name <- Shape in shapes + field sides <- Shape in shapes + property describe(self) -> str <- Shape in shapes + class Meta <- Shape in shapes + field label <- Shape in shapes + and 22 from object + Unit (line 12) + constructor __init__(self, radius: float) -> None <- Circle in main + method area(self) -> float <- Circle in main + field radius <- Circle in main + field name <- Shape in shapes + property describe(self) -> str <- Shape in shapes + class Meta <- Shape in shapes + field label <- Shape in shapes + and 22 from object + Inner (line 15) + and 23 from object + "); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index e96f4decef..1c5da00e8c 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -33,6 +33,7 @@ mod hover; mod import_order; #[cfg(test)] mod importer; +mod inherited_members; mod injection; mod inlay_hints; mod markup; @@ -95,6 +96,7 @@ pub use goto::{goto_declaration, goto_definition, goto_type_definition}; pub use goto_implementation::goto_implementation; pub use hints::{Hint, HintKind, hints}; pub use hover::hover; +pub use inherited_members::{ClassInheritedMembers, InheritedMember, document_inherited_members}; pub use injection::{Injection, InjectionOrigin, Injections, injections}; pub use inlay_hints::{ InlayHintKind, InlayHintLabel, InlayHintSettings, InlayHintTextEdit, inlay_hints, diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index d0ab70415c..bf76a23923 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -62,11 +62,11 @@ pub use types::extensions::{ }; pub use types::ide_support::{ ImplementationsFinder, ImportAliasResolution, OverridableMember, ResolvedDefinition, - TypeHierarchyClass, class_bases, definitions_for_attribute, definitions_for_bin_op, - definitions_for_case_name, definitions_for_context_sensitive_name, + TypeHierarchyClass, class_bases, class_definition_mro, definitions_for_attribute, + definitions_for_bin_op, definitions_for_case_name, definitions_for_context_sensitive_name, definitions_for_django_lookup_root, definitions_for_imported_symbol, definitions_for_name, definitions_for_unary_op, instance_member_names, is_within_classes, map_stub_definition, - type_hierarchy_prepare, type_hierarchy_subtypes, type_hierarchy_supertypes, + type_hierarchy_mro, type_hierarchy_prepare, type_hierarchy_subtypes, type_hierarchy_supertypes, }; pub use types::implicit_names::implicit_names; pub use types::reified_infer::{ diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 60f0b9fa10..7e419adab1 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -2510,6 +2510,39 @@ pub fn type_hierarchy_supertypes<'db>( supertypes } +/// the classes the class `ty` names inherits from, in its method resolution order: its bases +/// and theirs as python looks a member up through them, `object` last. the class itself is not +/// among them, and neither is a base that is no class, such as one of `Unknown` type +/// +/// when the type given doesn't correspond to a class literal, then this always +/// returns an empty sequence +pub fn type_hierarchy_mro<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, +) -> Vec> { + extract_class_literal(db, env, ty).map_or_else(Vec::new, |class| class_mro(db, class)) +} + +/// [`type_hierarchy_mro`] of the class a `class` statement makes, as it is written rather than +/// as its decorators return it +pub fn class_definition_mro<'db>( + db: &'db dyn Db, + definition: Definition<'db>, +) -> Vec> { + original_class_type(db, definition).map_or_else(Vec::new, |class| class_mro(db, class)) +} + +fn class_mro<'db>(db: &'db dyn Db, class: ClassLiteral<'db>) -> Vec> { + class + .default_specialization(db) + .iter_mro(db) + .skip(1) + .filter_map(ClassBase::into_class) + .map(|class| class_literal_to_hierarchy_info(db, class.class_literal(db))) + .collect() +} + /// Get the direct subtypes of the class given in `modules`. /// /// When the type given doesn't correspond to a class literal, then this always diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 998fa990b1..e91878b019 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -91,6 +91,12 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::DocumentInheritedMembersRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::Worker, + ) + } requests::DocumentHighlightRequestHandler::METHOD => background_document_request_task::< requests::DocumentHighlightRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index adf40925e4..23992685e0 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -31,6 +31,7 @@ mod detached_transpile; mod diagnostic; mod doc_highlights; mod docstring_stub; +mod document_inherited_members; mod document_super_members; mod document_symbols; mod document_usage_counts; @@ -100,6 +101,7 @@ pub(super) use declaration::DeclarationRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; pub(super) use docstring_stub::DocstringStubRequestHandler; +pub(super) use document_inherited_members::DocumentInheritedMembersRequestHandler; pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use document_usage_counts::DocumentUsageCountsRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/document_inherited_members.rs b/crates/ty_server/src/server/api/requests/document_inherited_members.rs new file mode 100644 index 0000000000..764a14a136 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/document_inherited_members.rs @@ -0,0 +1,160 @@ +//! `by/documentInheritedMembers`: the members each class of a document inherits +//! +//! what an editor's structure view lists under a class when it shows inherited members: every +//! member a class of its method resolution order declares, methods, properties, nested classes and +//! attributes alike, from the first class to declare the name, less the names the class declares +//! itself. each member is as that superclass's outline (`textDocument/documentSymbol`) has it, so +//! that a client draws it as it draws the class's own. see [`ty_ide::document_inherited_members`] +//! +//! one document at a time, whole, like the other document requests: a client asks once for all the +//! classes it shows. the answer depends on every file the classes inherit from, and the server +//! announces no change to it: a client asks again after any document of the workspace changes + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Range, Request, SymbolKind, TextDocumentIdentifier, Uri, +}; +use ruff_db::files::FileRange; +use ty_ide::document_inherited_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{FileRangeExt, ToRangeExt}; +use crate::server::api::symbols::convert_symbol_kind; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DocumentInheritedMembersRequest {} + +impl Request for DocumentInheritedMembersRequest { + type Params = DocumentInheritedMembersParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = + LspRequestMethod::Custom("by/documentInheritedMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DocumentInheritedMembersParams { + text_document: TextDocumentIdentifier, +} + +/// a class of the document, and what it inherits +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClassInheritedMembers { + /// the class's name where the document declares it: the `selectionRange` of its document + /// symbol + selection_range: Range, + /// in the class's method resolution order, then in each superclass's source order + members: Vec, +} + +/// a member a class inherits, as the outline of the superclass that declares it has it +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct InheritedMember { + name: String, + /// as `textDocument/documentSymbol` gives it: method, constructor, property, field, class + kind: SymbolKind, + /// as `textDocument/documentSymbol` gives it: a method's parameters, a class's bases + #[serde(skip_serializing_if = "Option::is_none")] + detail: Option, + /// the superclass that declares it + container_name: String, + /// the module that superclass is declared in, `shapes.base`, `builtins` for `object`. absent + /// for a superclass in a file no module is imported from + #[serde(skip_serializing_if = "Option::is_none")] + container_module: Option, + /// the file the superclass declares it in + uri: Uri, + /// the whole declaration + range: Range, + /// its name, which is where to go + selection_range: Range, +} + +pub(crate) struct DocumentInheritedMembersRequestHandler; + +impl RequestHandler for DocumentInheritedMembersRequestHandler { + type RequestType = DocumentInheritedMembersRequest; +} + +impl BackgroundDocumentRequestHandler for DocumentInheritedMembersRequestHandler { + // what the project's files say, which is the same whether this one is a buffer or the file on + // disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &DocumentInheritedMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, or for a notebook, whose ranges are per cell, or a + /// template. otherwise every class of the document's outline, nested ones among them, in + /// source order, each with the members it inherits, none for a class that inherits nothing + /// it does not declare itself + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + _params: DocumentInheritedMembersParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let encoding = snapshot.encoding(); + + Ok(Some( + document_inherited_members(db, db.program_file(file)) + .into_iter() + .filter_map(|class| { + let selection_range = class + .name_range + .to_lsp_range(db, file, encoding)? + .local_range(); + let members = class + .members + .into_iter() + .filter_map(|member| { + let location = FileRange::new(member.file, member.full_range) + .to_lsp_range(db, encoding)? + .into_location()?; + let selection_range = FileRange::new(member.file, member.name_range) + .to_lsp_range(db, encoding)? + .local_range(); + Some(InheritedMember { + name: member.name, + kind: convert_symbol_kind(member.kind), + detail: member.detail, + container_name: member.superclass.to_string(), + container_module: member.superclass_module, + uri: location.uri, + range: location.range, + selection_range, + }) + }) + .collect(); + Some(ClassInheritedMembers { + selection_range, + members, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for DocumentInheritedMembersRequestHandler {} diff --git a/crates/ty_server/tests/e2e/inherited_members.rs b/crates/ty_server/tests/e2e/inherited_members.rs new file mode 100644 index 0000000000..168155bbd3 --- /dev/null +++ b/crates/ty_server/tests/e2e/inherited_members.rs @@ -0,0 +1,121 @@ +//! `by/documentInheritedMembers` over the wire +//! +//! which members a class inherits is `ty_ide`'s and its unit tests cover it. what these cover is +//! the contract a client reads: each class by its name's range in the document, and each member +//! as its superclass's outline has it, with where it is + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::TestServerBuilder; + +/// the request as a client sends it, in json throughout +enum DocumentInheritedMembers {} + +impl Request for DocumentInheritedMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = + LspRequestMethod::Custom("by/documentInheritedMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const SHAPES: &str = "\ +class Shape: + def __init__(self, name: str) -> None: + self.name = name + + def describe(self) -> str: + return self.name +"; + +const MAIN: &str = "\ +from shapes import Shape + +class Circle(Shape): + def __init__(self) -> None: + super().__init__('circle') +"; + +#[test] +fn each_class_comes_with_its_superclasses_members() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.by"), SHAPES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + let uri = server.file_uri(SystemPath::new("src/main.by")); + let shapes = server.file_uri(SystemPath::new("src/shapes.by")); + + let answer = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + })) + .expect("the classes of a document"); + let classes = answer.as_array().expect("a list"); + assert_eq!(classes.len(), 1); + assert_eq!( + classes[0]["selectionRange"], + serde_json::json!({ + "start": { "line": 2, "character": 6 }, + "end": { "line": 2, "character": 12 }, + }) + ); + let members = classes[0]["members"].as_array().expect("members"); + // `Shape`'s own, then `object`'s; `__init__` is the class's own + let shape_members: Vec<_> = members + .iter() + .filter(|member| member["containerName"] == "Shape") + .collect(); + assert_eq!( + shape_members, + [ + &serde_json::json!({ + "name": "describe", + "kind": 6, + "detail": "(self) -> str", + "containerName": "Shape", + "containerModule": "shapes", + "uri": shapes, + "range": { + "start": { "line": 4, "character": 4 }, + "end": { "line": 5, "character": 24 }, + }, + "selectionRange": { + "start": { "line": 4, "character": 8 }, + "end": { "line": 4, "character": 16 }, + }, + }), + &serde_json::json!({ + "name": "name", + "kind": 8, + "containerName": "Shape", + "containerModule": "shapes", + "uri": shapes, + "range": { + "start": { "line": 2, "character": 8 }, + "end": { "line": 2, "character": 24 }, + }, + "selectionRange": { + "start": { "line": 2, "character": 13 }, + "end": { "line": 2, "character": 17 }, + }, + }), + ] + ); + assert!( + members + .iter() + .skip(shape_members.len()) + .all(|member| member["containerName"] == "object" + && member["containerModule"] == "builtins" + && member["name"] != "__init__"), + "{members:#?}" + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index f9d0f1fe5e..b6ac9e4f5f 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -52,6 +52,7 @@ mod folding_range; mod goto_definition; mod hover; mod implementation; +mod inherited_members; mod initialize; mod injections; mod inlay_hints; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 7659d68592..e3457b9bf4 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -403,6 +403,40 @@ from the syntax tree rather than copied from the source: a function's parameters and return annotation, `(self, radius: float) -> None`, and a class's bases, `(Shape)` +### inherited members + +a structure view that shows a class's inherited members asks +`by/documentInheritedMembers` with the document, once for all its classes. the +answer is every class of the outline, nested ones too, in source order, each by +the `selectionRange` of its name and with the `members` it inherits: what each +class of its method resolution order declares, as that class's outline has it, +attributes a method assigns on `self` among them, from the first class to +declare the name, and none the class declares itself. `object`'s are there, +last. each member is a document symbol of its superclass's file, with its +`name`, `kind`, `detail`, `range` and `selectionRange`, the `uri` of that file, +and the superclass as `containerName` and its module as `containerModule`: + +```json +[ + { + "selectionRange": { "start": { "line": 27, "character": 6 }, "end": { "line": 27, "character": 12 } }, + "members": [ + { + "name": "describe", "kind": 6, "detail": "(self) -> str", + "containerName": "Shape", "containerModule": "shapes.base", + "uri": "file:///src/shapes/base.by", + "range": { "start": { "line": 23, "character": 4 }, "end": { "line": 24, "character": 57 } }, + "selectionRange": { "start": { "line": 23, "character": 8 }, "end": { "line": 23, "character": 16 } } + } + ] + } +] +``` + +the answer depends on every file the classes inherit from, and the server +announces no change to it: a client asks again after any document of the +workspace changes. it is `null` for a notebook and a template + ## workspace symbols `workspace/symbol` gives each python symbol the module and the classes it is From 840c45a407c1be4acdb5f9121ab544d9823e2f2e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:00:18 +1000 Subject: [PATCH 043/110] type in by/expressionTypes the tuple of targets a value is unpacked into, kind, _, size as (str, str, str), the tuple of what each target is bound to with a starred one as its list's elements, rather than leaving it out for having no type of its own --- crates/ty_ide/src/expression_types.rs | 46 +++++++++++---- .../src/types/ide_support.rs | 56 +++++++++++++++++++ docs/basedpython/features/editor.md | 7 ++- 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/crates/ty_ide/src/expression_types.rs b/crates/ty_ide/src/expression_types.rs index 423278a5df..c8168d400d 100644 --- a/crates/ty_ide/src/expression_types.rs +++ b/crates/ty_ide/src/expression_types.rs @@ -1,33 +1,36 @@ -//! The expressions a range is in, each with its type: what an editor's *type info* offers. +//! the expressions a range is in, each with its type: what an editor's *type info* offers //! //! ```py //! total = total_area(shapes) * 2 //! ``` //! -//! On `shapes` the answer is `shapes`, `total_area(shapes)` and `total_area(shapes) * 2`, from the -//! innermost out, each with the type ty inferred for it: `list[Shape]`, `float`, `float`. On a -//! selection of `total_area(shapes)` it starts from that call. +//! on `shapes` the answer is `shapes`, `total_area(shapes)` and `total_area(shapes) * 2`, from the +//! innermost out, each with the type ty inferred for it: `list[Shape]`, `float`, `float`. on a +//! selection of `total_area(shapes)` it starts from that call. On `kind` in +//! `kind, _, size = spec.partition(":")` it is `kind` and the tuple of targets around it, typed as +//! the tuple of what each target is bound to. use ruff_db::parsed::parsed_module; use ruff_python_ast::find_node::covering_node; use ruff_text_size::{Ranged, TextRange}; use ty_python_core::ProgramFile; +use ty_python_semantic::types::ide_support::unpacking_target_type; use ty_python_semantic::{HasType, SemanticModel, with_display_for_file}; use crate::Db; -/// An expression and its type. +/// an expression and its type #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExpressionType { pub range: TextRange, - /// The type as the file would spell it: `1` in a basedpython file, `Literal[1]` in python. + /// the type as the file would spell it: `1` in a basedpython file, `Literal[1]` in python pub ty: String, } -/// The expressions of the statement `range` is in that contain all of it, innermost first, each -/// with its type. An expression ty has no type for, such as one inside a string annotation, is +/// the expressions of the statement `range` is in that contain all of it, innermost first, each +/// with its type. an expression ty has no type for, such as one inside a string annotation, is /// left out, and so is everything outside the statement: a statement has no type, and what -/// contains it is not an expression. +/// contains it is not an expression pub fn expression_types( db: &dyn Db, file: ProgramFile<'_>, @@ -53,7 +56,11 @@ pub fn expression_types( { continue; } - let Some(ty) = expression.inferred_type(&model) else { + // a tuple of targets has no type of its own, but is the tuple of its targets' + let Some(ty) = expression + .inferred_type(&model) + .or_else(|| unpacking_target_type(&model, expression)) + else { continue; }; found.push(ExpressionType { @@ -103,6 +110,25 @@ def main(shapes: list[int]) -> None: "); } + #[test] + fn a_tuple_of_targets_is_the_tuple_of_what_they_are_bound_to() { + let test = CursorTest::builder() + .source( + "main.py", + " +def main(text: str, numbers: list[int]) -> None: + (kind, _, size), (first, *rest) = text.partition(':'), numbers +", + ) + .build(); + + assert_snapshot!(types(&test, TextRange::empty(test.cursor.offset)), @" + size : str + (kind, _, size) : tuple[str, str, str] + (kind, _, size), (first, *rest) : tuple[tuple[str, str, str], tuple[int, *tuple[int, ...]]] + "); + } + #[test] fn a_selection_starts_from_the_expression_it_spans() { let test = CursorTest::builder() diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 7e419adab1..b53d5916d8 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -44,6 +44,7 @@ use crate::types::signatures::{ Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, is_placeholder_default, }; +use crate::types::tuple::TupleType; use crate::types::{ BoundTypeVarInstance, CallDunderError, ClassBase, ClassLiteral, ClassType, KnownClass, KnownFunction, KnownInstanceType, KnownUnion, PropertyAccessorRole, SpecialFormType, @@ -1828,6 +1829,61 @@ fn argument_form_from_binding<'db>( CallArgumentForm::Unknown } +/// The type of a tuple or a list of targets a value is unpacked into: `kind, _, size` in +/// `kind, _, size = spec.partition(":")` is `tuple[str, str, str]`, the tuple of what each of +/// its targets is bound to. A starred target, bound to a list of what it takes, stands for any +/// number of elements of that list's element type, so `first, *rest` over a `list[int]` is +/// `tuple[int, *tuple[int, ...]]`. +/// +/// ty records no type for such a target, since it is not evaluated: the value is, and each target +/// is bound to a part of it. `None` for anything else, and for a target part of which has no +/// type. +pub fn unpacking_target_type<'db>( + model: &SemanticModel<'db>, + target: ast::ExprRef<'_>, +) -> Option> { + let (ast::ExprRef::Tuple(ast::ExprTuple { + elts: elements, + ctx: ast::ExprContext::Store, + .. + }) + | ast::ExprRef::List(ast::ExprList { + elts: elements, + ctx: ast::ExprContext::Store, + .. + })) = target + else { + return None; + }; + let db = model.db(); + let env = model.program_environment(); + let mut prefix = Vec::new(); + let mut variable = None; + let mut suffix = Vec::new(); + for element in elements { + if let ast::Expr::Starred(starred) = element { + // a second starred target is a syntax error, and binds nothing + if variable.is_some() { + return None; + } + let list = starred.value.inferred_type(model)?; + variable = Some(list.iterate(db, &env).homogeneous_element_type(db, &env)); + continue; + } + let element = ast::ExprRef::from(element); + let ty = unpacking_target_type(model, element).or_else(|| element.inferred_type(model))?; + if variable.is_some() { + suffix.push(ty); + } else { + prefix.push(ty); + } + } + Some(match variable { + Some(variable) => Type::tuple(TupleType::mixed(db, &env, prefix, variable, suffix)), + None => Type::heterogeneous_tuple(db, &env, prefix), + }) +} + /// Whether `call` calls python's `isinstance`, by whatever name it is reached. pub fn calls_isinstance(model: &SemanticModel<'_>, call: &ast::ExprCall) -> bool { matches!( diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index e3457b9bf4..9a90d4b30b 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1167,8 +1167,11 @@ language spells it: ``` with a selection the first is the smallest expression containing it, which is -the selected one when the selection is exactly an expression. an expression ty -has no type for, such as one inside a string annotation, is left out. the answer +the selected one when the selection is exactly an expression. the targets a +value is unpacked into, `kind, _, size` in `kind, _, size = spec.partition(":")`, +are the tuple of what each is bound to, `(str, str, str)`, a starred one standing +for any number of its list's elements. an expression ty has no type for, such as +one inside a string annotation, is left out. the answer is an empty list where the range is in no expression, a `def`'s name or a keyword, and `null` when language services are off From a9b4edc87f7561d396abdf4673f615ffe5d9e7d5 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:08:52 +1000 Subject: [PATCH 044/110] answer every completion with the textEdit it replaces, what is typed of the name before the cursor, car in f"{car, or all of a string literal before it, so a client never filters by its own idea of the word, which inside an f-string is the whole string token, and write the inserted text rather than a label carrying its import --- crates/ty_ide/src/completion.rs | 39 +- .../src/server/api/requests/completion.rs | 13 +- crates/ty_server/tests/e2e/completions.rs | 537 +++++++++++++++++- .../snapshots/e2e__notebook__auto_import.snap | 52 ++ .../e2e__notebook__auto_import_docstring.snap | 52 ++ ...2e__notebook__auto_import_from_future.snap | 52 ++ .../e2e__notebook__auto_import_same_cell.snap | 52 ++ .../ty_server/tests/e2e/workspace_folders.rs | 15 +- docs/basedpython/features/editor.md | 7 + 9 files changed, 792 insertions(+), 27 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 9d073fa8f3..f0eac1f083 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -73,7 +73,8 @@ pub fn completion<'db>( if let Some(completions) = format_spec_completions(db, env, program_file, &model, &source, &context.cursor) { - return completions.into_completions(); + // a clause is written at the cursor, after what the spec has already + return replacing(completions.into_completions(), TextRange::empty(offset)); } if !matches!(context.kind, ContextKind::Keywords(_)) && context.cursor.is_in_string() { @@ -105,7 +106,18 @@ pub fn completion<'db>( &mut completions, ); - return completions.into_completions(); + // a literal's value stands for all the string holds before the cursor, not the word + // before it: `"slow-mo` completes to `"slow-mode` + let typed = string_expr + .value + .iter() + .find(|part| part.range().contains_inclusive(offset)) + .map(|part| part.content_range().start()) + .filter(|start| *start <= offset) + .map_or(TextRange::empty(offset), |start| { + TextRange::new(start, offset) + }); + return replacing(completions.into_completions(), typed); } let query = UserQuery::fuzzy(context.cursor.typed); @@ -284,7 +296,22 @@ pub fn completion<'db>( }, } - completions.into_completions() + replacing(completions.into_completions(), context.cursor.range) +} + +/// `completions`, each saying what it replaces: `typed` for one that does not say already. +/// +/// A client that is not told works the range out from its own idea of the word before the +/// cursor, which is not python's: inside an f-string's `{car` it is the whole string token, and +/// every name is filtered out against `f"{car`. +fn replacing(completions: Vec>, typed: TextRange) -> Vec> { + completions + .into_iter() + .map(|mut completion| { + completion.replace.get_or_insert(typed); + completion + }) + .collect() } #[derive(Copy, Clone, Debug, Default)] @@ -507,9 +534,9 @@ pub struct Completion<'db> { pub insert_text_format: CompletionInsertTextFormat, /// The range that [`Self::insert`] replaces. /// - /// This is only set when the client's own idea of the word under the - /// cursor would be wrong, as it is for a django template path or a - /// namespaced url name. + /// [`completion`] answers every completion with one: what was typed of + /// the word before the cursor, or more where the completion stands for + /// more, as a django template path or a namespaced url name does. pub replace: Option, /// The text the client should match the user's typing against, when /// [`Self::label`] is not it. diff --git a/crates/ty_server/src/server/api/requests/completion.rs b/crates/ty_server/src/server/api/requests/completion.rs index aa8c89efbe..c656893749 100644 --- a/crates/ty_server/src/server/api/requests/completion.rs +++ b/crates/ty_server/src/server/api/requests/completion.rs @@ -127,6 +127,11 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { .collect(); let label = comp.label().to_string(); + // what accepting it writes, which the label shown for it may say more than + let new_text = comp + .insert + .as_ref() + .map_or_else(|| label.clone(), ToString::to_string); let import_suffix = comp.module_name.and_then(|name| { import_edit.is_some().then(|| format!(" (import {name})")) }); @@ -169,16 +174,16 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { CompletionInsertTextFormat::PlainText => None, CompletionInsertTextFormat::Snippet => Some(InsertTextFormat::Snippet), }; - // A completion that says what it replaces is one the client's own - // idea of the word under the cursor would get wrong, so the range - // is spelled out as an edit rather than left to be guessed. + // every completion says what it replaces: the client's own idea of the + // word before the cursor is not python's, and inside an f-string's + // `{car` it is the whole string token, against which nothing matches let text_edit = comp.replace.and_then(|replace| { let range = replace .to_lsp_range(db, file, snapshot.encoding())? .local_range(); Some(lsp_types::CompletionItemTextEdit::TextEdit(TextEdit { range, - new_text: insert_text.clone().unwrap_or_else(|| label.clone()), + new_text, })) }); diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 56ad18168a..f4abdc0245 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -32,6 +32,19 @@ walktr "kind": 3, "sortText": "0", "insertText": "walktree()", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 6 + } + }, + "newText": "walktree()" + }, "additionalTextEdits": [ { "range": { @@ -146,7 +159,20 @@ complete_parenth "detail": "def complete_parentheses(x: int)", "sortText": "0", "insertText": "complete_parentheses($0)", - "insertTextFormat": 2 + "insertTextFormat": 2, + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses($0)" + } } ] "#); @@ -187,6 +213,19 @@ complete_parenth "sortText": "0", "insertText": "complete_parentheses($0)", "insertTextFormat": 2, + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses($0)" + }, "command": { "title": "Trigger parameter hints", "command": "ty.triggerParameterHints" @@ -232,7 +271,20 @@ complete_parenth "kind": 3, "detail": "def complete_parentheses()", "sortText": "0", - "insertText": "complete_parentheses()" + "insertText": "complete_parentheses()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses()" + } } ] "#); @@ -271,7 +323,20 @@ complete_parenth "label": "complete_parentheses", "kind": 3, "detail": "def complete_parentheses(x: int)", - "sortText": "0" + "sortText": "0", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses" + } } ] "#); @@ -310,7 +375,20 @@ complete_parenth "detail": "def complete_parentheses(x: int)", "sortText": "0", "insertText": "complete_parentheses($0)", - "insertTextFormat": 2 + "insertTextFormat": 2, + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses($0)" + } } ] "#); @@ -347,7 +425,20 @@ complete_parenth "kind": 3, "detail": "def complete_parentheses()", "sortText": "0", - "insertText": "complete_parentheses()" + "insertText": "complete_parentheses()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 16 + } + }, + "newText": "complete_parentheses()" + } } ] "#); @@ -387,6 +478,19 @@ is_typedd "sortText": "0", "insertText": "typing.is_typeddict($0)", "insertTextFormat": 2, + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 8 + } + }, + "newText": "typing.is_typeddict($0)" + }, "command": { "title": "Trigger parameter hints", "command": "ty.triggerParameterHints" @@ -398,6 +502,19 @@ is_typedd "sortText": "1", "insertText": "is_typeddict($0)", "insertTextFormat": 2, + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 8 + } + }, + "newText": "is_typeddict($0)" + }, "additionalTextEdits": [ { "range": { @@ -455,19 +572,58 @@ TypedDi "label": "typing.TypedDict", "kind": 6, "sortText": "0", - "insertText": "typing.TypedDict" + "insertText": "typing.TypedDict", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "typing.TypedDict" + } }, { "label": "typing.is_typeddict", "kind": 3, "sortText": "1", - "insertText": "typing.is_typeddict()" + "insertText": "typing.is_typeddict()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "typing.is_typeddict()" + } }, { "label": "TypedDict (import typing_extensions)", "kind": 6, "sortText": "2", "insertText": "TypedDict", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "TypedDict" + }, "additionalTextEdits": [ { "range": { @@ -489,6 +645,19 @@ TypedDi "kind": 7, "sortText": "3", "insertText": "TypedDictFallback()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "TypedDictFallback()" + }, "additionalTextEdits": [ { "range": { @@ -510,6 +679,19 @@ TypedDi "kind": 3, "sortText": "4", "insertText": "is_typeddict()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "is_typeddict()" + }, "additionalTextEdits": [ { "range": { @@ -531,6 +713,19 @@ TypedDi "kind": 7, "sortText": "5", "insertText": "_FilterConfigurationTypedDict()", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "_FilterConfigurationTypedDict()" + }, "additionalTextEdits": [ { "range": { @@ -552,6 +747,19 @@ TypedDi "kind": 6, "sortText": "6", "insertText": "_FormatterConfigurationTypedDict", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "_FormatterConfigurationTypedDict" + }, "additionalTextEdits": [ { "range": { @@ -573,6 +781,19 @@ TypedDi "kind": 9, "sortText": "7", "insertText": "_typeshed.dbapi", + "textEdit": { + "range": { + "start": { + "line": 2, + "character": 0 + }, + "end": { + "line": 2, + "character": 7 + } + }, + "newText": "_typeshed.dbapi" + }, "additionalTextEdits": [ { "range": { @@ -624,6 +845,19 @@ TypedDi "kind": 6, "sortText": "0", "insertText": "TypedDict", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "TypedDict" + }, "additionalTextEdits": [ { "range": { @@ -645,6 +879,19 @@ TypedDi "kind": 3, "sortText": "1", "insertText": "is_typeddict()", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "is_typeddict()" + }, "additionalTextEdits": [ { "range": { @@ -666,6 +913,19 @@ TypedDi "kind": 6, "sortText": "2", "insertText": "TypedDict", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "TypedDict" + }, "additionalTextEdits": [ { "range": { @@ -687,6 +947,19 @@ TypedDi "kind": 7, "sortText": "3", "insertText": "TypedDictFallback()", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "TypedDictFallback()" + }, "additionalTextEdits": [ { "range": { @@ -708,6 +981,19 @@ TypedDi "kind": 3, "sortText": "4", "insertText": "is_typeddict()", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "is_typeddict()" + }, "additionalTextEdits": [ { "range": { @@ -729,6 +1015,19 @@ TypedDi "kind": 7, "sortText": "5", "insertText": "_FilterConfigurationTypedDict()", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "_FilterConfigurationTypedDict()" + }, "additionalTextEdits": [ { "range": { @@ -750,6 +1049,19 @@ TypedDi "kind": 6, "sortText": "6", "insertText": "_FormatterConfigurationTypedDict", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "_FormatterConfigurationTypedDict" + }, "additionalTextEdits": [ { "range": { @@ -771,6 +1083,19 @@ TypedDi "kind": 9, "sortText": "7", "insertText": "_typeshed.dbapi", + "textEdit": { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 7 + } + }, + "newText": "_typeshed.dbapi" + }, "additionalTextEdits": [ { "range": { @@ -822,7 +1147,20 @@ re.match('', '', fla "kind": 6, "detail": "FlagsType", "sortText": "0", - "insertText": "flags=" + "insertText": "flags=", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 17 + }, + "end": { + "line": 1, + "character": 20 + } + }, + "newText": "flags=" + } }, { "label": "FloatingPointError", @@ -833,7 +1171,20 @@ re.match('', '', fla "value": "Floating-point operation failed.\n" }, "sortText": "1", - "insertText": "FloatingPointError()" + "insertText": "FloatingPointError()", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 17 + }, + "end": { + "line": 1, + "character": 20 + } + }, + "newText": "FloatingPointError()" + } }, { "label": "PythonFinalizationError", @@ -844,7 +1195,20 @@ re.match('', '', fla "value": "Operation blocked during Python finalization.\n" }, "sortText": "2", - "insertText": "PythonFinalizationError()" + "insertText": "PythonFinalizationError()", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 17 + }, + "end": { + "line": 1, + "character": 20 + } + }, + "newText": "PythonFinalizationError()" + } }, { "label": "float", @@ -855,7 +1219,20 @@ re.match('', '', fla "value": "Convert a string or number to a floating-point number, if possible.\n" }, "sortText": "3", - "insertText": "float()" + "insertText": "float()", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 17 + }, + "end": { + "line": 1, + "character": 20 + } + }, + "newText": "float()" + } } ] "#); @@ -895,7 +1272,20 @@ x: Literal[\"apple\"] = \"app\" "kind": 12, "detail": "Literal[\"apple\"]", "sortText": "0", - "insertText": "apple" + "insertText": "apple", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 23 + }, + "end": { + "line": 1, + "character": 26 + } + }, + "newText": "apple" + } } ] "#); @@ -1000,7 +1390,20 @@ x: Literal[\"apple\"] = \"app\" "kind": 12, "detail": "\"apple\"", "sortText": "0", - "insertText": "apple" + "insertText": "apple", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 23 + }, + "end": { + "line": 1, + "character": 26 + } + }, + "newText": "apple" + } } ] "#); @@ -1043,21 +1446,60 @@ take({\"\"}) "kind": 12, "detail": "Literal[\"x\"]", "sortText": "0", - "insertText": "x" + "insertText": "x", + "textEdit": { + "range": { + "start": { + "line": 9, + "character": 7 + }, + "end": { + "line": 9, + "character": 7 + } + }, + "newText": "x" + } }, { "label": "y", "kind": 12, "detail": "Literal[\"y\"]", "sortText": "1", - "insertText": "y" + "insertText": "y", + "textEdit": { + "range": { + "start": { + "line": 9, + "character": 7 + }, + "end": { + "line": 9, + "character": 7 + } + }, + "newText": "y" + } }, { "label": "z", "kind": 12, "detail": "Literal[\"z\"]", "sortText": "2", - "insertText": "z" + "insertText": "z", + "textEdit": { + "range": { + "start": { + "line": 9, + "character": 7 + }, + "end": { + "line": 9, + "character": 7 + } + }, + "newText": "z" + } } ] "#); @@ -1189,3 +1631,66 @@ walktr Ok(()) } + +/// every completion says what it replaces, so that a client never falls back on its own idea of +/// the word before the cursor: inside an f-string's replacement field that is the whole string +/// token, against which no name matches, and inside a literal it is the last word of it +#[test] +fn every_completion_says_what_it_replaces() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +from typing import Literal + +def pick(mode: Literal['slow-mode']) -> None: ... + +cart = 1 +label = f'{car}' +pick('slow-mo') +"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(foo, foo_content, 1); + let uri = server.file_uri(foo); + + let replaced = |completions: &[lsp_types::CompletionItem], label: &str| { + assert!( + completions.iter().all(|item| item.text_edit.is_some()), + "{completions:#?}" + ); + completions + .iter() + .find(|item| item.label == label) + .and_then(|item| match &item.text_edit { + Some(lsp_types::CompletionItemTextEdit::TextEdit(edit)) => { + Some((edit.range, edit.new_text.clone())) + } + _ => None, + }) + }; + + // `car` in the field, not `f'{car` + let in_field = server.completion_request(&uri, Position::new(5, 14)); + assert_eq!( + replaced(&in_field, "cart"), + Some(( + lsp_types::Range::new(Position::new(5, 11), Position::new(5, 14)), + "cart".to_string() + )) + ); + + // all of the literal before the cursor, not its last word `mo` + let in_literal = server.completion_request(&uri, Position::new(6, 13)); + assert_eq!( + replaced(&in_literal, "slow-mode"), + Some(( + lsp_types::Range::new(Position::new(6, 6), Position::new(6, 13)), + "slow-mode".to_string() + )) + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap index 0c246de7f8..c36665953c 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap @@ -8,6 +8,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -29,6 +42,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { @@ -50,6 +76,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -71,6 +110,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap index 0c246de7f8..c36665953c 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap @@ -8,6 +8,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -29,6 +42,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { @@ -50,6 +76,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -71,6 +110,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap index 0c246de7f8..c36665953c 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap @@ -8,6 +8,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -29,6 +42,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { @@ -50,6 +76,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -71,6 +110,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap index a3a9b5d385..36f23f1405 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap @@ -8,6 +8,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -29,6 +42,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { @@ -50,6 +76,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "Literal", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "Literal" + }, "additionalTextEdits": [ { "range": { @@ -71,6 +110,19 @@ expression: completions "kind": 6, "sortText": "[RANKING]", "insertText": "LiteralString", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 3 + }, + "end": { + "line": 1, + "character": 9 + } + }, + "newText": "LiteralString" + }, "additionalTextEdits": [ { "range": { diff --git a/crates/ty_server/tests/e2e/workspace_folders.rs b/crates/ty_server/tests/e2e/workspace_folders.rs index 2a01b97d3c..aecbb491bb 100644 --- a/crates/ty_server/tests/e2e/workspace_folders.rs +++ b/crates/ty_server/tests/e2e/workspace_folders.rs @@ -542,7 +542,20 @@ fn different_settings() -> Result<()> { "kind": "plaintext", "value": "The type of the None singleton.\n" }, - "sortText": "0" + "sortText": "0", + "textEdit": { + "range": { + "start": { + "line": 1, + "character": 0 + }, + "end": { + "line": 1, + "character": 4 + } + }, + "newText": "ZQZQZQ" + } } ] "#); diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 9a90d4b30b..491bd377ce 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -46,6 +46,13 @@ expression that can actually be awaited ## completions +every completion carries a `textEdit` saying what it replaces: what is typed of +the name before the cursor, `car` in `f"{car`, all of a string literal before +the cursor, `slow-mo` in `"slow-mo`, and more where a completion stands for more, +as a postfix template does for the expression it rewrites. so an editor never +works out the range from its own idea of a word, which inside an f-string is the +whole string token, against which nothing would match + ### the entry point at module level, `main` completes to the whole [entry point](main-function.md) From 98e8f8c52771db4a82640450d4c57ca8702075db Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:12:34 +1000 Subject: [PATCH 045/110] answer by/mro, the classes a class inherits from in its method resolution order, Shape, ABC and object for Circle, a diamond's shared base after both sides, so that go to super on a class lists them without a client linearizing typeHierarchy/supertypes itself --- crates/ty_ide/src/lib.rs | 3 +- crates/ty_ide/src/type_hierarchy.rs | 61 +++++++++++++ crates/ty_server/src/server/api.rs | 3 + crates/ty_server/src/server/api/requests.rs | 2 + .../ty_server/src/server/api/requests/mro.rs | 90 +++++++++++++++++++ crates/ty_server/tests/e2e/type_hierarchy.rs | 60 +++++++++++++ docs/basedpython/features/editor.md | 10 ++- 7 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 crates/ty_server/src/server/api/requests/mro.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 1c5da00e8c..b334240d1d 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -138,7 +138,8 @@ pub use syntax_outline::{ pub use test_links::{NewTest, SubjectTests, TestLink, subject_tests, test_subjects}; pub use ty_python_semantic::types::ide_support::InheritedMethodKind; pub use type_hierarchy::{ - TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_subtypes, type_hierarchy_supertypes, + TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_mro, type_hierarchy_subtypes, + type_hierarchy_supertypes, }; pub use unwrap::{UnwrapOption, unwrap_options}; pub use usage_counts::{ diff --git a/crates/ty_ide/src/type_hierarchy.rs b/crates/ty_ide/src/type_hierarchy.rs index 3cd59a8a13..c444d81ccf 100644 --- a/crates/ty_ide/src/type_hierarchy.rs +++ b/crates/ty_ide/src/type_hierarchy.rs @@ -60,6 +60,24 @@ pub fn type_hierarchy_supertypes( .collect() } +/// the classes the class at `offset` inherits from, in its method resolution order: where python +/// looks a member up after the class itself, `object` last. `None` where there is no class +pub fn type_hierarchy_mro( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option> { + let ty = resolve_type_at(db, file, offset)?; + let env = ProgramEnvironment::from_file(file); + ty_python_semantic::type_hierarchy_prepare(db, &env, ty)?; + Some( + ty_python_semantic::type_hierarchy_mro(db, &env, ty) + .into_iter() + .map(|class| type_hierarchy_class_to_item(db, class)) + .collect(), + ) +} + /// Get the subtypes (derived classes) of a type hierarchy item. /// /// This scans all available modules and can be expensive in large projects. @@ -150,6 +168,49 @@ mod tests { insta::assert_snapshot!(snapshot(&test.db, &[item]), @"/main.py:7:14 MyClass :: main"); } + #[test] + fn mro_lists_every_ancestor_in_lookup_order() { + let test = cursor_test( + r#" + class A: ... + class B(A): ... + class C(A): ... + class D(B, C): ... + "#, + ); + + let items = type_hierarchy_mro( + &test.db, + test.program_file(test.cursor.file), + test.cursor.offset, + ) + .expect("a class"); + insta::assert_snapshot!(snapshot(&test.db, &items), @" + /main.py:20:21 B :: main + /main.py:36:37 C :: main + /main.py:7:8 A :: main + vendored://stdlib/builtins.byi:2319:2325 object :: builtins + "); + } + + #[test] + fn mro_of_a_non_class_is_none() { + let test = cursor_test( + r#" + x = 42 + "#, + ); + + assert!( + type_hierarchy_mro( + &test.db, + test.program_file(test.cursor.file), + test.cursor.offset + ) + .is_none() + ); + } + #[test] fn prepare_type_hierarchy_on_non_class() { let test = cursor_test( diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index e91878b019..9194706bb4 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -91,6 +91,9 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::MroRequestHandler::METHOD => background_document_request_task::< + requests::MroRequestHandler, + >(req, BackgroundSchedule::Worker), requests::DocumentInheritedMembersRequestHandler::METHOD => { background_document_request_task::( req, diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 23992685e0..e672402e91 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -52,6 +52,7 @@ mod inlay_hints; mod module_name; mod move_members; mod move_symbol; +mod mro; mod on_type_formatting; mod overridable_members; mod override_members; @@ -125,6 +126,7 @@ pub(super) use move_members::{ PushMembersDownRequestHandler, }; pub(super) use move_symbol::{MoveSymbolRequestHandler, PrepareMoveSymbolRequestHandler}; +pub(super) use mro::MroRequestHandler; pub(super) use on_type_formatting::OnTypeFormattingRequestHandler; pub(super) use overridable_members::OverridableMembersRequestHandler; pub(super) use override_members::OverrideMembersRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/mro.rs b/crates/ty_server/src/server/api/requests/mro.rs new file mode 100644 index 0000000000..4dc66c90d8 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/mro.rs @@ -0,0 +1,90 @@ +//! `by/mro`: the classes a class inherits from, in its method resolution order +//! +//! where python looks a member up after the class itself: what an editor's *go to super* offers on +//! a class, as PyCharm lists a class's superclasses. `typeHierarchy/supertypes` answers only the +//! direct bases, and putting their ancestors in python's order is C3 linearization over every +//! base, which a client should not redo. see [`ty_ide::type_hierarchy_mro`] + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Request, TextDocumentIdentifier, + TypeHierarchyItem, Uri, +}; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::PositionExt; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::server::api::type_hierarchy::convert_to_lsp_item; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum MroRequest {} + +impl Request for MroRequest { + type Params = MroParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/mro"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// a position on a class's name, where it is declared or where it is used, as +/// `textDocument/prepareTypeHierarchy` takes it +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct MroParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +pub(crate) struct MroRequestHandler; + +impl RequestHandler for MroRequestHandler { + type RequestType = MroRequest; +} + +impl BackgroundDocumentRequestHandler for MroRequestHandler { + fn document_uri(params: &MroParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, and where there is no class. otherwise the classes + /// the class inherits from, without itself, `object` last, each as `typeHierarchy/supertypes` + /// answers a class + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: MroParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) + else { + return Ok(None); + }; + let Some(classes) = ty_ide::type_hierarchy_mro(db, db.program_file(file), offset) else { + return Ok(None); + }; + Ok(Some( + classes + .into_iter() + .filter_map(|class| convert_to_lsp_item(db, class, snapshot.encoding())) + .collect(), + )) + } +} + +impl RetriableRequestHandler for MroRequestHandler {} diff --git a/crates/ty_server/tests/e2e/type_hierarchy.rs b/crates/ty_server/tests/e2e/type_hierarchy.rs index fe9c02e4f0..0459b847f4 100644 --- a/crates/ty_server/tests/e2e/type_hierarchy.rs +++ b/crates/ty_server/tests/e2e/type_hierarchy.rs @@ -197,3 +197,63 @@ fn subtypes( partial_result_params: PartialResultParams::default(), }) } + +/// the request as a client sends it, in json throughout +enum Mro {} + +impl lsp_types::Request for Mro { + type Params = serde_json::Value; + type Result = Option>; + const METHOD: lsp_types::LspRequestMethod<'static> = + lsp_types::LspRequestMethod::Custom("by/mro"); + const MESSAGE_DIRECTION: lsp_types::MessageDirection = + lsp_types::MessageDirection::ClientToServer; +} + +/// `by/mro` answers every class a class inherits from in python's lookup order, a diamond's +/// shared base after both of its sides, each as `typeHierarchy/supertypes` answers a class +#[test] +fn mro_is_every_ancestor_in_lookup_order() -> anyhow::Result<()> { + let content = r#"class A: ... +class B(A): ... +class C(A): ... +class D(B, C): ... + +x = D() +"#; + + let mut server = TestServerBuilder::new()? + .with_file("foo.py", content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document("foo.py", content, 1); + let uri = server.file_uri("foo.py"); + + let mro = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": Position::new(3, 6), + })) + .expect("a class"); + let names: Vec<_> = mro.iter().map(|item| item.name.as_str()).collect(); + assert_eq!(names, ["B", "C", "A", "object"]); + assert_eq!(mro[0].uri, uri); + assert_eq!(mro[0].selection_range.start, Position::new(1, 6)); + + // and the same from a use of the class + let from_use = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": Position::new(5, 4), + })) + .expect("a class"); + assert_eq!(from_use, mro); + + // where there is no name, nothing + let none = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": Position::new(4, 0), + })); + assert_eq!(none, None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 491bd377ce..f42aa2b322 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1215,7 +1215,15 @@ own, defines somewhere else. `by/superMembers` goes up: given a position on the member is declared with — a `def` or a nested class, a name the class body assigns, annotates or captures, or an import's alias — it answers the superclass members that member overrides, each with its class, its file and the -name to land on. a class goes up with `typeHierarchy/supertypes` +name to land on. a class goes up with `by/mro` + +`by/mro` takes a position on a class's name, where it is declared or used, as +`textDocument/prepareTypeHierarchy` does, and answers every class it inherits +from in python's lookup order, its method resolution order without itself: +`Shape`, `ABC`, `object` for `class Circle(Shape)` with `class Shape(ABC)`, and +a diamond's shared base after both its sides. each is a type hierarchy item as +`typeHierarchy/supertypes` answers one, which gives only the direct bases. the +answer is `null` where there is no class what counts as an override is the override checks' own walk up the MRO, from the class as it is written rather than as a decorator returns it. of what that From dfceb6d780b4c0856c7a91364b1dddaa826c633b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:25:28 +1000 Subject: [PATCH 046/110] label each postfix completion by its name, for, rather than by the snippet it writes, and offer PyCharm's python postfix templates under its names, len, str, set, tuple, dict, isinstance, the comprehensions, yield, while, with, ifn, ifnn and the enumerate loops, each that needs something of the expression only where its type has it, a loop on what can be iterated and with on a context manager --- crates/ty_ide/src/completion.rs | 399 ++++++++++++++++-- crates/ty_ide/src/expression_types.rs | 4 +- .../src/types/ide_support.rs | 47 ++- docs/basedpython/features/editor.md | 47 ++- 4 files changed, 447 insertions(+), 50 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index f0eac1f083..dedc89c305 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -29,8 +29,8 @@ use ty_python_semantic::dependencies::{self, ImportStanding}; use ty_python_semantic::importer::{ImportRequest, Importer, MembersInScope}; use ty_python_semantic::types::format::{SpecLanguage, spec_language}; use ty_python_semantic::types::ide_support::{ - context_sensitive_members, extension_members, is_awaitable, overridable_members, - takes_no_arguments, + ValueUse, context_sensitive_members, extension_members, is_awaitable, overridable_members, + takes_no_arguments, value_supports, }; use ty_python_semantic::types::{SpecialFormType, UnionType}; use ty_python_semantic::{ @@ -3556,22 +3556,43 @@ fn add_main_completion( /// A postfix template: written after an expression's `.`, selecting one /// rewrites the whole `expr.name` rather than reading as attribute access. /// -/// basedpython's `.await` is not one of these — it is real syntax, and so -/// completes as an ordinary word. +/// The set is PyCharm's python one, under its names, less `cast` and `main`, +/// with basedpython's `let` and `var` and python's `match`, `type` and `repr` +/// besides. basedpython's `.await` is not one of these — it is real syntax, and +/// so completes as an ordinary word. #[derive(Clone, Copy)] enum Postfix { Print, Not, Type, Repr, + Len, + Str, List, + Set, + Tuple, + Dict, + Isinstance, Par, + Compl, + Comps, + Compg, + Compd, Return, Raise, + Yield, If, + Ifn, + Ifnn, + While, + With, Let, Var, For, + Iter, + Fore, + Itere, + Enum, Match, } @@ -3581,14 +3602,33 @@ impl Postfix { Postfix::Not, Postfix::Type, Postfix::Repr, + Postfix::Len, + Postfix::Str, Postfix::List, + Postfix::Set, + Postfix::Tuple, + Postfix::Dict, + Postfix::Isinstance, Postfix::Par, + Postfix::Compl, + Postfix::Comps, + Postfix::Compg, + Postfix::Compd, Postfix::Return, Postfix::Raise, + Postfix::Yield, Postfix::If, + Postfix::Ifn, + Postfix::Ifnn, + Postfix::While, + Postfix::With, Postfix::Let, Postfix::Var, Postfix::For, + Postfix::Iter, + Postfix::Fore, + Postfix::Itere, + Postfix::Enum, Postfix::Match, ]; @@ -3598,14 +3638,33 @@ impl Postfix { Postfix::Not => "not", Postfix::Type => "type", Postfix::Repr => "repr", + Postfix::Len => "len", + Postfix::Str => "str", Postfix::List => "list", + Postfix::Set => "set", + Postfix::Tuple => "tuple", + Postfix::Dict => "dict", + Postfix::Isinstance => "isinstance", Postfix::Par => "par", + Postfix::Compl => "compl", + Postfix::Comps => "comps", + Postfix::Compg => "compg", + Postfix::Compd => "compd", Postfix::Return => "return", Postfix::Raise => "raise", + Postfix::Yield => "yield", Postfix::If => "if", + Postfix::Ifn => "ifn", + Postfix::Ifnn => "ifnn", + Postfix::While => "while", + Postfix::With => "with", Postfix::Let => "let", Postfix::Var => "var", Postfix::For => "for", + Postfix::Iter => "iter", + Postfix::Fore => "fore", + Postfix::Itere => "itere", + Postfix::Enum => "enum", Postfix::Match => "match", } } @@ -3617,10 +3676,19 @@ impl Postfix { self, Postfix::Return | Postfix::Raise + | Postfix::Yield | Postfix::If + | Postfix::Ifn + | Postfix::Ifnn + | Postfix::While + | Postfix::With | Postfix::Let | Postfix::Var | Postfix::For + | Postfix::Iter + | Postfix::Fore + | Postfix::Itere + | Postfix::Enum | Postfix::Match ) } @@ -3631,12 +3699,49 @@ impl Postfix { matches!(self, Postfix::Let | Postfix::Var) } + /// What the rewrite puts the expression to, which a template is not offered + /// for an expression known not to support. + const fn needs(self) -> Option { + match self { + Postfix::For + | Postfix::Iter + | Postfix::Fore + | Postfix::Itere + | Postfix::Enum + | Postfix::Compl + | Postfix::Comps + | Postfix::Compg + | Postfix::Compd => Some(ValueUse::Iterate), + Postfix::Len => Some(ValueUse::Measure), + Postfix::With => Some(ValueUse::Enter), + Postfix::Ifn | Postfix::Ifnn => Some(ValueUse::TestForNone), + _ => None, + } + } + /// Whether the rewrite leaves somewhere for the cursor to go — a name to /// fill in, or a suite to write. const fn has_tab_stops(self) -> bool { matches!( self, - Postfix::If | Postfix::Let | Postfix::Var | Postfix::For | Postfix::Match + Postfix::Isinstance + | Postfix::Compl + | Postfix::Comps + | Postfix::Compg + | Postfix::Compd + | Postfix::If + | Postfix::Ifn + | Postfix::Ifnn + | Postfix::While + | Postfix::With + | Postfix::Let + | Postfix::Var + | Postfix::For + | Postfix::Iter + | Postfix::Fore + | Postfix::Itere + | Postfix::Enum + | Postfix::Match ) } @@ -3646,14 +3751,32 @@ impl Postfix { Postfix::Not => "negate the expression", Postfix::Type => "the expression's runtime type", Postfix::Repr => "the expression's `repr`", + Postfix::Len => "the expression's length", + Postfix::Str => "the expression as a `str`", Postfix::List => "collect the expression into a `list`", + Postfix::Set => "collect the expression into a `set`", + Postfix::Tuple => "collect the expression into a `tuple`", + Postfix::Dict => "make a `dict` of the expression", + Postfix::Isinstance => "test the expression's class", Postfix::Par => "wrap the expression in parentheses", + Postfix::Compl => "a list comprehension over the expression", + Postfix::Comps => "a set comprehension over the expression", + Postfix::Compg => "a generator expression over the expression", + Postfix::Compd => "a dict comprehension over the expression", Postfix::Return => "return the expression", Postfix::Raise => "raise the expression", + Postfix::Yield => "yield the expression", Postfix::If => "branch on the expression", + Postfix::Ifn => "branch on the expression being `None`", + Postfix::Ifnn => "branch on the expression not being `None`", + Postfix::While => "loop while the expression holds", + Postfix::With => "enter the expression as a context manager", Postfix::Let => "bind the expression to a `let`", Postfix::Var => "bind the expression to a `var`", - Postfix::For => "loop over the expression", + Postfix::For | Postfix::Iter => "loop over the expression", + Postfix::Fore | Postfix::Itere | Postfix::Enum => { + "loop over the expression with each element's index" + } Postfix::Match => "match on the expression", } } @@ -3676,20 +3799,67 @@ impl Postfix { text.to_string() } }; + // the same stop again, which a snippet mirrors as its first is typed + let again = |number: u32, text: &str| { + if snippets { + format!("${number}") + } else { + text.to_string() + } + }; let end = if snippets { "$0" } else { "" }; match self { Postfix::Print => format!("print({expr})"), Postfix::Not => format!("not {expr}"), Postfix::Type => format!("type({expr})"), Postfix::Repr => format!("repr({expr})"), + Postfix::Len => format!("len({expr})"), + Postfix::Str => format!("str({expr})"), Postfix::List => format!("list({expr})"), + Postfix::Set => format!("set({expr})"), + Postfix::Tuple => format!("tuple({expr})"), + Postfix::Dict => format!("dict({expr})"), + Postfix::Isinstance => format!("isinstance({expr}, {}){end}", stop(1, "object")), Postfix::Par => format!("({expr})"), + Postfix::Compl => format!( + "[{} for {} in {expr}]{end}", + stop(1, "item"), + again(1, "item") + ), + Postfix::Comps => format!( + "{{{} for {} in {expr}}}{end}", + stop(1, "item"), + again(1, "item") + ), + Postfix::Compg => format!( + "({} for {} in {expr}){end}", + stop(1, "item"), + again(1, "item") + ), + Postfix::Compd => format!( + "{{{}: {} for {} in {expr}}}{end}", + stop(1, "item"), + again(1, "item"), + again(1, "item") + ), Postfix::Return => format!("return {expr}"), Postfix::Raise => format!("raise {expr}"), + Postfix::Yield => format!("yield {expr}"), Postfix::If => format!("if {expr}:\n{indent}{end}"), + Postfix::Ifn => format!("if {expr} is None:\n{indent}{end}"), + Postfix::Ifnn => format!("if {expr} is not None:\n{indent}{end}"), + Postfix::While => format!("while {expr}:\n{indent}{end}"), + Postfix::With => format!("with {expr} as {}:\n{indent}{end}", stop(1, "name")), Postfix::Let => format!("let {} = {expr}{end}", stop(1, "name")), Postfix::Var => format!("var {} = {expr}{end}", stop(1, "name")), - Postfix::For => format!("for {} in {expr}:\n{indent}{end}", stop(1, "item")), + Postfix::For | Postfix::Iter => { + format!("for {} in {expr}:\n{indent}{end}", stop(1, "item")) + } + Postfix::Fore | Postfix::Itere | Postfix::Enum => format!( + "for {}, {} in enumerate({expr}):\n{indent}{end}", + stop(1, "index"), + stop(2, "item") + ), Postfix::Match => format!("match {expr}:\n{indent}case {end}"), } } @@ -3777,10 +3947,17 @@ fn add_postfix_completions<'db>( format!("{own}{}", stylist.indentation().as_str()) }); + // an expression whose type cannot be read could be anything, and is offered every template + let ty = expr.value.inferred_type(model); for &postfix in Postfix::ALL { if postfix.is_basedpython() && !source_type.is_basedpython() { continue; } + if let (Some(ty), Some(use_)) = (ty, postfix.needs()) + && !value_supports(model.db(), env, ty, use_) + { + continue; + } let indent = match (postfix.is_statement(), indent.as_deref()) { (true, None) => continue, (true, Some(indent)) => indent, @@ -3788,7 +3965,10 @@ fn add_postfix_completions<'db>( }; let snippets = capabilities.snippets && postfix.has_tab_stops(); let insert = postfix.expand(text, indent, snippets); + // the label is the template's name, as an editor lists a postfix template, rather than + // the text it writes, which would otherwise stand in for it let builder = Completion::builder(postfix.name()) + .label(postfix.name()) .kind(CompletionKind::Snippet) .replace(replace) .filter_text(format!("{text}.{}", postfix.name())) @@ -7190,11 +7370,11 @@ quux. builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" bar :: int baz :: int + dict(quux) :: postfix foo :: int - for item in quux: - :: postfix if quux: :: postfix + isinstance(quux, object) :: postfix list(quux) :: postfix match quux: case :: postfix @@ -7204,7 +7384,13 @@ quux. raise quux :: postfix repr(quux) :: postfix return quux :: postfix + set(quux) :: postfix + str(quux) :: postfix + tuple(quux) :: postfix type(quux) :: postfix + while quux: + :: postfix + yield quux :: postfix __annotations__ :: dict[str, Any] __class__() :: type[Quux] __delattr__() :: bound method Quux.__delattr__(name: str, /) @@ -7273,10 +7459,10 @@ C. assert_snapshot!( builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" - for item in C: - :: postfix + dict(C) :: postfix if C: :: postfix + isinstance(C, object) :: postfix list(C) :: postfix match C: case :: postfix @@ -7288,7 +7474,13 @@ C. raise C :: postfix repr(C) :: postfix return C :: postfix + set(C) :: postfix + str(C) :: postfix + tuple(C) :: postfix type(C) :: postfix + while C: + :: postfix + yield C :: postfix __annotate__() :: (() -> dict[str, AnnotationForm]) | None __annotations__ :: dict[str, Any] __base__ :: type | None @@ -7359,10 +7551,10 @@ Meta. { assert_snapshot!( builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" - for item in Meta: - :: postfix + dict(Meta) :: postfix if Meta: :: postfix + isinstance(Meta, object) :: postfix list(Meta) :: postfix match Meta: case :: postfix @@ -7374,7 +7566,13 @@ Meta. raise Meta :: postfix repr(Meta) :: postfix return Meta :: postfix + set(Meta) :: postfix + str(Meta) :: postfix + tuple(Meta) :: postfix type(Meta) :: postfix + while Meta: + :: postfix + yield Meta :: postfix __base__ :: type | None __bases__ :: tuple[type, ...] __basicsize__ :: int @@ -7436,11 +7634,11 @@ class Quux: assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" bar baz + dict(self) foo - for item in self: - if self: + isinstance(self, object) list(self) match self: case @@ -7450,7 +7648,13 @@ class Quux: raise self repr(self) return self + set(self) + str(self) + tuple(self) type(self) + while self: + + yield self __annotations__ __class__() __delattr__() @@ -7510,10 +7714,10 @@ Quux. assert_snapshot!( builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" - for item in Quux: - :: postfix + dict(Quux) :: postfix if Quux: :: postfix + isinstance(Quux, object) :: postfix list(Quux) :: postfix match Quux: case :: postfix @@ -7524,12 +7728,18 @@ Quux. raise Quux :: postfix repr(Quux) :: postfix return Quux :: postfix + set(Quux) :: postfix some_attribute :: int some_class_method() :: bound method .some_class_method() -> int some_method() :: def some_method(self) -> int some_property :: property some_static_method() :: def some_static_method(self) -> int + str(Quux) :: postfix + tuple(Quux) :: postfix type(Quux) :: postfix + while Quux: + :: postfix + yield Quux :: postfix __annotate__() :: (() -> dict[str, AnnotationForm]) | None __annotations__ :: dict[str, Any] __base__ :: type | None @@ -7599,10 +7809,25 @@ Answer. builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @" NO :: Literal[Answer.NO] YES :: Literal[Answer.YES] + {item: item for item in Answer} :: postfix + (item for item in Answer) :: postfix + [item for item in Answer] :: postfix + {item for item in Answer} :: postfix + dict(Answer) :: postfix + for index, item in enumerate(Answer): + :: postfix for item in Answer: :: postfix + for index, item in enumerate(Answer): + :: postfix if Answer: :: postfix + isinstance(Answer, object) :: postfix + for item in Answer: + :: postfix + for index, item in enumerate(Answer): + :: postfix + len(Answer) :: postfix list(Answer) :: postfix match Answer: case :: postfix @@ -7614,8 +7839,14 @@ Answer. raise Answer :: postfix repr(Answer) :: postfix return Answer :: postfix + set(Answer) :: postfix + str(Answer) :: postfix + tuple(Answer) :: postfix type(Answer) :: postfix value :: enum.property + while Answer: + :: postfix + yield Answer :: postfix __annotations__ :: dict[str, Any] __base__ :: type | None __bases__ :: tuple[type, ...] @@ -7699,12 +7930,27 @@ quux. assert_snapshot!( builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" + {item: item for item in quux} :: postfix + (item for item in quux) :: postfix + [item for item in quux] :: postfix + {item for item in quux} :: postfix count() :: bound method Quux.count(value: Any, /) -> int + dict(quux) :: postfix + for index, item in enumerate(quux): + :: postfix for item in quux: :: postfix + for index, item in enumerate(quux): + :: postfix if quux: :: postfix index() :: bound method Quux.index(value: Any, start: SupportsIndex = 0, stop: SupportsIndex = ..., /) -> int + isinstance(quux, object) :: postfix + for item in quux: + :: postfix + for index, item in enumerate(quux): + :: postfix + len(quux) :: postfix list(quux) :: postfix match quux: case :: postfix @@ -7714,9 +7960,15 @@ quux. raise quux :: postfix repr(quux) :: postfix return quux :: postfix + set(quux) :: postfix + str(quux) :: postfix + tuple(quux) :: postfix type(quux) :: postfix + while quux: + :: postfix x :: int y :: str + yield quux :: postfix __add__() :: Overload[(value: tuple[int | str, ...], /) -> tuple[int | str, ...], [T](value: tuple[T, ...], /) -> tuple[int | str | T, ...]] __annotations__ :: dict[str, Any] __class__() :: type[Quux] @@ -8256,11 +8508,13 @@ Re #[test] fn attribute_access_set_keyword_prefix() { + // the `.isinstance` postfix template rewrites the set, which is not the builtin leaking in let builder = completion_test_builder( "\ {1}.is ", - ); + ) + .filter(|c| c.detail.as_deref() != Some("postfix")); builder .build() @@ -8365,10 +8619,10 @@ b.a. assert_snapshot!(builder.skip_dunders().build().snapshot(), @" - for item in b.a: - + dict(b.a) if b.a: + isinstance(b.a, object) list(b.a) match b.a: case @@ -8378,8 +8632,14 @@ b.a. raise b.a repr(b.a) return b.a + set(b.a) + str(b.a) + tuple(b.a) type(b.a) + while b.a: + x + yield b.a "); } @@ -8576,10 +8836,10 @@ q.foo.xyz ); assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" - for item in ...: - + dict(...) if ...: + isinstance(..., object) list(...) match ...: case @@ -8589,7 +8849,13 @@ q.foo.xyz raise ... repr(...) return ... + set(...) + str(...) + tuple(...) type(...) + while ...: + + yield ... __annotations__ __class__() __delattr__() @@ -13853,7 +14119,7 @@ if foo: /// ones that spell basedpython syntax are held back. #[test] fn postfix_in_python() { - completion_test_builder("x = 1\nx.") + completion_test_builder("x = [1]\nx.") .build() .contains("print") .contains("if") @@ -13885,10 +14151,43 @@ if foo: assert_eq!( names, [ - "for", "if", "let", "list", "match", "not", "par", "print", "raise", "repr", - "return", "type", "var", + "compd", + "compg", + "compl", + "comps", + "dict", + "enum", + "for", + "fore", + "if", + "isinstance", + "iter", + "itere", + "len", + "let", + "list", + "match", + "not", + "par", + "print", + "raise", + "repr", + "return", + "set", + "str", + "tuple", + "type", + "var", + "while", + "yield", ] ); + // listed by its name, as an editor lists a template, not by what it writes + assert!( + test.completions() + .iter() + .all(|completion| completion.label() == completion.name.as_str()) + ); assert_eq!(inserted_text(&test, "not"), "not x"); assert_eq!(inserted_text(&test, "type"), "type(x)"); @@ -13902,6 +14201,55 @@ if foo: assert_eq!(inserted_text(&test, "var"), "var name = x"); assert_eq!(inserted_text(&test, "for"), "for item in x:\n "); assert_eq!(inserted_text(&test, "match"), "match x:\n case "); + assert_eq!(inserted_text(&test, "len"), "len(x)"); + assert_eq!(inserted_text(&test, "isinstance"), "isinstance(x, object)"); + assert_eq!(inserted_text(&test, "compl"), "[item for item in x]"); + assert_eq!(inserted_text(&test, "compd"), "{item: item for item in x}"); + assert_eq!(inserted_text(&test, "while"), "while x:\n "); + assert_eq!(inserted_text(&test, "yield"), "yield x"); + assert_eq!( + inserted_text(&test, "fore"), + "for index, item in enumerate(x):\n " + ); + } + + /// A template is offered only for an expression it can be used on, when the expression's + /// type says: a loop for what can be iterated, `len` for what has a length, `with` for a + /// context manager, and a test for `None` for what may be `None`. + #[test] + fn postfix_templates_follow_the_expression_type() { + let offered = |source: &str| { + let builder = CursorTest::builder() + .source("main.py", source) + .completion_test_builder() + .filter(|c| c.detail.as_deref() == Some("postfix")); + let test = builder.build(); + let mut names: Vec = test + .completions() + .iter() + .map(|completion| completion.name.to_string()) + .filter(|name| { + ["for", "len", "with", "ifn", "ifnn", "print"].contains(&name.as_str()) + }) + .collect(); + names.sort_unstable(); + names.join(" ") + }; + assert_eq!(offered("x = 1\nx."), "print"); + assert_eq!(offered("x = 'ab'\nx."), "for len print"); + assert_eq!( + offered("def f(x: int | None):\n x."), + "ifn ifnn print" + ); + assert_eq!( + offered("import threading\ndef f(x: threading.Lock):\n x."), + "print with" + ); + // a type that says nothing is offered everything + assert_eq!( + offered("from typing import Any\ndef f(x: Any):\n x."), + "for ifn ifnn len print with" + ); } /// A statement-shaped rewrite continues onto a line of its own, indented @@ -16183,6 +16531,7 @@ def f(x: UnsafeUnion[int, str]): isdecimal() isdigit() isidentifier() + isinstance(x, object) islower() isnumeric() isprintable() diff --git a/crates/ty_ide/src/expression_types.rs b/crates/ty_ide/src/expression_types.rs index c8168d400d..1d20706b94 100644 --- a/crates/ty_ide/src/expression_types.rs +++ b/crates/ty_ide/src/expression_types.rs @@ -6,9 +6,9 @@ //! //! on `shapes` the answer is `shapes`, `total_area(shapes)` and `total_area(shapes) * 2`, from the //! innermost out, each with the type ty inferred for it: `list[Shape]`, `float`, `float`. on a -//! selection of `total_area(shapes)` it starts from that call. On `kind` in +//! selection of `total_area(shapes)` it starts from that call. on `kind` in //! `kind, _, size = spec.partition(":")` it is `kind` and the tuple of targets around it, typed as -//! the tuple of what each target is bound to. +//! the tuple of what each target is bound to use ruff_db::parsed::parsed_module; use ruff_python_ast::find_node::covering_node; diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index b53d5916d8..53ecfc8184 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -1829,15 +1829,15 @@ fn argument_form_from_binding<'db>( CallArgumentForm::Unknown } -/// The type of a tuple or a list of targets a value is unpacked into: `kind, _, size` in +/// the type of a tuple or a list of targets a value is unpacked into: `kind, _, size` in /// `kind, _, size = spec.partition(":")` is `tuple[str, str, str]`, the tuple of what each of -/// its targets is bound to. A starred target, bound to a list of what it takes, stands for any +/// its targets is bound to. a starred target, bound to a list of what it takes, stands for any /// number of elements of that list's element type, so `first, *rest` over a `list[int]` is -/// `tuple[int, *tuple[int, ...]]`. +/// `tuple[int, *tuple[int, ...]]` /// /// ty records no type for such a target, since it is not evaluated: the value is, and each target /// is bound to a part of it. `None` for anything else, and for a target part of which has no -/// type. +/// type pub fn unpacking_target_type<'db>( model: &SemanticModel<'db>, target: ast::ExprRef<'_>, @@ -4829,6 +4829,45 @@ pub fn is_awaitable<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Typ ty.try_await(db, env).is_ok() } +/// A use a value can be put to, as a postfix template written after it puts it to one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ValueUse { + /// looping over it + Iterate, + /// measuring it with `len` + Measure, + /// entering it with `with` + Enter, + /// testing it for being `None`, which is only worth doing when it may be + TestForNone, +} + +/// Whether a value of type `ty` can be put to `use_`. A type ty cannot read, `Unknown` among +/// them, can be put to any. +pub fn value_supports<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + use_: ValueUse, +) -> bool { + match use_ { + ValueUse::Iterate => ty.try_iterate(db, env).is_ok(), + ValueUse::Measure => ty + .try_call_dunder( + db, + env, + "__len__", + CallArguments::none(), + TypeContext::default(), + ) + .is_ok(), + ValueUse::Enter => ty + .try_enter_with_mode(db, env, ty_python_core::EvaluationMode::Sync) + .is_ok(), + ValueUse::TestForNone => Type::none(db, env).is_assignable_to(db, env, ty), + } +} + /// What calling `ty` with no arguments gives, or `None` when it can't be called /// that way. /// diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index f42aa2b322..1fe0880297 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -15,25 +15,34 @@ on `xs` replaces `xs.print` with `print(xs)`: xs.print # → print(xs) ``` -| template | rewrites `x` into | -| ------------- | ----------------- | -| `.print` | `print(x)` | -| `.not` | `not x` | -| `.type` | `type(x)` | -| `.repr` | `repr(x)` | -| `.list` | `list(x)` | -| `.par` | `(x)` | -| `.return` | `return x` | -| `.raise` | `raise x` | -| `.if` | an `if` statement | -| `.for` | a `for` loop | -| `.match` | a `match` | -| `.let` `.var` | a binding | - -the templates from `.return` down are statements, so they are only offered -where the expression is the whole statement — `f(xs.print)` offers `.print` -but not `.for`. the ones that open a suite or need a name to bind place the -cursor for you, when your editor supports snippets +| template | rewrites `x` into | offered for | +| ----------------------------------- | ----------------------------------------------------- | ------------------------- | +| `.print` | `print(x)` | | +| `.not` | `not x` | | +| `.type` `.repr` `.str` `.len` | `type(x)`, `repr(x)`, `str(x)`, `len(x)` | `.len`: what has a length | +| `.list` `.set` `.tuple` `.dict` | `list(x)`, `set(x)`, `tuple(x)`, `dict(x)` | | +| `.isinstance` | `isinstance(x, object)` | | +| `.par` | `(x)` | | +| `.compl` `.comps` `.compg` `.compd` | a list, set, generator or dict comprehension over `x` | what can be iterated | +| `.return` `.raise` `.yield` | `return x`, `raise x`, `yield x` | | +| `.if` `.while` | an `if` or a `while` on `x` | | +| `.ifn` `.ifnn` | `if x is None:`, `if x is not None:` | what may be `None` | +| `.with` | `with x as name:` | a context manager | +| `.for` `.iter` | a `for` loop over `x` | what can be iterated | +| `.fore` `.itere` `.enum` | `for index, item in enumerate(x):` | what can be iterated | +| `.match` | a `match` | | +| `.let` `.var` | a binding | | + +these are PyCharm's python templates, under its names, less `cast` and +`main`. one that needs something of `x` is offered only where `x`'s type has +it, and on an expression whose type says nothing, any. the templates from +`.return` down are statements, so they are only offered where the expression is +the whole statement — `f(xs.print)` offers `.print` but not `.for`. the ones +that open a suite or need a name to bind place the cursor for you, when your +editor supports snippets + +each is a completion labelled with its name, `for`, and the detail `postfix`, +whose `textEdit` replaces the whole `x.for` and whose `filterText` is that text `.let` and `.var` spell basedpython bindings; the rest are valid python and are offered in `.py` files too From 89715554945a1df0c0d5ac088c84aad1302028da Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:30:29 +1000 Subject: [PATCH 047/110] answer by/rules, every rule the type checker reports under by the name a diagnostic carries as its code, with its summary, the level it has when the configuration names none and whether it is stable, in preview or deprecated, for an editor to give each rule an inspection of its own before any diagnostic arrives --- crates/ty_server/src/server/api.rs | 3 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/rules.rs | 162 ++++++++++++++++++ docs/basedpython/features/editor.md | 20 +++ 4 files changed, 187 insertions(+) create mode 100644 crates/ty_server/src/server/api/requests/rules.rs diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 9194706bb4..db3be980ee 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -138,6 +138,9 @@ pub(super) fn request(req: server::Request) -> Task { requests::ExplainRuleHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } + requests::RulesHandler::METHOD => { + background_request_task::(req, BackgroundSchedule::Worker) + } // The program model a run configuration is built from. Asked by URI rather than by open // document, because the module a configuration names is usually not open, and asked off // an editor's own schedule, so none of the three is latency sensitive. diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index e672402e91..f5a28e8491 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -62,6 +62,7 @@ mod prepare_rename; mod prepare_type_hierarchy; pub(super) mod references; mod rename; +mod rules; mod run_modules; mod selection_range; mod semantic_tokens; @@ -136,6 +137,7 @@ pub(super) use prepare_rename::PrepareRenameRequestHandler; pub(super) use prepare_type_hierarchy::PrepareTypeHierarchyRequestHandler; pub(super) use references::ReferencesRequestHandler; pub(super) use rename::RenameRequestHandler; +pub(super) use rules::RulesHandler; pub(super) use run_modules::RunModulesHandler; pub(super) use selection_range::SelectionRangeRequestHandler; pub(super) use semantic_tokens::SemanticTokensRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/rules.rs b/crates/ty_server/src/server/api/requests/rules.rs new file mode 100644 index 0000000000..3e3e6f81aa --- /dev/null +++ b/crates/ty_server/src/server/api/requests/rules.rs @@ -0,0 +1,162 @@ +//! `by/rules` — every rule the type checker reports under, for an editor to list them +//! +//! an editor that gives each rule an entry of its own, an inspection with a severity its user can +//! change, needs the rules before any diagnostic arrives: each diagnostic names the rule it is +//! reported under as its `code`, and this is the list those codes come from. what a rule means is +//! `by/explainRule`'s, and the linter's rules are `buff`'s, as they are for `buff/explainRule` + +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ty_python_semantic::default_lint_registry; +use ty_python_semantic::lint::{Level, LintStatus}; + +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum RulesRequest {} + +impl Request for RulesRequest { + type Params = Option; + type Result = Vec; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/rules"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// nothing, an empty object or no `params` at all: the rules are the server's, whatever the +/// project +#[derive(Debug, Default, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[expect( + clippy::empty_structs_with_brackets, + reason = "the brackets make it the empty object a client sends, which a unit struct is not" +)] +pub(crate) struct RulesParams {} + +/// a rule, as a diagnostic reported under it names it +#[derive(Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Rule { + /// what a diagnostic reported under the rule carries as its `code`: `unresolved-reference` + name: String, + /// a one-line summary + summary: String, + /// its level when the configuration names no other + default_level: RuleLevel, + /// whether the rule is stable, in preview or deprecated + status: RuleStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum RuleLevel { + /// not reported + Ignore, + /// reported as a warning + Warn, + /// reported as an error + Error, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum RuleStatus { + Stable, + /// its behaviour may still change + Preview, + /// no longer recommended; its summary says what to use instead + Deprecated, +} + +pub(crate) struct RulesHandler; + +impl RequestHandler for RulesHandler { + type RequestType = RulesRequest; +} + +impl BackgroundRequestHandler for RulesHandler { + fn run( + _snapshot: &SessionSnapshot, + _client: &Client, + _params: Option, + ) -> crate::server::Result> { + Ok(rules()) + } +} + +/// every rule a diagnostic can be reported under, by name. a removed rule reports nothing, and +/// is left out +fn rules() -> Vec { + let mut rules: Vec = default_lint_registry() + .lints() + .iter() + .filter_map(|lint| { + let status = match lint.status { + LintStatus::Stable { .. } => RuleStatus::Stable, + LintStatus::Preview { .. } => RuleStatus::Preview, + LintStatus::Deprecated { .. } => RuleStatus::Deprecated, + LintStatus::Removed { .. } => return None, + }; + Some(Rule { + name: lint.name().as_str().to_string(), + summary: lint.summary().to_string(), + default_level: match lint.default_level() { + Level::Ignore => RuleLevel::Ignore, + Level::Warn => RuleLevel::Warn, + Level::Error => RuleLevel::Error, + }, + status, + }) + }) + .collect(); + rules.sort_by(|left, right| left.name.cmp(&right.name)); + rules +} + +impl RetriableRequestHandler for RulesHandler {} + +#[cfg(test)] +mod tests { + use super::*; + + /// the wire form, exactly as a client sends it: no parameters at all, or an empty object + #[test] + fn the_params_a_client_sends_parse() { + let empty: Option = serde_json::from_str("{}").expect("an empty object"); + assert!(empty.is_some()); + let absent: Option = + serde_json::from_value(serde_json::Value::Null).expect("no params"); + assert!(absent.is_none()); + } + + #[test] + fn every_rule_is_listed_once_by_its_name() { + let rules = rules(); + let unresolved = rules + .iter() + .find(|rule| rule.name == "unresolved-reference") + .expect("a rule of the type checker"); + assert_eq!(unresolved.default_level, RuleLevel::Error); + assert_eq!(unresolved.status, RuleStatus::Stable); + assert!(!unresolved.summary.is_empty()); + assert!( + rules.windows(2).all(|pair| pair[0].name < pair[1].name), + "sorted and without duplicates" + ); + // the linter's are not this server's + assert!(rules.iter().all(|rule| rule.name != "F401")); + } + + #[test] + fn a_rule_serializes_as_a_client_reads_it() { + let rule = rules() + .into_iter() + .find(|rule| rule.name == "unresolved-reference") + .expect("a rule"); + let json = serde_json::to_value(&rule).expect("json"); + assert_eq!(json["name"], "unresolved-reference"); + assert_eq!(json["defaultLevel"], "error"); + assert_eq!(json["status"], "stable"); + } +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 1fe0880297..78701d3fcb 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1042,6 +1042,26 @@ writes by default, when none of them says. the answer is `null` anywhere that is not inside a triple-quoted docstring of a function holding nothing but whitespace on a line of its own +## rules + +each diagnostic names the rule it is reported under as its `code`, +`unresolved-reference`. an editor that gives every rule an entry of its own, an +inspection whose severity its user can change, asks `by/rules`, with no +parameters, for the rules before any diagnostic arrives. the answer is every +rule, by name, each with its one-line `summary`, its `defaultLevel` (`ignore`, +`warn` or `error`), the level it has when the configuration names no other, and +its `status` (`stable`, `preview` or `deprecated`): + +```json +[ + { "name": "unresolved-reference", "summary": "detects references to names that are not defined", "defaultLevel": "error", "status": "stable" } +] +``` + +a removed rule is left out. `by/explainRule` gives a rule's whole +documentation, and the linter's rules, `F401` among them, are `buff`'s. rules +have no categories + ## suppressing a diagnostic every diagnostic has two quick fixes that silence it rather than fix it. From 8a121853e15580e240c9c3db9931e6d944090bdd Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:45:31 +1000 Subject: [PATCH 048/110] leave a constructor and the methods it calls out of by/documentUsageCounts, as by/documentSuperMembers does, since a subclass's __init__ overrides nothing of its base's, counting other dunders such as __eq__ for their overrides as before --- crates/ty_ide/src/usage_counts.rs | 53 +++++++++++++------ .../api/requests/document_usage_counts.rs | 4 +- docs/basedpython/features/editor.md | 13 +++-- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 3ced11449e..93b98e85ba 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -17,10 +17,15 @@ //! the declarations counted are the document's outline ([`crate::document_symbols()`]) //! at the depth a reader scans a module at: its classes and functions, and the //! methods and properties of those classes. a dunder method is counted for its -//! overrides alone, as PyCharm marks `__init__` overridden: its usages are not -//! counted, since the language calls it rather than its name. a module's dunder -//! function is left out, and so is an `extension` block, which names a class it -//! does not declare. a nested function or a class inside a class is not counted +//! overrides alone, `__eq__` overridden like any method: its usages are not +//! counted, since the language calls it rather than its name. a constructor and +//! the methods it calls (`__init__`, `__new__`, `__post_init__`, +//! `__init_subclass__`) are left out, since a subclass's is no override of its +//! base's: it takes what its own class needs, and neither the override checks +//! nor `by/documentSuperMembers` hold it to the one it replaces. a module's +//! dunder function is left out, and so is an `extension` block, which names a +//! class it does not declare. a nested function or a class inside a class is not +//! counted use ruff_db::parsed::parsed_module; use ruff_python_ast::AnyNodeRef; @@ -28,7 +33,9 @@ use ruff_python_ast::find_node::covering_node; use ruff_python_ast::helpers::is_dunder; use ruff_text_size::TextRange; use ty_python_core::ProgramFile; -use ty_python_semantic::types::ide_support::{class_member_definitions, overridden_members}; +use ty_python_semantic::types::ide_support::{ + class_member_definitions, is_constructor_like_method, overridden_members, +}; use crate::declaration::ClassName; use crate::find_references::find_references_to_each; @@ -173,8 +180,13 @@ fn counted_declarations(db: &dyn Db, file: ProgramFile<'_>) -> Vec !is_constructor_like_method(name), + DeclarationKind::Class | DeclarationKind::Function => !is_dunder(name), + }; + if counted { declarations.push(DeclarationUsages { name: name.to_string(), kind, @@ -437,16 +449,22 @@ def __getattr__(name: str) -> int: "); } - /// A dunder method is counted for the methods overriding it, as PyCharm marks a base's - /// `__init__` overridden, but not for its usages: the language calls it, not its name. + /// a dunder method is counted for the methods overriding it, but not for its usages: the + /// language calls it, not its name. a constructor, and a method it calls, is not counted at + /// all: a subclass's is no override of its base's #[test] - fn a_dunder_method_is_counted_for_its_overrides_alone() { + fn a_dunder_method_is_counted_for_its_overrides_alone_and_a_constructor_not_at_all() { let test = cursor_test( " class Shape: def __init__(self, name: str) -> None: self.name = name + def __new__(cls, *args: object) -> 'Shape': + return super().__new__(cls) + + def __init_subclass__(cls) -> None: ... + def __eq__(self, other: object) -> bool: return False @@ -454,15 +472,20 @@ class Circle(Shape): def __init__(self) -> None: super().__init__('circle') + def __new__(cls, *args: object) -> 'Circle': + return super().__new__(cls) + + def __eq__(self, other: object) -> bool: + return True + Circle() == Shape('shape') ", ); assert_snapshot!(test.usage_counts(), @" - class Shape 2 refs=2 impls=1 - method __init__ 3 impls=1 - method __eq__ 6 impls=0 - class Circle 9 refs=1 impls=0 - method __init__ 10 impls=0 + class Shape 2 refs=3 impls=1 + method __eq__ 11 impls=1 + class Circle 14 refs=2 impls=0 + method __eq__ 21 impls=0 "); } } diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs index e7be58e322..4174c0e9a3 100644 --- a/crates/ty_server/src/server/api/requests/document_usage_counts.rs +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -148,7 +148,9 @@ impl BackgroundDocumentRequestHandler for DocumentUsageCountsRequestHandler { /// `null` when language services are off, or for a document whose declarations are not /// counted here: a notebook, whose ranges are per cell, or a template. otherwise the /// declarations in source order: the document's classes and functions, and the methods and - /// properties of those classes, less `extension` blocks and a module's dunder functions + /// properties of those classes, less `extension` blocks, a module's dunder functions and a + /// class's constructors and the methods they call (`__init__`, `__new__`, `__post_init__`, + /// `__init_subclass__`), which override nothing fn run_with_snapshot( db: &ProjectDatabase, snapshot: &DocumentSnapshot, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 78701d3fcb..ea837848b0 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1295,11 +1295,14 @@ each of its declarations and how many classes subclass it or members override it: what an editor draws above each one as "3 usages" or "2 overrides". the declarations are the document's outline at the depth a reader scans a module at — its classes and functions, and the methods and properties of those -classes. a dunder method is there for what overrides it, as a base's `__init__` -is marked overridden in PyCharm, but its usages are not counted, the language -calling it rather than its name: it has no `references`, even when they are -asked for. a module's dunder function is left out, and so is an `extension` -block, which names a class it does not declare +classes. a dunder method such as `__eq__` is there for what overrides it, but +its usages are not counted, the language calling it rather than its name: it +has no `references`, even when they are asked for. a constructor and the +methods it calls, `__init__`, `__new__`, `__post_init__` and +`__init_subclass__`, are left out, as `by/documentSuperMembers` leaves them +out: a subclass's constructor takes what its own class needs, so it overrides +nothing and nothing overrides it. a module's dunder function is left out, and so +is an `extension` block, which names a class it does not declare each declaration comes with its `name`, its `kind` (`class`, `function` or `method`), its whole `range` from its first decorator, its `selectionRange` on From 2e8029159c970bdf6fdc2c40f39cd67746c5ca01 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:49:19 +1000 Subject: [PATCH 049/110] answer by/smartCompletion, the completions that fit the type the position expects, an argument's parameter, a return's declared type, a declared assignment's or a case's subject, as values of that type and the classes and functions whose call makes one, for an editor's smart completion, and null where the position expects no type --- crates/ty_ide/src/completion.rs | 232 +++++++++++++++- crates/ty_ide/src/lib.rs | 2 +- .../src/types/ide_support.rs | 170 +++++++++++- crates/ty_server/src/server/api.rs | 6 + crates/ty_server/src/server/api/requests.rs | 4 +- .../src/server/api/requests/completion.rs | 257 +++++++++--------- .../server/api/requests/smart_completion.rs | 96 +++++++ crates/ty_server/tests/e2e/completions.rs | 62 +++++ docs/basedpython/features/editor.md | 16 ++ 9 files changed, 710 insertions(+), 135 deletions(-) create mode 100644 crates/ty_server/src/server/api/requests/smart_completion.rs diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index dedc89c305..9fd5c6c81d 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -29,8 +29,9 @@ use ty_python_semantic::dependencies::{self, ImportStanding}; use ty_python_semantic::importer::{ImportRequest, Importer, MembersInScope}; use ty_python_semantic::types::format::{SpecLanguage, spec_language}; use ty_python_semantic::types::ide_support::{ - ValueUse, context_sensitive_members, extension_members, is_awaitable, overridable_members, - takes_no_arguments, value_supports, + ValueUse, call_result_assignable_to, context_sensitive_members, declared_assignment_type, + expected_return_type, extension_members, is_awaitable, is_known_assignable_to, + overridable_members, takes_no_arguments, value_supports, }; use ty_python_semantic::types::{SpecialFormType, UnionType}; use ty_python_semantic::{ @@ -299,6 +300,60 @@ pub fn completion<'db>( replacing(completions.into_completions(), context.cursor.range) } +/// the completions at `offset` that fit the type the position expects: what an editor's smart, +/// type-matching completion offers +/// +/// the positions read are an argument, which expects its parameter's type; a `return`, which +/// expects what its function declares it returns; the value of a declared assignment, which +/// expects the declared type; and a `case` pattern, which expects the subject's. of the +/// completions [`completion`] answers there, the ones kept are a value whose type is assignable +/// to the expected one, and a function or class whose call gives such a value: on `x: Shape = ` +/// the variables holding a `Shape`, and `Circle` and `make_shape`. a completion whose type is +/// `Any` or `Unknown`, or not known at all, fits nothing for certain, and is left out. nothing is +/// imported, since smart completion offers what is already at hand +/// +/// `None` where the position expects no type the source states, or one that is `Any` or +/// `Unknown` +pub fn smart_completion<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + settings: &CompletionSettings, + capabilities: CompletionCapabilities, + file: ProgramFile<'db>, + offset: TextSize, +) -> Option>> { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let source = source_text(db, file.file(db)); + let context = Context::new(db, file, &parsed, &source, offset)?; + let model = SemanticModel::new(db, file); + let expected = context.cursor.smart_expected_type(db, file, &model)?; + if matches!(expected, Type::Dynamic(_)) { + return None; + } + let settings = CompletionSettings { + auto_import: false, + ..settings.clone() + }; + Some( + completion(db, env, &settings, capabilities, file, offset) + .into_iter() + // a keyword argument's name, which is what writes `name=`, is no value + .filter(|completion| { + !completion + .insert + .as_deref() + .is_some_and(|insert| insert.ends_with('=')) + }) + .filter(|completion| { + completion.ty.is_some_and(|ty| { + is_known_assignable_to(db, env, ty, expected) + || call_result_assignable_to(db, env, ty, expected) + }) + }) + .collect(), + ) +} + /// `completions`, each saying what it replaces: `typed` for one that does not say already. /// /// A client that is not told works the range out from its own idea of the word before the @@ -2044,6 +2099,67 @@ impl<'m> ContextCursor<'m> { self.expected_type_at(model, token.range()) } + /// the type the position under the cursor expects a value of, for smart + /// completion: an argument's parameter type, a `return`'s declared return + /// type, and what [`Self::expected_type`] reads. the innermost position the + /// cursor is in decides: in `x: int = f()` it is `f`'s parameter + fn smart_expected_type<'db>( + &self, + db: &'db dyn Db, + file: ProgramFile<'db>, + model: &SemanticModel<'db>, + ) -> Option> { + // a position that has only been opened holds no node the cursor falls in, `return ` + // ends before it, so it is asked again from the token that opened it + let preceding = match self.typed { + Some(_) => &self.tokens_before[..self.tokens_before.len().saturating_sub(1)], + None => self.tokens_before, + }; + let opener = preceding + .iter() + .rev() + .find(|token| !token.kind().is_trivia()) + .map(Ranged::range); + for probe in std::iter::once(self.range).chain(opener) { + for node in self.covering_node(probe).ancestors() { + match node { + ast::AnyNodeRef::ExprCall(call) + if call.arguments.start() < self.offset + && call.arguments.range().contains_inclusive(self.offset) => + { + let help = signature_help::signature_help(db, file, self.offset)?; + let signature = help.signatures.get(help.active_signature?)?; + let parameter = signature.parameters.get(signature.active_parameter?)?; + return Some(parameter.ty); + } + ast::AnyNodeRef::StmtReturn(_) => { + let function = self.covering_node(probe).ancestors().find_map( + |node| match node { + ast::AnyNodeRef::StmtFunctionDef(function) => Some(Some(function)), + ast::AnyNodeRef::ExprLambda(_) + | ast::AnyNodeRef::StmtClassDef(_) => Some(None), + _ => None, + }, + )??; + return expected_return_type(model, function); + } + // the declared type, whatever the value typed so far makes of the target + ast::AnyNodeRef::StmtAnnAssign(assignment) => { + return (probe.start() >= assignment.annotation.end()) + .then(|| declared_assignment_type(model, assignment)) + .flatten(); + } + ast::AnyNodeRef::StmtMatch(_) => { + return self.expected_type_at(model, probe); + } + _ if node.is_statement() => break, + _ => {} + } + } + } + None + } + /// The expected type at `probe`, which is either the cursor or the token /// that opened the position it sits in. fn expected_type_at<'db>( @@ -3556,10 +3672,10 @@ fn add_main_completion( /// A postfix template: written after an expression's `.`, selecting one /// rewrites the whole `expr.name` rather than reading as attribute access. /// -/// The set is PyCharm's python one, under its names, less `cast` and `main`, +/// the set is PyCharm's python one, under its names, less `cast` and `main`, /// with basedpython's `let` and `var` and python's `match`, `type` and `repr` /// besides. basedpython's `.await` is not one of these — it is real syntax, and -/// so completes as an ordinary word. +/// so completes as an ordinary word #[derive(Clone, Copy)] enum Postfix { Print, @@ -3699,8 +3815,8 @@ impl Postfix { matches!(self, Postfix::Let | Postfix::Var) } - /// What the rewrite puts the expression to, which a template is not offered - /// for an expression known not to support. + /// what the rewrite puts the expression to, which a template is not offered + /// for an expression known not to support const fn needs(self) -> Option { match self { Postfix::For @@ -14213,9 +14329,109 @@ if foo: ); } - /// A template is offered only for an expression it can be used on, when the expression's + /// what smart completion offers at the cursor of `source`, by name in the order offered, or + /// `none` where the position expects no type + fn smart(source: &str) -> String { + let test = CursorTest::builder().source("main.py", source).build(); + let file = test.program_file(test.cursor.file); + let env = ty_python_semantic::ProgramEnvironment::from_file(file); + match super::smart_completion( + &test.db, + &env, + &CompletionSettings::default(), + CompletionCapabilities::default(), + file, + test.cursor.offset, + ) { + None => "none".to_string(), + Some(completions) => completions + .iter() + .map(|completion| completion.name.as_str()) + .collect::>() + .join(" "), + } + } + + const SHAPES: &str = "\ +class Shape: ... +class Circle(Shape): ... +class Label: ... + +def make_shape() -> Shape: + return Circle() + +def draw(shape: Shape, label: Label) -> None: ... + +circle = Circle() +label = Label() +count = 3 +"; + + /// smart completion keeps what fits the type the position expects: a value of it, and a + /// class or function whose call makes one + #[test] + fn smart_completion_offers_what_fits_an_argument() { + assert_eq!( + smart(&format!("{SHAPES}draw()")), + "Circle Shape circle make_shape" + ); + // the second argument expects a label + assert_eq!( + smart(&format!("{SHAPES}draw(circle, )")), + "Label label" + ); + assert_eq!( + smart(&format!("{SHAPES}draw(label=)")), + "Label label" + ); + } + + #[test] + fn smart_completion_offers_what_fits_a_declaration_or_a_return() { + // the name being declared is among the names of its scope, as any completion has it + assert_eq!( + smart(&format!("{SHAPES}x: Shape = ")), + "Circle Shape circle make_shape x" + ); + // `f` returns a label too, called again + assert_eq!( + smart(&format!("{SHAPES}def f() -> Label:\n return ")), + "Label f label" + ); + // what is typed narrows it further, as it does any completion + assert_eq!( + smart(&format!("{SHAPES}x: Shape = ci")), + "Circle circle" + ); + } + + /// the innermost position decides: an argument of a call in a declared value expects the + /// parameter's type, not the declaration's + #[test] + fn smart_completion_reads_the_innermost_position() { + assert_eq!( + smart(&format!("{SHAPES}x: int = len()")), + smart(&format!("{SHAPES}len()")) + ); + assert_ne!(smart(&format!("{SHAPES}x: int = len()")), "count"); + } + + #[test] + fn smart_completion_has_nothing_to_go_on_without_an_expected_type() { + assert_eq!(smart(&format!("{SHAPES}x = ")), "none"); + assert_eq!( + smart(&format!("{SHAPES}def f():\n return ")), + "none" + ); + assert_eq!( + smart(&format!("{SHAPES}def g(x): ...\ng()")), + "none" + ); + } + + /// a template is offered only for an expression it can be used on, when the expression's /// type says: a loop for what can be iterated, `len` for what has a length, `with` for a - /// context manager, and a test for `None` for what may be `None`. + /// context manager, and a test for `None` for what may be `None` #[test] fn postfix_templates_follow_the_expression_type() { let offered = |source: &str| { diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index b334240d1d..a86e396255 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -71,7 +71,7 @@ pub use code_action::{FileEdit, QuickFix, code_actions}; pub use complete_statement::{CompletedStatement, StatementEdit, complete_statement}; pub use completion::{ AutoImportStyle, Completion, CompletionCapabilities, CompletionCommand, - CompletionInsertTextFormat, CompletionKind, CompletionSettings, completion, + CompletionInsertTextFormat, CompletionKind, CompletionSettings, completion, smart_completion, }; pub use data_flow::{Finding, FindingKind, data_flow_at}; pub use declaration::{ClassName, Declaration, DeclarationAt, DeclaredKind, declaration_at}; diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 53ecfc8184..7919b082a9 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -4829,7 +4829,96 @@ pub fn is_awaitable<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Typ ty.try_await(db, env).is_ok() } -/// A use a value can be put to, as a postfix template written after it puts it to one. +/// the type a `return` in `function`'s body is checked against: what the function declares it +/// returns, the return type of the generator a generator function returns, and not the coroutine +/// an `async def`'s call returns. `None` for a function that declares no return type, and for a +/// basedpython `-> asserts x`, which declares a guard rather than a type +pub fn expected_return_type<'db>( + model: &SemanticModel<'db>, + function: &ast::StmtFunctionDef, +) -> Option> { + if function.returns.is_none() || function.is_asserts_return { + return None; + } + let db = model.db(); + let env = model.program_environment(); + let index = semantic_index(db, model.program_file()); + let definition = function.definition(model); + let function_type = infer_definition_types(db, definition).function_type(definition)?; + let return_ty = crate::types::function::same_module_uncached_raw_signature( + db, + function_type, + ReturnCallableTypeVarScope::Lexical, + ) + .return_ty; + let body = index.try_node_scope(NodeWithScopeRef::Function(function))?; + Some(if body.is_generator_function(index) { + return_ty + .generator_return_type(db, &env) + .unwrap_or(return_ty) + } else { + return_ty + }) +} + +/// the type an annotated assignment declares its target, `Shape` in `x: Shape = make()`, whatever +/// value it is given. `None` where the annotation declares no type of a value, as the target of +/// `items[0]: int` has none +pub fn declared_assignment_type<'db>( + model: &SemanticModel<'db>, + assignment: &ast::StmtAnnAssign, +) -> Option> { + let db = model.db(); + let index = semantic_index(db, model.program_file()); + let definition = index.try_definition(assignment)?; + match infer_definition_types(db, definition).inferred_declaration(definition) { + crate::types::infer::InferredDeclaration::Declared(declared) => Some(declared.inner_type()), + crate::types::infer::InferredDeclaration::Rejected => None, + } +} + +/// whether `ty` is assignable to `expected` for a reason its type says: not because either is, +/// or holds, `Any` or `Unknown`, which every type is assignable to and from, nor because `ty` is +/// `Never`, which is assignable to every type and is never a value +/// +/// that is assignability whatever the gradual parts stand for: the widest type `ty` can stand for +/// is assignable to the narrowest `expected` can, so `list[Any]` is not known to be a `list[int]` +pub fn is_known_assignable_to<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ty: Type<'db>, + expected: Type<'db>, +) -> bool { + !ty.is_never() + && ty.top_materialization(db, env).is_assignable_to( + db, + env, + expected.bottom_materialization(db, env), + ) +} + +/// whether calling `callable`, with whatever arguments it takes, gives a value +/// [`is_known_assignable_to`] `expected`: an instance of a class, or what a function declares it +/// returns. `false` for what cannot be called +pub fn call_result_assignable_to<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + callable: Type<'db>, + expected: Type<'db>, +) -> bool { + let Some(callables) = callable.try_upcast_to_callable(db, env) else { + return false; + }; + (&callables).into_iter().any(|callable| { + callable + .signatures(db) + .overloads + .iter() + .any(|signature| is_known_assignable_to(db, env, signature.return_ty, expected)) + }) +} + +/// a use a value can be put to, as a postfix template written after it puts it to one #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ValueUse { /// looping over it @@ -4842,8 +4931,8 @@ pub enum ValueUse { TestForNone, } -/// Whether a value of type `ty` can be put to `use_`. A type ty cannot read, `Unknown` among -/// them, can be put to any. +/// whether a value of type `ty` can be put to `use_`. a type ty cannot read, `Unknown` among +/// them, can be put to any pub fn value_supports<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -5416,6 +5505,81 @@ def f(declared: int, text: str, flag: bool, subclass: Number, either: int | str) Ok(()) } + #[test] + fn a_type_is_known_assignable_only_when_no_gradual_part_admits_it() -> anyhow::Result<()> { + // each pair of expression statements is a type and the type it is to be assigned to + let pairs = describe_statement_types( + r#" +from typing import Any, Callable, Never + +def f( + ints: list[int], + anys: list[Any], + number: int, + either: int | Any, + boolean: bool, + callback: Callable[[Any], int], + int_callback: Callable[[int], int], + gradual: Any, + never: Never, +): + ints + ints + anys + ints + ints + anys + boolean + number + number + either + either + number + int_callback + callback + callback + int_callback + gradual + number + number + gradual + never + number +"#, + |db, env, types, _| { + types + .chunks(2) + .map(|pair| match pair { + [ty, expected] => format!( + "{} to {}: {}", + ty.display(db, env), + expected.display(db, env), + super::is_known_assignable_to(db, env, *ty, *expected) + ), + _ => "an odd statement out".to_string(), + }) + .collect() + }, + )?; + assert_eq!( + pairs, + [ + "list[int] to list[int]: true", + "list[Any] to list[int]: false", + "list[int] to list[Any]: false", + "bool to int: true", + "int to int | Any: true", + "int | Any to int: false", + "(int, /) -> int to (Any, /) -> int: false", + "(Any, /) -> int to (int, /) -> int: false", + "Any to int: false", + "int to Any: false", + "Never to int: false", + ] + ); + Ok(()) + } + #[test] fn builtin_definition_lookup_does_not_infer_scope() -> anyhow::Result<()> { let mut db = TestDbBuilder::new() diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index db3be980ee..d23b12418a 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -91,6 +91,12 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::SmartCompletionRequestHandler::METHOD => { + background_document_request_task::( + req, + BackgroundSchedule::LatencySensitive, + ) + } requests::MroRequestHandler::METHOD => background_document_request_task::< requests::MroRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index f5a28e8491..d2d59ecd67 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -23,7 +23,7 @@ mod code_action; mod code_action_resolve; mod code_lens; mod complete_statement; -mod completion; +pub(super) mod completion; mod copied_imports; mod data_flow; mod declaration; @@ -69,6 +69,7 @@ mod semantic_tokens; mod semantic_tokens_range; mod shutdown; mod signature_help; +mod smart_completion; mod super_members; mod syntax_outline; mod test_items; @@ -144,6 +145,7 @@ pub(super) use semantic_tokens::SemanticTokensRequestHandler; pub(super) use semantic_tokens_range::SemanticTokensRangeRequestHandler; pub(super) use shutdown::ShutdownHandler; pub(super) use signature_help::SignatureHelpRequestHandler; +pub(super) use smart_completion::SmartCompletionRequestHandler; pub(super) use super_members::SuperMembersRequestHandler; pub(super) use syntax_outline::SyntaxOutlineRequestHandler; pub(super) use test_items::TestItemsHandler; diff --git a/crates/ty_server/src/server/api/requests/completion.rs b/crates/ty_server/src/server/api/requests/completion.rs index c656893749..da41a8c4ba 100644 --- a/crates/ty_server/src/server/api/requests/completion.rs +++ b/crates/ty_server/src/server/api/requests/completion.rs @@ -90,136 +90,149 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { return Ok(None); } - // Safety: we just checked that completions is not empty. - let max_index_len = OneIndexed::new(completions.len()).unwrap().digits().get(); - // the type shown beside a suggestion is spelled in the syntax of the - // file being completed: `1`, not `Literal[1]`, in a basedpython file - let items: Vec = with_display_for_file(db, file, || { - completions - .into_iter() - .enumerate() - .map(|(i, comp)| { - let kind = comp.kind.map(ty_kind_to_lsp_kind); - // a format spec clause has no type to show, so it carries its - // own few words instead - let type_display = comp - .ty - .map(|ty| ty.display(db, &env).to_string()) - .or_else(|| comp.detail.as_ref().map(ToString::to_string)); - let to_text_edit = |edit: &Edit| { - let range = edit - .range() - .to_lsp_range(db, file, snapshot.encoding())? - .local_range(); - Some(TextEdit { - range, - new_text: edit.content().map(ToString::to_string).unwrap_or_default(), - }) + let list = completion_list(db, snapshot, file, &env, completions); + let len = list.items.len(); + let response = CompletionResponse::CompletionList(list); + tracing::debug!( + "Completions request returned {len} suggestions in {elapsed:?}", + elapsed = Instant::now().duration_since(start) + ); + Ok(Some(response)) + } +} + +/// `completions`, as a client reads them: each with what it replaces, its type spelled as the +/// file's language spells it, and the imports it brings +pub(super) fn completion_list<'db>( + db: &'db ProjectDatabase, + snapshot: &DocumentSnapshot, + file: ruff_db::files::File, + env: &ProgramEnvironment<'db>, + completions: Vec>, +) -> CompletionList { + let client_capabilities = snapshot.resolved_client_capabilities(); + let max_index_len = OneIndexed::new(completions.len()).map_or(1, |len| len.digits().get()); + // the type shown beside a suggestion is spelled in the syntax of the + // file being completed: `1`, not `Literal[1]`, in a basedpython file + let items: Vec = with_display_for_file(db, file, || { + completions + .into_iter() + .enumerate() + .map(|(i, comp)| { + let kind = comp.kind.map(ty_kind_to_lsp_kind); + // a format spec clause has no type to show, so it carries its + // own few words instead + let type_display = comp + .ty + .map(|ty| ty.display(db, env).to_string()) + .or_else(|| comp.detail.as_ref().map(ToString::to_string)); + let to_text_edit = |edit: &Edit| { + let range = edit + .range() + .to_lsp_range(db, file, snapshot.encoding())? + .local_range(); + Some(TextEdit { + range, + new_text: edit.content().map(ToString::to_string).unwrap_or_default(), + }) + }; + let import_edit = comp.import.as_ref().and_then(to_text_edit); + // an import is not the only edit a completion can need: a + // name completed inside a plain string also brings the `f` + // that makes the string an f-string + let additional_text_edits: Vec = import_edit + .iter() + .cloned() + .chain(comp.additional_edit.as_ref().and_then(to_text_edit)) + .collect(); + + let label = comp.label().to_string(); + // what accepting it writes, which the label shown for it may say more than + let new_text = comp + .insert + .as_ref() + .map_or_else(|| label.clone(), ToString::to_string); + let import_suffix = comp + .module_name + .and_then(|name| import_edit.is_some().then(|| format!(" (import {name})"))); + let (label, label_details) = if snapshot + .resolved_client_capabilities() + .supports_completion_item_label_details() + { + let label_details = CompletionItemLabelDetails { + detail: import_suffix, + description: type_display.clone(), }; - let import_edit = comp.import.as_ref().and_then(to_text_edit); - // an import is not the only edit a completion can need: a - // name completed inside a plain string also brings the `f` - // that makes the string an f-string - let additional_text_edits: Vec = import_edit - .iter() - .cloned() - .chain(comp.additional_edit.as_ref().and_then(to_text_edit)) - .collect(); + (label, Some(label_details)) + } else { + let label = import_suffix + .map(|suffix| format!("{label}{suffix}")) + .unwrap_or(label); + (label, None) + }; - let label = comp.label().to_string(); - // what accepting it writes, which the label shown for it may say more than - let new_text = comp - .insert - .as_ref() - .map_or_else(|| label.clone(), ToString::to_string); - let import_suffix = comp.module_name.and_then(|name| { - import_edit.is_some().then(|| format!(" (import {name})")) - }); - let (label, label_details) = if snapshot + let documentation = comp.documentation.map(|docstring| { + let (kind, value) = if snapshot .resolved_client_capabilities() - .supports_completion_item_label_details() + .prefers_markdown_in_completion() { - let label_details = CompletionItemLabelDetails { - detail: import_suffix, - description: type_display.clone(), - }; - (label, Some(label_details)) + ( + lsp_types::MarkupKind::Markdown, + docstring.render(MarkupKind::Markdown), + ) } else { - let label = import_suffix - .map(|suffix| format!("{label}{suffix}")) - .unwrap_or(label); - (label, None) + ( + lsp_types::MarkupKind::PlainText, + docstring.render(MarkupKind::PlainText), + ) }; - let documentation = comp.documentation.map(|docstring| { - let (kind, value) = if snapshot - .resolved_client_capabilities() - .prefers_markdown_in_completion() - { - ( - lsp_types::MarkupKind::Markdown, - docstring.render(MarkupKind::Markdown), - ) - } else { - ( - lsp_types::MarkupKind::PlainText, - docstring.render(MarkupKind::PlainText), - ) - }; - - Documentation::MarkupContent(lsp_types::MarkupContent { kind, value }) - }); - let insert_text = comp.insert.map(String::from); - let insert_text_format = match comp.insert_text_format { - CompletionInsertTextFormat::PlainText => None, - CompletionInsertTextFormat::Snippet => Some(InsertTextFormat::Snippet), - }; - // every completion says what it replaces: the client's own idea of the - // word before the cursor is not python's, and inside an f-string's - // `{car` it is the whole string token, against which nothing matches - let text_edit = comp.replace.and_then(|replace| { - let range = replace - .to_lsp_range(db, file, snapshot.encoding())? - .local_range(); - Some(lsp_types::CompletionItemTextEdit::TextEdit(TextEdit { - range, - new_text, - })) - }); + Documentation::MarkupContent(lsp_types::MarkupContent { kind, value }) + }); + let insert_text = comp.insert.map(String::from); + let insert_text_format = match comp.insert_text_format { + CompletionInsertTextFormat::PlainText => None, + CompletionInsertTextFormat::Snippet => Some(InsertTextFormat::Snippet), + }; + // every completion says what it replaces: the client's own idea of the + // word before the cursor is not python's, and inside an f-string's + // `{car` it is the whole string token, against which nothing matches + let text_edit = comp.replace.and_then(|replace| { + let range = replace + .to_lsp_range(db, file, snapshot.encoding())? + .local_range(); + Some(lsp_types::CompletionItemTextEdit::TextEdit(TextEdit { + range, + new_text, + })) + }); - CompletionItem { - label, - kind, - sort_text: Some(format!("{i:-max_index_len$}")), - detail: type_display, - label_details, - insert_text, - insert_text_format, - filter_text: comp.filter.map(String::from), - text_edit, - additional_text_edits: (!additional_text_edits.is_empty()) - .then_some(additional_text_edits), - documentation, - command: comp - .command - .and_then(|command| to_lsp_command(command, client_capabilities)), - ..Default::default() - } - }) - .collect() - }); - let len = items.len(); - let response = CompletionResponse::CompletionList(CompletionList { - is_incomplete: true, - items, - item_defaults: None, - apply_kind: None, - }); - tracing::debug!( - "Completions request returned {len} suggestions in {elapsed:?}", - elapsed = Instant::now().duration_since(start) - ); - Ok(Some(response)) + CompletionItem { + label, + kind, + sort_text: Some(format!("{i:-max_index_len$}")), + detail: type_display, + label_details, + insert_text, + insert_text_format, + filter_text: comp.filter.map(String::from), + text_edit, + additional_text_edits: (!additional_text_edits.is_empty()) + .then_some(additional_text_edits), + documentation, + command: comp + .command + .and_then(|command| to_lsp_command(command, client_capabilities)), + ..Default::default() + } + }) + .collect() + }); + CompletionList { + is_incomplete: true, + items, + item_defaults: None, + apply_kind: None, } } diff --git a/crates/ty_server/src/server/api/requests/smart_completion.rs b/crates/ty_server/src/server/api/requests/smart_completion.rs new file mode 100644 index 0000000000..ad22cba35a --- /dev/null +++ b/crates/ty_server/src/server/api/requests/smart_completion.rs @@ -0,0 +1,96 @@ +//! `by/smartCompletion`: the completions that fit the type the position expects +//! +//! what an editor's smart, type-matching completion offers: on `draw(` the values that are the +//! `Shape` its first parameter takes, and the classes and functions whose call makes one. LSP's +//! completion request has no kind of completion to ask for, so this is a request of its own, +//! taking what `textDocument/completion` takes and answering in the same form, items for items. +//! see [`ty_ide::smart_completion`] for which positions expect a type and which completions fit + +use std::borrow::Cow; + +use lsp_types::{ + CompletionList, CompletionParams, LspRequestMethod, MessageDirection, Request, Uri, +}; +use ty_ide::{CompletionCapabilities, smart_completion}; +use ty_project::{ProjectDatabase, SemanticDb as _}; +use ty_python_semantic::ProgramEnvironment; + +use crate::document::PositionExt; +use crate::server::api::requests::completion::completion_list; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum SmartCompletionRequest {} + +impl Request for SmartCompletionRequest { + type Params = CompletionParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/smartCompletion"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +pub(crate) struct SmartCompletionRequestHandler; + +impl RequestHandler for SmartCompletionRequestHandler { + type RequestType = SmartCompletionRequest; +} + +impl BackgroundDocumentRequestHandler for SmartCompletionRequestHandler { + fn document_uri(params: &CompletionParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document_position_params.text_document.uri) + } + + /// `null` when language services are off, in a template, and where the position expects no + /// type the source states, which leaves the client to say there is nothing to match. otherwise + /// a list, empty when nothing at hand fits + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: CompletionParams, + ) -> crate::server::Result> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return Ok(None); + } + let Some(file) = snapshot.document().to_notebook_or_file(db) else { + return Ok(None); + }; + let Some(offset) = params.text_document_position_params.position.to_text_size( + db, + file, + snapshot.document().uri(), + snapshot.encoding(), + ) else { + return Ok(None); + }; + let program_file = db.program_file(file); + let env = ProgramEnvironment::from_file(program_file); + let Some(completions) = smart_completion( + db, + &env, + snapshot.workspace_settings().completions(), + CompletionCapabilities::default().snippets( + snapshot + .resolved_client_capabilities() + .supports_completion_item_snippets(), + ), + program_file, + offset, + ) else { + return Ok(None); + }; + let mut list = completion_list(db, snapshot, file, &env, completions); + // the list is whole: typing on narrows it the way the client narrows any list + list.is_incomplete = false; + Ok(Some(list)) + } +} + +impl RetriableRequestHandler for SmartCompletionRequestHandler {} diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index f4abdc0245..6580a02883 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -1694,3 +1694,65 @@ pick('slow-mo') ); Ok(()) } + +/// `by/smartCompletion` as a client sends it, in json throughout +enum SmartCompletion {} + +impl lsp_types::Request for SmartCompletion { + type Params = serde_json::Value; + type Result = Option; + const METHOD: lsp_types::LspRequestMethod<'static> = + lsp_types::LspRequestMethod::Custom("by/smartCompletion"); + const MESSAGE_DIRECTION: lsp_types::MessageDirection = + lsp_types::MessageDirection::ClientToServer; +} + +/// smart completion answers, in the form `textDocument/completion` answers, only what fits the +/// type the position expects, and `null` where it expects none +#[test] +fn smart_completion_answers_what_fits_the_expected_type() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +class Shape: ... +class Label: ... + +def draw(shape: Shape) -> None: ... + +shape = Shape() +label = Label() +draw() +x = +"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(foo, foo_content, 1); + let uri = server.file_uri(foo); + let ask = |server: &mut crate::TestServer, position: Position| { + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": position, + })) + }; + + let in_argument = ask(&mut server, Position::new(7, 5)).expect("an argument expects a type"); + let labels: Vec<_> = in_argument + .items + .iter() + .map(|item| item.label.as_str()) + .collect(); + assert_eq!(labels, ["Shape", "shape"]); + assert!( + in_argument + .items + .iter() + .all(|item| item.text_edit.is_some()) + ); + + assert_eq!(ask(&mut server, Position::new(8, 4)), None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ea837848b0..8ce1715aaa 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -62,6 +62,22 @@ as a postfix template does for the expression it rewrites. so an editor never works out the range from its own idea of a word, which inside an f-string is the whole string token, against which nothing would match +### smart completion + +an editor's smart, type-matching completion asks `by/smartCompletion`, which +takes what `textDocument/completion` takes and answers a completion list of the +same items, kept to those that fit the type the position expects: an argument +expects its parameter's type, a `return` what its function declares it returns, +the value of a declared assignment the declared type, and a `case` pattern the +subject's. a value fits when its type is assignable to the expected one, and a +class or a function when calling it makes such a value, so on `draw(` for a +`draw(shape: Shape)` the list is the variables holding a `Shape`, `Shape` and +its subclasses, and the functions returning one. a completion of type `Any` or +`Unknown`, a function that never returns and a keyword argument's name fit +nothing for certain, and nothing is imported. of nested positions the innermost +decides: in `x: int = len(` it is `len`'s parameter. the answer is `null` where +the position expects no type the source states, or `Any` + ### the entry point at module level, `main` completes to the whole [entry point](main-function.md) From e47353599525b1b695ae15f3f5d7a25e1352ed50 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:52:20 +1000 Subject: [PATCH 050/110] give a completion labelled with more than its name the name as its filterText, so inspect.walktree, which a module style auto-import writes, matches walktre where a client matching against the label dropped it --- crates/ty_ide/src/completion.rs | 19 ++++++++++++++----- crates/ty_server/tests/e2e/completions.rs | 4 ++++ docs/basedpython/features/editor.md | 5 ++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 9fd5c6c81d..1c38874ef5 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -354,16 +354,25 @@ pub fn smart_completion<'db>( ) } -/// `completions`, each saying what it replaces: `typed` for one that does not say already. +/// `completions`, each saying what it replaces: `typed` for one that does not say already /// -/// A client that is not told works the range out from its own idea of the word before the +/// a client that is not told works the range out from its own idea of the word before the /// cursor, which is not python's: inside an f-string's `{car` it is the whole string token, and -/// every name is filtered out against `f"{car`. +/// every name is filtered out against `f"{car` +/// +/// what was typed is matched against the name, so a completion whose label is more than its +/// name says so in its filter text: a client matches the typed text against the filter text, or +/// the label when there is none, and `walktre` is not how `inspect.walktree` starts fn replacing(completions: Vec>, typed: TextRange) -> Vec> { completions .into_iter() .map(|mut completion| { - completion.replace.get_or_insert(typed); + if completion.replace.is_none() { + completion.replace = Some(typed); + if completion.filter.is_none() && completion.label() != completion.name.as_str() { + completion.filter = Some(completion.name.clone()); + } + } completion }) .collect() @@ -591,7 +600,7 @@ pub struct Completion<'db> { /// /// [`completion`] answers every completion with one: what was typed of /// the word before the cursor, or more where the completion stands for - /// more, as a django template path or a namespaced url name does. + /// more, as a django template path or a namespaced url name does pub replace: Option, /// The text the client should match the user's typing against, when /// [`Self::label`] is not it. diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 6580a02883..3e0de7cde4 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -476,6 +476,7 @@ is_typedd "label": "typing.is_typeddict", "kind": 3, "sortText": "0", + "filterText": "is_typeddict", "insertText": "typing.is_typeddict($0)", "insertTextFormat": 2, "textEdit": { @@ -572,6 +573,7 @@ TypedDi "label": "typing.TypedDict", "kind": 6, "sortText": "0", + "filterText": "TypedDict", "insertText": "typing.TypedDict", "textEdit": { "range": { @@ -591,6 +593,7 @@ TypedDi "label": "typing.is_typeddict", "kind": 3, "sortText": "1", + "filterText": "is_typeddict", "insertText": "typing.is_typeddict()", "textEdit": { "range": { @@ -1147,6 +1150,7 @@ re.match('', '', fla "kind": 6, "detail": "FlagsType", "sortText": "0", + "filterText": "flags", "insertText": "flags=", "textEdit": { "range": { diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 8ce1715aaa..647b70fc4e 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -60,7 +60,10 @@ the name before the cursor, `car` in `f"{car`, all of a string literal before the cursor, `slow-mo` in `"slow-mo`, and more where a completion stands for more, as a postfix template does for the expression it rewrites. so an editor never works out the range from its own idea of a word, which inside an f-string is the -whole string token, against which nothing would match +whole string token, against which nothing would match. what is typed is matched +against the name, so a completion labelled with more than its name carries the +name as its `filterText`: `inspect.walktree`, which `completions.importStyle` +`module` writes, matches `walktre` ### smart completion From 986bd66238dbc330d57953064bef68b77a93fd75 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:00:40 +1000 Subject: [PATCH 051/110] say in a resolved extraction's data where the name it made up is, rename, a location in the document as the edit leaves it at the place the rewritten code uses the name, for an editor to rename the new function or variable in place --- crates/ty_ide/src/refactor.rs | 101 +- crates/ty_ide/src/refactor/data_class.rs | 2 + crates/ty_ide/src/refactor/evaluation.rs | 34 +- .../ty_ide/src/refactor/extract_function.rs | 40 +- .../ty_ide/src/refactor/extract_variable.rs | 150 +- crates/ty_ide/src/refactor/inline_variable.rs | 6 +- crates/ty_ide/src/refactor/intentions.rs | 1256 ++++++++++++++--- .../ty_ide/src/refactor/return_annotation.rs | 1 + crates/ty_ide/src/refactor/split_join.rs | 1 + .../src/server/api/requests/code_action.rs | 6 + .../api/requests/code_action_resolve.rs | 36 +- crates/ty_server/tests/e2e/refactors.rs | 43 + docs/basedpython/features/editor.md | 11 + 13 files changed, 1479 insertions(+), 208 deletions(-) diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index af90ef4860..c63f481436 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -36,7 +36,7 @@ use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_db::source::{SourceText, source_text}; use ruff_diagnostics::Edit; use ruff_python_codegen::Stylist; -use ruff_text_size::TextRange; +use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_project::Db; use ty_python_core::ProgramFile; use ty_python_semantic::SemanticModel; @@ -161,6 +161,10 @@ pub struct Refactor { pub kind: RefactorKind, pub title: String, pub edits: Vec, + /// the name the rewrite made up, in the text the edits leave behind in the refactored file: + /// the new function's or variable's where the rewritten code uses it, for its user to rename + /// in place. `None` for a rewrite that makes up no name + pub name: Option, } /// Why a refactoring does not apply: either it is not about what is at the @@ -185,6 +189,47 @@ impl Refusal { pub(crate) struct Plan { title: String, edits: Vec, + /// the name the rewrite makes up, which its user will want to choose: the new + /// function's or the new variable's, where it is declared + name: Option, +} + +/// where a made-up name is in a plan's edits: the edit whose content writes it, and its range +/// within that content +pub(crate) struct PlannedName { + edit: usize, + range: TextRange, +} + +impl PlannedName { + /// the name `len` bytes long at byte `start` of the content of edit `edit` + fn at(edit: usize, start: usize, len: usize) -> Self { + let start = TextSize::try_from(start).unwrap_or_default(); + let len = TextSize::try_from(len).unwrap_or_default(); + Self { + edit, + range: TextRange::at(start, len), + } + } + + /// where the name is in the text `edits` leave behind, the edits being the plan's + fn in_edited_text(&self, edits: &[Edit]) -> Option { + let edit = edits.get(self.edit)?; + // what the edits before this one in the text add or remove before it + let mut shift: i64 = 0; + for other in edits { + if other.start() < edit.start() { + let added = other.content().map_or(0, str::len); + shift += i64::try_from(added).ok()? - i64::from(u32::from(other.range().len())); + } + } + let start = + i64::from(u32::from(edit.start())) + shift + i64::from(u32::from(self.range.start())); + Some(TextRange::at( + TextSize::from(u32::try_from(start).ok()?), + self.range.len(), + )) + } } /// Everything a refactoring reads about the file it rewrites. @@ -346,6 +391,7 @@ pub fn refactor( Ok(plan) => Ok(Refactor { kind, title: plan.title, + name: plan.name.and_then(|name| name.in_edited_text(&plan.edits)), edits: context.file_edits(plan.edits), }), Err(Refusal::NotApplicable) => Err(NOT_HERE.to_string()), @@ -363,6 +409,7 @@ fn is_refactorable(db: &dyn Db, file: ProgramFile<'_>) -> bool { pub(crate) mod test_support { use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; + use ruff_python_ast::PythonVersion; use ruff_python_trivia::textwrap::dedent; use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_project::{ProjectMetadata, SemanticDb as _}; @@ -385,6 +432,12 @@ pub(crate) mod test_support { Self::with_files("main.by", source, &[]) } + /// the test, read as python `version` reads it + pub(crate) fn with_python_version(mut self, version: PythonVersion) -> Self { + self.db.set_python_version(version); + self + } + pub(crate) fn with_files(path: &str, source: &str, others: &[(&str, &str)]) -> Self { let mut db = ty_project::TestDb::new(ProjectMetadata::new("test", SystemPathBuf::from("/"))); @@ -445,6 +498,14 @@ pub(crate) mod test_support { .join("\n") } + /// where the name `kind` makes up is in the text its edits leave, when it makes one up + pub(crate) fn name(&self, kind: RefactorKind) -> Option { + let program_file = self.db.program_file(self.file); + refactor(&self.db, program_file, kind, self.range) + .ok() + .and_then(|result| result.name) + } + /// The file after applying `kind`, or why it was not offered or refused. pub(crate) fn apply(&self, kind: RefactorKind) -> String { let program_file = self.db.program_file(self.file); @@ -481,6 +542,16 @@ pub(crate) mod test_support { edit.content().unwrap_or_default(), ); } + // the name a rewrite made up is where it says, in the text it leaves: the + // name its title ends a quoted `name` or `self.name` with + if let Some(name) = result.name { + let written = &text[name.start().to_usize()..name.end().to_usize()]; + assert!( + result.title.contains(&format!("{written}`")), + "the made-up name is at {name:?}, which reads `{written}`, not the name `{}` gives", + result.title + ); + } format!("{}\n---\n{text}", offer.title) } Err(reason) => { @@ -491,3 +562,31 @@ pub(crate) mod test_support { } } } + +#[cfg(test)] +mod tests { + use super::RefactorKind; + use super::test_support::RefactorTest; + + /// an extraction says where the name it made up is, where the rewritten code uses it, for an + /// editor to rename it in place; a rewrite that makes up no name says nothing + #[test] + fn extractions_say_where_their_new_name_is() { + for (kind, source) in [ + ( + RefactorKind::ExtractFunction, + "def f(r: float) -> float:\n total = 3.14 * r ** 2\n return total\n", + ), + ( + RefactorKind::ExtractVariable, + "def f(r: float) -> float:\n return 3.14 * r ** 2\n", + ), + ] { + let test = RefactorTest::python(source); + assert!(test.apply(kind).contains("---"), "{kind:?} applies"); + assert!(test.name(kind).is_some(), "{kind:?} names its made-up name"); + } + let test = RefactorTest::python("x = [1, 2]\n"); + assert_eq!(test.name(RefactorKind::SplitList), None); + } +} diff --git a/crates/ty_ide/src/refactor/data_class.rs b/crates/ty_ide/src/refactor/data_class.rs index 519dade0f8..c6132534f2 100644 --- a/crates/ty_ide/src/refactor/data_class.rs +++ b/crates/ty_ide/src/refactor/data_class.rs @@ -79,6 +79,7 @@ pub(super) fn plan( Ok(Plan { title: title.to_string(), edits: vec![Edit::range_deletion(marker.range())], + name: None, }) } (Direction::ToDataClass, None) => { @@ -105,6 +106,7 @@ pub(super) fn plan( Ok(Plan { title: title.to_string(), edits: vec![Edit::insertion("data ".to_string(), keyword.start())], + name: None, }) } _ => Err(Refusal::NotApplicable), diff --git a/crates/ty_ide/src/refactor/evaluation.rs b/crates/ty_ide/src/refactor/evaluation.rs index 043e1742c8..a5594f222a 100644 --- a/crates/ty_ide/src/refactor/evaluation.rs +++ b/crates/ty_ide/src/refactor/evaluation.rs @@ -185,12 +185,15 @@ pub(crate) fn is_stable_value(model: &SemanticModel<'_>, expr: &Expr) -> bool { /// operands themselves — runs no code of the program's and cannot raise. /// /// That is an operator of python's own over builtin scalars that is defined for every -/// pair of values it is given: `+`, `-` and `*` over numbers, `+` joining two strings -/// or two byte strings and `*` repeating one, the bitwise operators over integers, -/// equality, and ordering among numbers that are not complex or among strings. `/`, -/// `//`, `%`, `**` and the shifts are left out because each raises for some operands, -/// and something that may raise before an expression decides whether it runs. Reading -/// an attribute of a module is quiet too. +/// pair of values it is given: `+`, `-` and `*` over numbers that python does not convert +/// to compute them, `+` joining two strings or two byte strings, the bitwise operators +/// over integers, equality, and ordering among numbers that are not complex or among +/// strings. an `int` meeting a `float` or a `complex` is converted to one, which raises +/// `OverflowError` for an `int` too large for it, as `10**400 + 0.5` does, and repeating +/// a string or a byte string raises it for a count too large for an index. `/`, `//`, +/// `%`, `**` and the shifts are left out because each raises for some operands, and +/// something that may raise before an expression decides whether it runs. reading an +/// attribute of a module is quiet too fn operation_is_quiet(model: &SemanticModel<'_>, expr: &Expr) -> bool { use BuiltinScalar::{Bool, Bytes, Complex, Float, Int, Str}; const NUMBERS: &[BuiltinScalar] = &[Bool, Int, Float, Complex]; @@ -201,6 +204,16 @@ fn operation_is_quiet(model: &SemanticModel<'_>, expr: &Expr) -> bool { let all_in = |kinds: &[BuiltinScalar], allowed: &[BuiltinScalar]| { kinds.iter().all(|kind| allowed.contains(kind)) }; + // an `int` is converted to the other operand's `float` or `complex` + let converts = |from: &[BuiltinScalar], to: &[BuiltinScalar]| { + from.contains(&Int) && to.iter().any(|kind| matches!(kind, Float | Complex)) + }; + let arithmetic = |left: &[BuiltinScalar], right: &[BuiltinScalar]| { + all_in(left, NUMBERS) + && all_in(right, NUMBERS) + && !converts(left, right) + && !converts(right, left) + }; match expr { Expr::BinOp(bin_op) => { @@ -209,16 +222,11 @@ fn operation_is_quiet(model: &SemanticModel<'_>, expr: &Expr) -> bool { }; match bin_op.op { Operator::Add => { - (all_in(&left, NUMBERS) && all_in(&right, NUMBERS)) + arithmetic(&left, &right) || (left == [Str] && right == [Str]) || (left == [Bytes] && right == [Bytes]) } - Operator::Sub => all_in(&left, NUMBERS) && all_in(&right, NUMBERS), - Operator::Mult => { - (all_in(&left, NUMBERS) && all_in(&right, NUMBERS)) - || ((left == [Str] || left == [Bytes]) && all_in(&right, INTEGERS)) - || (all_in(&left, INTEGERS) && (right == [Str] || right == [Bytes])) - } + Operator::Sub | Operator::Mult => arithmetic(&left, &right), Operator::BitAnd | Operator::BitOr | Operator::BitXor => { all_in(&left, INTEGERS) && all_in(&right, INTEGERS) } diff --git a/crates/ty_ide/src/refactor/extract_function.rs b/crates/ty_ide/src/refactor/extract_function.rs index 466188594f..d60ab5cf62 100644 --- a/crates/ty_ide/src/refactor/extract_function.rs +++ b/crates/ty_ide/src/refactor/extract_function.rs @@ -33,7 +33,7 @@ use super::flow::{reaching_definitions, resolving_scope}; use super::hazards::context_dependent_construct; use super::names::{fresh_name, names_in_file}; use super::text::{leading_comments_start, nested_suites, statement_ancestors}; -use super::{Plan, RefactorContext, Refusal}; +use super::{Plan, PlannedName, RefactorContext, Refusal}; pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result { let Some(selection) = select_statements(context, range) else { @@ -332,13 +332,14 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result format!("{}.{name}", method.receiver), None => name, }; + let await_ = if scan.awaits { "await " } else { "" }; let call = format!( "{await_}{callee}({arguments})", - await_ = if scan.awaits { "await " } else { "" }, arguments = parameters .iter() .map(ToString::to_string) @@ -351,20 +352,35 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result {} + None => { + return Err(refuse( + "the checker cannot tell what the method is passed".to_string(), + )); + } + Some(MethodKind::Class) => { + return Err(refuse( + "the method is passed its class rather than an instance".to_string(), + )); + } + Some(MethodKind::Static) => { + return Err(refuse( + "the method is static, so it is passed no instance".to_string(), + )); + } + } if !function.decorator_list.is_empty() { return Err(refuse( "the method is decorated, so its first parameter need not be the instance" @@ -334,12 +356,15 @@ fn plan_selection( } }; + // the name where the expression was, after the receiver for a field + let name_at = PlannedName::at(1, replacement.len() - name.len(), name.len()); Ok(Plan { title, edits: vec![ insertion, Edit::range_replacement(replacement, selection.replaced), ], + name: Some(name_at), }) } @@ -865,33 +890,45 @@ mod tests { /// the same value because adding integers is associative. #[test] fn operands_of_an_associative_chain() { - assert_snapshot!(extract( + assert_snapshot!(RefactorTest::basedpython( " - def f(a: int, b: int, c: int, d: int): + def f(a: final int, b: final int, c: final int, d: final int): return a + b + c + d ", - ), @" + ).apply(RefactorKind::ExtractVariable), @" Extract variable `value` --- - def f(a: int, b: int, c: int, d: int): + def f(a: final int, b: final int, c: final int, d: final int): value = b + c return a + value + d "); } + /// an `int` may be an instance of a subclass, whose `+` is its own and need not be + /// associative #[test] - fn operands_of_a_string_concatenation() { + fn operands_of_ints_that_may_be_subclasses_are_not_offered() { assert_snapshot!(extract( " - def f(name: str): + def f(a: int, b: int, c: int, d: int): + return a + b + c + d + ", + ), @"not offered"); + } + + #[test] + fn operands_of_a_string_concatenation() { + assert_snapshot!(RefactorTest::basedpython( + " + def f(name: final str): return 'hello ' + name + '!' ", - ), @" + ).apply(RefactorKind::ExtractVariable), @" Extract variable `value` --- - def f(name: str): + def f(name: final str): value = name + '!' return 'hello ' + value "); @@ -920,39 +957,65 @@ mod tests { #[test] fn operands_after_a_call_are_refused() { - assert_snapshot!(extract( + assert_snapshot!(RefactorTest::basedpython( " - def f(b: int, c: int): + def f(b: final int, c: final int): return g() + b + c - def g() -> int: ... + def g() -> final int: ... ", - ), @"refused: Extract variable `value` (something with effects is evaluated before the expression, which would now run after it)"); + ).apply(RefactorKind::ExtractVariable), @"refused: Extract variable `value` (something with effects is evaluated before the expression, which would now run after it)"); } - /// reading a module's attribute and multiplying two numbers run nothing of the program's, - /// so evaluating the power first changes nothing + /// reading a module's attribute and multiplying two floats run nothing of the program's, so + /// evaluating the power first changes nothing. a `float` of a `.py` file may be an `int`, + /// which a `float` it meets converts, so this is written where `float` is a float alone, and + /// any `float` may be an instance of a subclass, so the floats are `final` #[test] fn after_a_module_attribute_and_arithmetic() { - assert_snapshot!(extract( + assert_snapshot!(RefactorTest::basedpython( " import math - def f(radius: float, scale: int): - return scale * math.pi * radius ** 2 + def f(radius: final float, scale: final float): + return math.pi, scale * scale * radius ** 2 ", - ), @" + ).apply(RefactorKind::ExtractVariable), @" Extract variable `value` --- import math - def f(radius: float, scale: int): + def f(radius: final float, scale: final float): value = radius ** 2 - return scale * math.pi * value + return math.pi, scale * scale * value "); } + /// an `int` too large for a `float` raises `OverflowError` when it is added to one, so the + /// addition before the expression may decide whether it runs + #[test] + fn after_arithmetic_mixing_int_and_float_is_refused() { + assert_snapshot!(extract( + " + def f(count: int, ratio: float, n: int): + return count * ratio + n * n + ", + ), @"refused: Extract variable `value` (something with effects is evaluated before the expression, which would now run after it)"); + } + + /// repeating a string as many times as an `int` too large for an index raises + /// `OverflowError` + #[test] + fn after_repeating_a_string_is_refused() { + assert_snapshot!(extract( + " + def f(text: str, count: int, n: int): + return text * count, n * n + ", + ), @"refused: Extract variable `value` (something with effects is evaluated before the expression, which would now run after it)"); + } + /// `self.price` may be a property, which could see the program differently once the /// expression after it is evaluated first #[test] @@ -1385,7 +1448,48 @@ mod tests { def double(x: int): return x * 2 ", - ), @"refused: Introduce attribute `self.value` (the method is decorated, so its first parameter need not be the instance)"); + ), @"refused: Introduce attribute `self.value` (the method is static, so it is passed no instance)"); + } + + /// a method is static by what its decorator resolves to, under whatever name + #[test] + fn field_in_a_static_method_under_another_name_is_refused() { + assert_snapshot!(field( + " + static = staticmethod + + class Math: + @static + def double(x: int): + return x * 2 + ", + ), @"refused: Introduce attribute `self.value` (the method is static, so it is passed no instance)"); + } + + /// `__init_subclass__` is passed the class, so an attribute assigned on its first parameter + /// would be the class's + #[test] + fn field_in_init_subclass_is_refused() { + assert_snapshot!(field( + " + class Base: + def __init_subclass__(cls, size: int): + print(size * 2) + ", + ), @"refused: Introduce attribute `self.value` (the method is passed its class rather than an instance)"); + } + + /// `__new__` is passed the class, not an instance + #[test] + fn field_in_new_is_refused() { + assert_snapshot!(field( + " + class Base: + def __new__(cls, size: int): + print(size * 2) + return super().__new__(cls) + ", + ), @"refused: Introduce attribute `self.value` (the method is static, so it is passed no instance)"); } #[test] diff --git a/crates/ty_ide/src/refactor/inline_variable.rs b/crates/ty_ide/src/refactor/inline_variable.rs index 8536179aab..cddd3771e5 100644 --- a/crates/ty_ide/src/refactor/inline_variable.rs +++ b/crates/ty_ide/src/refactor/inline_variable.rs @@ -180,7 +180,11 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result= 1` for a float that is `nan` — and is written `not (...)` //! otherwise. De Morgan's law changes what an `and` or an `or` evaluates to, which is one of its -//! operands, into a `bool`, so it is offered where only the truth of the result is read. A loop -//! becomes a comprehension only where nothing could tell the difference: the loop's variable is -//! not read after it, which a comprehension does not leave bound, and the collection is not read -//! while it is built. +//! operands, into a `bool`, so it is offered where only the truth of the result is read. a loop +//! becomes a comprehension only where nothing could tell the difference: nothing outside the loop +//! reads the loop's variable, which a comprehension does not bind where the loop is, nothing reads +//! the collection while it is built, and the loop has nothing a comprehension cannot hold — a +//! comment, or a `yield` or a walrus python does not allow in one use ruff_diagnostics::Edit; +use ruff_python_ast::PythonVersion; use ruff_python_ast::find_node::covering_node; -use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; -use ruff_python_ast::{self as ast, AnyNodeRef, BoolOp, CmpOp, Expr, Stmt, StringFlags, UnaryOp}; +use ruff_python_ast::token::TokenKind; +use ruff_python_ast::token::parenthesized_range; +use ruff_python_ast::visitor::source_order::{ + SourceOrderVisitor, TraversalSignal, walk_expr, walk_node, walk_stmt, +}; +use ruff_python_ast::{ + self as ast, AnyNodeRef, BoolOp, CmpOp, Expr, ExprRef, OperatorPrecedence, Stmt, StringFlags, + UnaryOp, +}; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; -use ty_python_semantic::types::ide_support::{BuiltinScalar, builtin_scalar_kinds}; +use ty_python_semantic::HasType; +use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::{BuiltinScalar, builtin_scalar_kinds, is_super}; +use ty_python_core::scope::FileScopeId; use ty_python_core::semantic_index; use super::evaluation::is_effect_free; use super::flow::{reaching_definitions, resolving_scope}; +use super::hazards::contains_named_expression; use super::{Plan, RefactorContext, Refusal}; -/// Invert an `if` with an `else`: negate its condition and swap its two bodies. +/// invert an `if` with an `else`: negate its condition and swap its two bodies pub(super) fn invert_if(context: &RefactorContext<'_>, range: TextRange) -> Result { let source = context.source(); let covering = covering_node(context.parsed.syntax().into(), range); @@ -74,7 +87,18 @@ pub(super) fn invert_if(context: &RefactorContext<'_>, range: TextRange) -> Resu } let body = TextRange::new(header_end, source.full_line_end(last.end())); let else_body = TextRange::new(else_header_end, source.full_line_end(else_last.end())); - let condition = negated(context, &statement.test).text; + let condition = negated(context, &statement.test); + if let Some(line) = lost_comment(context, &[statement.test.range()], &condition.kept) { + return Err(Refusal::refused( + title, + format!("the comment on line {line} would be lost"), + )); + } + let condition = condition.in_place_of( + context, + ExprRef::from(&*statement.test), + Some(AnyNodeRef::StmtIf(statement)), + ); Ok(Plan { title: title.to_string(), edits: vec![ @@ -82,6 +106,7 @@ pub(super) fn invert_if(context: &RefactorContext<'_>, range: TextRange) -> Resu Edit::range_replacement(with_line_end(&source[else_body]), body), Edit::range_replacement(with_line_end(&source[body]), else_body), ], + name: None, }) } @@ -93,8 +118,30 @@ fn with_line_end(text: &str) -> String { } } -/// Rewrite the innermost `and` or `or` around the range by De Morgan's law: `a and b` as -/// `not (not a or not b)`, and `not (a or b)` as `not a and not b`. +/// the one-based line of a comment inside one of `replaced` that none of `kept` holds, which a +/// rewrite writing only `kept`'s source in place of `replaced` would lose +fn lost_comment( + context: &RefactorContext<'_>, + replaced: &[TextRange], + kept: &[TextRange], +) -> Option { + let comment = context.parsed.tokens().iter().find(|token| { + token.kind() == TokenKind::Comment + && replaced + .iter() + .any(|range| range.contains_range(token.range())) + && !kept.iter().any(|range| range.contains_range(token.range())) + })?; + Some( + context + .source() + .count_lines(TextRange::up_to(comment.start())) + + 1, + ) +} + +/// rewrite the innermost `and` or `or` around the range by De Morgan's law: `a and b` as +/// `not (not a or not b)`, and `not (a or b)` as `not a and not b` pub(super) fn de_morgan(context: &RefactorContext<'_>, range: TextRange) -> Result { let covering = covering_node(context.parsed.syntax().into(), range); let ancestors: Vec = covering.ancestors().collect(); @@ -109,18 +156,30 @@ pub(super) fn de_morgan(context: &RefactorContext<'_>, range: TextRange) -> Resu }; let title = "Apply De Morgan's law".to_string(); let joined = match bool_op.op { - BoolOp::And => "or", - BoolOp::Or => "and", + BoolOp::And => BoolOp::Or, + BoolOp::Or => BoolOp::And, }; // `not (a or b)` becomes `not a and not b`, which is a `bool` either way if let Some(AnyNodeRef::ExprUnaryOp(not)) = ancestors.get(position + 1) && not.op == UnaryOp::Not { - let text = join(context, bool_op, joined); + let joined = join(context, bool_op, joined); + if let Some(line) = lost_comment(context, &[not.range()], &joined.kept) { + return Err(Refusal::refused( + title, + format!("the comment on line {line} would be lost"), + )); + } + let text = joined.in_place_of( + context, + ExprRef::UnaryOp(not), + ancestors.get(position + 2).copied(), + ); return Ok(Plan { title, edits: vec![Edit::range_replacement(text, not.range())], + name: None, }); } @@ -135,36 +194,55 @@ pub(super) fn de_morgan(context: &RefactorContext<'_>, range: TextRange) -> Resu )); } let inner = join(context, bool_op, joined); + if let Some(line) = lost_comment(context, &[bool_op.range()], &inner.kept) { + return Err(Refusal::refused( + title, + format!("the comment on line {line} would be lost"), + )); + } + let text = Written { + text: format!("not ({})", inner.text), + precedence: OperatorPrecedence::Not, + kept: inner.kept, + } + .in_place_of( + context, + ExprRef::BoolOp(bool_op), + ancestors.get(position + 1).copied(), + ); Ok(Plan { title, - edits: vec![Edit::range_replacement( - format!("not ({inner})"), - bool_op.range(), - )], + edits: vec![Edit::range_replacement(text, bool_op.range())], + name: None, }) } -/// The operands of `bool_op`, each negated, joined by `joined`. -fn join(context: &RefactorContext<'_>, bool_op: &ast::ExprBoolOp, joined: &str) -> String { - bool_op +/// the operands of `bool_op`, each negated, joined by `joined` +fn join(context: &RefactorContext<'_>, bool_op: &ast::ExprBoolOp, joined: BoolOp) -> Written { + let precedence = OperatorPrecedence::from(joined); + let mut kept = Vec::new(); + let text = bool_op .values .iter() .map(|value| { - let negated = negated(context, value); - // an `or` among the operands of an `and` keeps its own parentheses - if negated.is_or && joined == "and" { - format!("({})", negated.text) - } else { - negated.text - } + let operand = negated(context, value); + kept.extend_from_slice(&operand.kept); + // `and` and `or` evaluate to the same however a run of them is grouped, so an + // operand as tight as the operator it is joined by needs no parentheses + operand.within(|operand| operand >= precedence) }) .collect::>() - .join(&format!(" {joined} ")) + .join(&format!(" {joined} ")); + Written { + text, + precedence, + kept, + } } -/// Whether what `ancestors[0]` evaluates to is only read for its truth: it is the condition of +/// whether what `ancestors[0]` evaluates to is only read for its truth: it is the condition of /// an `if`, a `while`, an `assert`, a conditional expression or a comprehension's `if`, the -/// operand of `not`, or an operand of an `and` or an `or` that is itself only read for its truth. +/// operand of `not`, or an operand of an `and` or an `or` that is itself only read for its truth fn only_truth_is_read(ancestors: &[AnyNodeRef]) -> bool { let Some((node, rest)) = ancestors.split_first() else { return false; @@ -191,57 +269,150 @@ fn only_truth_is_read(ancestors: &[AnyNodeRef]) -> bool { } } -/// An expression negated, and whether the negation is an `or` at its top. -struct Negated { +/// the text of an expression the rewrite writes, how tightly its operator binds, and the ranges +/// of the source it copies, outside which a comment it replaces is lost +struct Written { text: String, - is_or: bool, + precedence: OperatorPrecedence, + kept: Vec, +} + +impl Written { + /// the text as an operand, in parentheses unless `fits` says its precedence lets it stand + /// without them + fn within(self, fits: impl FnOnce(OperatorPrecedence) -> bool) -> String { + if fits(self.precedence) { + self.text + } else { + format!("({})", self.text) + } + } + + /// the text written in place of `node`, a child of `parent`: in parentheses where python + /// would otherwise read it differently there, or where it spans lines, which only brackets + /// allow. `node`'s own parentheses stay, since the edit replaces what is inside them + fn in_place_of( + self, + context: &RefactorContext<'_>, + node: ExprRef<'_>, + parent: Option>, + ) -> String { + let Some(parent) = parent else { + return format!("({})", self.text); + }; + if parenthesized_range(node, parent, context.parsed.tokens()).is_some() { + return self.text; + } + let multiline = self.text.contains(['\n', '\r']); + self.within(|precedence| !multiline && fits_in(precedence, parent, node.range())) + } +} + +/// whether an expression of `precedence`, written without parentheses as the child of `parent` +/// at `child`, is read as that one child +fn fits_in(precedence: OperatorPrecedence, parent: AnyNodeRef<'_>, child: TextRange) -> bool { + match parent { + // `and` and `or` evaluate to the same however a run of them is grouped + AnyNodeRef::ExprBoolOp(bool_op) => precedence >= OperatorPrecedence::from(bool_op.op), + AnyNodeRef::ExprUnaryOp(unary) => precedence >= OperatorPrecedence::from(unary.op), + // a comparison inside another would chain with it + AnyNodeRef::ExprCompare(_) => { + precedence > OperatorPrecedence::ComparisonsMembershipIdentity + } + AnyNodeRef::ExprBinOp(bin_op) => { + let operator = OperatorPrecedence::from(bin_op.op); + match precedence.cmp(&operator) { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Less => false, + // `**` groups from the right, everything else from the left + std::cmp::Ordering::Equal => { + (bin_op.left.range() == child) != operator.is_right_associative() + } + } + } + // `a if b else c` takes an `or` at most as its first two operands, and a conditional + // expression as its last + AnyNodeRef::ExprIf(if_expr) => { + if if_expr.orelse.range() == child { + precedence >= OperatorPrecedence::IfElse + } else { + precedence > OperatorPrecedence::IfElse + } + } + AnyNodeRef::Comprehension(_) => precedence > OperatorPrecedence::IfElse, + // a condition may also be a walrus + AnyNodeRef::StmtIf(_) | AnyNodeRef::ElifElseClause(_) | AnyNodeRef::StmtWhile(_) => { + precedence >= OperatorPrecedence::IfElse || precedence == OperatorPrecedence::Assign + } + AnyNodeRef::StmtAssert(_) + | AnyNodeRef::StmtReturn(_) + | AnyNodeRef::StmtExpr(_) + | AnyNodeRef::StmtAssign(_) + | AnyNodeRef::StmtAnnAssign(_) + | AnyNodeRef::Arguments(_) + | AnyNodeRef::Keyword(_) + | AnyNodeRef::ExprList(_) + | AnyNodeRef::ExprSet(_) + | AnyNodeRef::ExprTuple(_) + | AnyNodeRef::ExprLambda(_) + | AnyNodeRef::ExprNamed(_) => precedence >= OperatorPrecedence::IfElse, + _ => precedence >= OperatorPrecedence::CallAttribute, + } } /// `expr` negated for its truth: `not` taken off, a comparison's operator flipped where that -/// means the same, an `and` and an `or` swapped by De Morgan's law, and `not (...)` otherwise. -fn negated(context: &RefactorContext<'_>, expr: &Expr) -> Negated { +/// means the same, an `and` and an `or` swapped by De Morgan's law, and `not` written before it +/// otherwise +fn negated(context: &RefactorContext<'_>, expr: &Expr) -> Written { let source = context.source(); - let text = |expr: &Expr| source[expr.range()].to_string(); match expr { - Expr::UnaryOp(unary) if unary.op == UnaryOp::Not => Negated { - is_or: matches!(&*unary.operand, Expr::BoolOp(inner) if inner.op == BoolOp::Or), - text: if unary.operand.is_named_expr() { - format!("({})", text(&unary.operand)) - } else { - text(&unary.operand) - }, + Expr::UnaryOp(unary) if unary.op == UnaryOp::Not => Written { + text: source[unary.operand.range()].to_string(), + precedence: OperatorPrecedence::from_expr(&unary.operand), + kept: vec![unary.operand.range()], }, Expr::Compare(compare) if let [left, right] = &*compare.operands && let Some(flipped) = flipped(context, compare) => { - Negated { - text: format!("{} {} {}", text(left), flipped.as_str(), text(right)), - is_or: false, + // each operand keeps the parentheses it is written in + let operand = |operand: &Expr| { + context.parenthesized_range(operand, AnyNodeRef::ExprCompare(compare)) + }; + let (left, right) = (operand(left), operand(right)); + Written { + text: format!("{} {} {}", &source[left], flipped.as_str(), &source[right]), + precedence: OperatorPrecedence::ComparisonsMembershipIdentity, + kept: vec![left, right], } } - Expr::BoolOp(bool_op) => { - let joined = match bool_op.op { - BoolOp::And => "or", - BoolOp::Or => "and", - }; - Negated { - text: join(context, bool_op, joined), - is_or: joined == "or", + Expr::BoolOp(bool_op) => join( + context, + bool_op, + match bool_op.op { + BoolOp::And => BoolOp::Or, + BoolOp::Or => BoolOp::And, + }, + ), + _ => { + // a comparison is parenthesised too, though `not` binds looser than it, so that + // `not (x > 1)` does not read as `not x` compared with `1` + let operand = Written { + text: source[expr.range()].to_string(), + precedence: OperatorPrecedence::from_expr(expr), + kept: Vec::new(), + } + .within(|precedence| precedence > OperatorPrecedence::ComparisonsMembershipIdentity); + Written { + text: format!("not {operand}"), + precedence: OperatorPrecedence::Not, + kept: vec![expr.range()], } } - Expr::Name(_) | Expr::Attribute(_) | Expr::Call(_) | Expr::Subscript(_) => Negated { - text: format!("not {}", text(expr)), - is_or: false, - }, - _ => Negated { - text: format!("not ({})", text(expr)), - is_or: false, - }, } } -/// The operator that negates `compare`'s, where swapping it means the same. +/// the operator that negates `compare`'s, where swapping it means the same fn flipped(context: &RefactorContext<'_>, compare: &ast::ExprCompare) -> Option { const INTEGERS: &[BuiltinScalar] = &[BuiltinScalar::Bool, BuiltinScalar::Int]; const NUMBERS: &[BuiltinScalar] = &[ @@ -288,8 +459,8 @@ fn flipped(context: &RefactorContext<'_>, compare: &ast::ExprCompare) -> Option< } } -/// Write a string literal in the other quotes: `'it'` as `"it"`, escaping what the new quote -/// would end and unescaping the old one. +/// write a string literal in the other quotes: `'it'` as `"it"`, escaping what the new quote +/// would end and unescaping the old one pub(super) fn flip_quotes( context: &RefactorContext<'_>, range: TextRange, @@ -352,10 +523,11 @@ pub(super) fn flip_quotes( format!("{}{new}{converted}{new}", &text[..prefix_len]), literal_range, )], + name: None, }) } -/// Rewrite a loop that only adds to a collection made empty right above it as a comprehension: +/// rewrite a loop that only adds to a collection made empty right above it as a comprehension: /// /// ```py /// result = [] result = [item * 2 for item in items if item > 0] @@ -364,7 +536,7 @@ pub(super) fn flip_quotes( /// result.append(item * 2) /// ``` /// -/// for a list's `append`, a set's `add` and a dict's item assignment. +/// for a list's `append`, a set's `add` and a dict's item assignment pub(super) fn loop_to_comprehension( context: &RefactorContext<'_>, range: TextRange, @@ -406,23 +578,26 @@ pub(super) fn loop_to_comprehension( [addition] => (None, addition), _ => return Err(Refusal::NotApplicable), }; - let (open, close, element) = match (&*initial.value, addition) { + // what the comprehension writes of the loop: its element, as the expressions it is made of + let (open, close, kind, elements): (_, _, _, Vec<&Expr>) = match (&*initial.value, addition) { (Expr::List(list), Stmt::Expr(expression)) if list.elts.is_empty() => { let element = added(&expression.value, name, "append").ok_or(Refusal::NotApplicable)?; - ("[", "]", source[element.range()].to_string()) + ("[", "]", "list", vec![element]) } (Expr::Call(call), Stmt::Expr(expression)) if matches!(&*call.func, Expr::Name(func) if func.id.as_str() == "set") && call.arguments.is_empty() => { let element = added(&expression.value, name, "add").ok_or(Refusal::NotApplicable)?; - ("{", "}", source[element.range()].to_string()) + ("{", "}", "set", vec![element]) } (Expr::Dict(dict), Stmt::Assign(assign)) if dict.is_empty() => { let [Expr::Subscript(target)] = assign.targets.as_slice() else { return Err(Refusal::NotApplicable); }; - if !matches!(&*target.value, Expr::Name(value) if value.id.as_str() == name) { + if !matches!(&*target.value, Expr::Name(value) if value.id.as_str() == name) + || target.slice.is_slice_expr() + { return Err(Refusal::NotApplicable); } // the loop evaluates the value before the key, and a comprehension the key first @@ -435,23 +610,10 @@ pub(super) fn loop_to_comprehension( after it", )); } - ( - "{", - "}", - format!( - "{}: {}", - &source[target.slice.range()], - &source[assign.value.range()] - ), - ) + ("{", "}", "dict", vec![&*target.slice, &*assign.value]) } _ => return Err(Refusal::NotApplicable), }; - let kind = match open { - "[" => "list", - _ if element.contains(": ") && matches!(&*initial.value, Expr::Dict(_)) => "dict", - _ => "set", - }; let title = format!("Convert loop to {kind} comprehension"); if !for_loop.orelse.is_empty() { @@ -460,11 +622,11 @@ pub(super) fn loop_to_comprehension( "its `else` runs after the loop, which a comprehension has no place for", )); } + let index = semantic_index(context.db, context.file); // `set` is python's own unless the program binds the name if let Expr::Call(call) = &*initial.value && let Expr::Name(func) = &*call.func { - let index = semantic_index(context.db, context.file); let shadowed = index .try_expression_scope_id(&ast::ExprRef::Name(func)) .is_none_or(|scope| resolving_scope(index, scope, func.id.as_str()).is_some()); @@ -472,6 +634,79 @@ pub(super) fn loop_to_comprehension( return Err(Refusal::refused(title, "`set` is not python's own here")); } } + // a class body's names are not visible inside a comprehension + if in_class_body(context, for_loop) { + return Err(Refusal::refused( + title, + "a comprehension in a class body cannot see the class's names", + )); + } + let Some(loop_scope) = index.try_expression_scope_id(&ast::ExprRef::from(&*for_loop.iter)) + else { + return Err(Refusal::NotApplicable); + }; + let mut bound = Vec::new(); + names_in(&for_loop.target, &mut bound); + + // what python does not allow in a comprehension + let kept: Vec<&Expr> = elements.iter().copied().chain(condition).collect(); + if kept.iter().any(|expr| yields(expr)) { + return Err(Refusal::refused( + title, + "it adds what a `yield` evaluates to, which a comprehension cannot hold", + )); + } + if let Some(variable) = bound + .iter() + .find(|variable| kept.iter().any(|expr| assigns_with_walrus(expr, variable))) + { + return Err(Refusal::refused( + title, + format!("a walrus assigns `{variable}`, which a comprehension does not allow"), + )); + } + if contains_named_expression(&for_loop.iter) { + return Err(Refusal::refused( + title, + "its iterable assigns with a walrus, which a comprehension does not allow", + )); + } + // before 3.12 a comprehension runs as a function of its own, whose first parameter is the + // iterator, which is what a zero-argument `super()` in it would be passed as its receiver + if context.file.python_version(context.db) < PythonVersion::PY312 + && kept + .iter() + .copied() + .chain([&*for_loop.target]) + .any(|expr| calls_zero_argument_super(context, expr)) + { + return Err(Refusal::refused( + title, + "it calls `super()` without arguments, which in a comprehension before python 3.12 \ + is not passed the method's receiver", + )); + } + + // a comment the comprehension does not keep would be lost + let removed = [ + initial.value.range(), + TextRange::new( + source.full_line_end(initial.end()), + source.full_line_end(for_loop.end()), + ), + ]; + let kept_ranges: Vec = kept + .iter() + .map(Ranged::range) + .chain([for_loop.target.range(), for_loop.iter.range()]) + .collect(); + if let Some(line) = lost_comment(context, &removed, &kept_ranges) { + return Err(Refusal::refused( + title, + format!("the comment on line {line} has no place in a comprehension"), + )); + } + // read while it is built, it would be read before it is bound let mut collection_reads = Reads::new(name); collection_reads.visit_expr(&for_loop.iter); @@ -482,51 +717,161 @@ pub(super) fn loop_to_comprehension( format!("the loop reads `{name}` while it is built"), )); } - // a comprehension does not leave its variable bound after it - let mut bound = Vec::new(); - names_in(&for_loop.target, &mut bound); - let after = TextRange::new(for_loop.end(), scope_end(context, for_loop)); + if let Some(collection_scope) = resolving_scope(index, loop_scope, name) { + let in_outer_loop = covering_node(context.parsed.syntax().into(), for_loop.range()) + .ancestors() + .skip_while(|node| node.range() == for_loop.range()) + .take_while(|node| !is_scope_node(*node)) + .any(|node| matches!(node, AnyNodeRef::StmtFor(_) | AnyNodeRef::StmtWhile(_))); + let read_while_built = loads_of(context, name).into_iter().any(|load| { + let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { + return true; + }; + if scope == loop_scope || resolving_scope(index, scope, name) != Some(collection_scope) + { + return false; + } + // a function made after the loop is not there to be called while it runs, unless + // the loop runs again after it + in_outer_loop + || made_in(context, load, loop_scope_node(context, for_loop)) < for_loop.end() + }); + if read_while_built { + return Err(Refusal::refused( + title, + format!("a function reads `{name}`, and may be called while the loop builds it"), + )); + } + } + + // a comprehension does not bind its variable where the loop is for variable in &bound { - if reads_binding_after(context, variable, after, for_loop.target.range()) { + let table = index.place_table(loop_scope); + if table.symbol_id(variable).is_some_and(|symbol| { + let symbol = table.symbol(symbol); + symbol.is_global() || symbol.is_nonlocal() + }) { return Err(Refusal::refused( title, format!( - "`{variable}` is read after the loop, which a comprehension leaves unbound" + "`{variable}` is declared outside the function, where a comprehension would not assign it" ), )); } - } - // a class body's names are not visible inside a comprehension - if in_class_body(context, for_loop) { - return Err(Refusal::refused( - title, - "a comprehension in a class body cannot see the class's names", - )); + if let Some(read) = reads_loop_binding(context, variable, loop_scope, for_loop) { + return Err(Refusal::refused( + title, + match read { + Read::After => format!( + "`{variable}` is read after the loop, which a comprehension leaves unbound" + ), + Read::Before => format!( + "`{variable}` is read above the loop, where the loop may have bound it, \ + which a comprehension does not" + ), + Read::Nested => format!( + "a function reads `{variable}`, which a comprehension leaves unbound" + ), + }, + )); + } } + // a list's or a set's element may be a walrus, a dict's key and value no looser than a + // lambda, and the iterable and the condition no looser than an `or` + let element = match elements.as_slice() { + [key, value] => format!( + "{}: {}", + comprehension_part(source, key, OperatorPrecedence::Lambda), + comprehension_part(source, value, OperatorPrecedence::Lambda) + ), + elements => elements + .iter() + .map(|element| comprehension_part(source, element, OperatorPrecedence::Assign)) + .collect(), + }; let mut comprehension = format!( "{open}{element} for {} in {}", &source[for_loop.target.range()], - &source[for_loop.iter.range()] + comprehension_part(source, &for_loop.iter, OperatorPrecedence::Or) ); if let Some(condition) = condition { comprehension.push_str(" if "); - comprehension.push_str(&source[condition.range()]); + comprehension.push_str(&comprehension_part( + source, + condition, + OperatorPrecedence::Or, + )); } comprehension.push_str(close); Ok(Plan { title, edits: vec![ Edit::range_replacement(comprehension, initial.value.range()), - Edit::range_deletion(TextRange::new( - source.full_line_end(initial.end()), - source.full_line_end(for_loop.end()), - )), + Edit::range_deletion(removed[1]), ], + name: None, }) } -/// The element `name.method(element)` adds. +/// whether `expr` calls the builtin `super` without arguments, under whatever name it is bound to, +/// or reads basedpython's `super` keyword, which stands for such a call +fn calls_zero_argument_super(context: &RefactorContext<'_>, expr: &Expr) -> bool { + struct Super<'c, 'db> { + context: &'c RefactorContext<'db>, + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for Super<'_, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + let model = &self.context.model; + match expr { + Expr::Call(call) + if call.arguments.is_empty() + && call.func.inferred_type(model).is_some_and(|ty| { + !matches!(ty, Type::BoundSuper(_)) && is_super(model.db(), ty) + }) => + { + self.found = true; + } + // the keyword is `super` written as an attribute's base, which the checker gives + // the bound super the lowering's `super()` evaluates to + Expr::Attribute(attribute) + if matches!(&*attribute.value, Expr::Name(name) if name.id.as_str() == "super") + && matches!( + attribute.value.inferred_type(model), + Some(Type::BoundSuper(_)) + ) => + { + self.found = true; + } + _ => {} + } + walk_expr(self, expr); + } + } + let mut calls = Super { + context, + found: false, + }; + calls.visit_expr(expr); + calls.found +} + +/// `expr`'s text as a part of a comprehension where nothing looser than `loosest` may stand +/// without parentheses, in them otherwise, and where it is a tuple or a generator that has none +/// of its own, which a comprehension would read as more of itself +fn comprehension_part(source: &str, expr: &Expr, loosest: OperatorPrecedence) -> String { + let text = &source[expr.range()]; + let unbracketed = matches!(expr, Expr::Tuple(tuple) if !tuple.parenthesized) + || matches!(expr, Expr::Generator(generator) if !generator.parenthesized); + if unbracketed || OperatorPrecedence::from_expr(expr) < loosest { + format!("({text})") + } else { + text.to_string() + } +} + +/// the element `name.method(element)` adds fn added<'a>(expr: &'a Expr, name: &str, method: &str) -> Option<&'a Expr> { let Expr::Call(call) = expr else { return None; @@ -545,7 +890,7 @@ fn added<'a>(expr: &'a Expr, name: &str, method: &str) -> Option<&'a Expr> { } } -/// The suite `for_loop` is written in, and its index there. +/// the suite `for_loop` is written in, and its index there fn find_suite<'a>(body: &'a [Stmt], for_loop: &ast::StmtFor) -> Option<(&'a [Stmt], usize)> { for (index, statement) in body.iter().enumerate() { if statement.range() == for_loop.range() { @@ -562,7 +907,7 @@ fn find_suite<'a>(body: &'a [Stmt], for_loop: &ast::StmtFor) -> Option<(&'a [Stm None } -/// Counts the loads of a name. +/// counts the loads of a name struct Reads<'a> { name: &'a str, count: usize, @@ -601,38 +946,65 @@ fn names_in(target: &Expr, names: &mut Vec) { } } -/// Where the scope the loop is in ends: its function, or the module. -fn scope_end(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> TextSize { - let covering = covering_node(context.parsed.syntax().into(), for_loop.range()); - covering +/// whether a node starts a scope of its own +fn is_scope_node(node: AnyNodeRef<'_>) -> bool { + matches!( + node, + AnyNodeRef::StmtFunctionDef(_) + | AnyNodeRef::StmtClassDef(_) + | AnyNodeRef::ExprLambda(_) + | AnyNodeRef::ExprListComp(_) + | AnyNodeRef::ExprSetComp(_) + | AnyNodeRef::ExprDictComp(_) + | AnyNodeRef::ExprGenerator(_) + ) +} + +/// the range of the function the loop is in, or of the module +fn loop_scope_node(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> TextRange { + covering_node(context.parsed.syntax().into(), for_loop.range()) .ancestors() .find_map(|node| match node { - AnyNodeRef::StmtFunctionDef(function) => Some(function.end()), + AnyNodeRef::StmtFunctionDef(function) => Some(function.range()), _ => None, }) - .unwrap_or_else(|| context.parsed.syntax().end()) + .unwrap_or_else(|| context.parsed.syntax().range()) } -/// Whether a read of `name` in `range` may read what the loop bound to it, the binding in -/// `target`: a read the loop's binding reaches, or a read in a nested scope, which could run at -/// any time. -fn reads_binding_after( - context: &RefactorContext<'_>, - name: &str, - range: TextRange, - target: TextRange, -) -> bool { +/// where the scope that `load` is read in is made, as it is written in the scope `within` +/// spans: the start of the outermost function, class, lambda or comprehension around the load +/// inside `within` +fn made_in(context: &RefactorContext<'_>, load: &ast::ExprName, within: TextRange) -> TextSize { + covering_node(context.parsed.syntax().into(), load.range()) + .ancestors() + .take_while(|node| node.range() != within) + .filter(|node| is_scope_node(*node)) + .last() + .map_or(load.start(), |node| node.start()) +} + +/// every read of `name` in the module: its loads, and the `del` and augmented assignment of it, +/// which read it too +fn loads_of<'a>(context: &'a RefactorContext<'_>, name: &'a str) -> Vec<&'a ast::ExprName> { struct Loads<'a> { name: &'a str, - range: TextRange, found: Vec<&'a ast::ExprName>, } impl<'a> SourceOrderVisitor<'a> for Loads<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::AugAssign(assign) = stmt + && let Expr::Name(target) = &*assign.target + && target.id.as_str() == self.name + { + self.found.push(target); + } + walk_stmt(self, stmt); + } + fn visit_expr(&mut self, expr: &'a Expr) { if let Expr::Name(name) = expr && name.id.as_str() == self.name - && name.ctx.is_load() - && self.range.contains_range(name.range()) + && (name.ctx.is_load() || name.ctx.is_del()) { self.found.push(name); } @@ -641,35 +1013,105 @@ fn reads_binding_after( } let mut loads = Loads { name, - range, found: Vec::new(), }; loads.visit_body(context.parsed.suite()); + loads.found +} + +/// where a read of `variable` outside the loop's body is +enum Read { + After, + Before, + Nested, +} + +/// a read of `variable` outside the loop's body that may read what the loop bound to it, which a +/// comprehension would not bind there: one the loop's binding reaches, one that finds the name +/// bound nowhere, where a comprehension would leave the name to the scopes around, or one in a +/// nested scope that resolves to the loop's, which could run at any time +fn reads_loop_binding( + context: &RefactorContext<'_>, + variable: &str, + loop_scope: FileScopeId, + for_loop: &ast::StmtFor, +) -> Option { let index = semantic_index(context.db, context.file); - let Some(loop_scope) = covering_node(context.parsed.syntax().into(), target) - .ancestors() - .find_map(|node| match node { - AnyNodeRef::StmtFor(for_loop) => { - index.try_expression_scope_id(&ast::ExprRef::from(&*for_loop.iter)) + let body = TextRange::new(for_loop.body.first()?.start(), for_loop.body.last()?.end()); + let target = for_loop.target.range(); + loads_of(context, variable) + .into_iter() + .filter(|load| !body.contains_range(load.range())) + .find_map(|load| { + let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { + return Some(Read::Nested); + }; + if scope != loop_scope { + return (resolving_scope(index, scope, variable) == Some(loop_scope)) + .then_some(Read::Nested); } - _ => None, + let reaching = reaching_definitions(context.db, index, scope, load); + let reads_it = reaching.maybe_unbound && reaching.definitions.is_empty() + || reaching.definitions.iter().any(|definition| { + target + .contains_range(definition.focus_range(context.db, &context.parsed).range()) + }); + reads_it.then(|| { + if load.start() >= for_loop.end() { + Read::After + } else { + Read::Before + } + }) }) - else { - return !loads.found.is_empty(); - }; - loads.found.into_iter().any(|load| { - let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { - return true; - }; - if scope != loop_scope { - return true; +} + +/// whether `expr` evaluates a `yield` in its own scope +fn yields(expr: &Expr) -> bool { + struct Yields(bool); + impl<'a> SourceOrderVisitor<'a> for Yields { + fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal { + if matches!( + node, + AnyNodeRef::ExprYield(_) | AnyNodeRef::ExprYieldFrom(_) + ) { + self.0 = true; + } + // a lambda's `yield` is the lambda's own + if self.0 || node.is_expr_lambda() { + TraversalSignal::Skip + } else { + TraversalSignal::Traverse + } } - let reaching = reaching_definitions(context.db, index, scope, load); - reaching.maybe_unbound && reaching.definitions.is_empty() - || reaching.definitions.iter().any(|definition| { - target.contains_range(definition.focus_range(context.db, &context.parsed).range()) - }) - }) + } + let mut yields = Yields(false); + walk_node(&mut yields, AnyNodeRef::from(expr)); + yields.0 +} + +/// whether `expr` assigns `variable` with a walrus +fn assigns_with_walrus(expr: &Expr, variable: &str) -> bool { + struct Walrus<'a> { + variable: &'a str, + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for Walrus<'_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Named(named) = expr + && matches!(&*named.target, Expr::Name(target) if target.id.as_str() == self.variable) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut walrus = Walrus { + variable, + found: false, + }; + walrus.visit_expr(expr); + walrus.found } fn in_class_body(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> bool { @@ -689,6 +1131,7 @@ fn in_class_body(context: &RefactorContext<'_>, for_loop: &ast::StmtFor) -> bool #[cfg(test)] mod tests { use insta::assert_snapshot; + use ruff_python_ast::PythonVersion; use crate::refactor::RefactorKind; use crate::refactor::test_support::RefactorTest; @@ -741,7 +1184,7 @@ mod tests { "); } - /// `not x > 1` is not `x <= 1` for a float that is `nan`. + /// `not x > 1` is not `x <= 1` for a float that is `nan` #[test] fn invert_if_keeps_not_for_a_float() { let test = RefactorTest::python( @@ -812,7 +1255,188 @@ mod tests { "); } - /// `a and b` is `a` or `b`, which the rewrite would make a `bool`. + /// the rewritten `or` binds looser than the `and` it is an operand of, so it is + /// parenthesised, on either side of the `and` + #[test] + fn de_morgan_parenthesises_an_or_under_an_and() { + let test = RefactorTest::python( + r#" + def f(a, b, c): + if c and not (a and b): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b, c): + if c and (not a or not b): + return 1 + "); + let test = RefactorTest::python( + r#" + def f(a, b, c): + if not (a and b) and c: + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b, c): + if (not a or not b) and c: + return 1 + "); + } + + /// an `and` binds tighter than the `or` it is an operand of, and an `or` among the operands + /// of another `or` groups the same either way, so neither is parenthesised + #[test] + fn de_morgan_leaves_an_operand_unparenthesised_where_it_binds_tightly_enough() { + let test = RefactorTest::python( + r#" + def f(a, b, c): + if c or not (a or b): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b, c): + if c or not a and not b: + return 1 + "); + let test = RefactorTest::python( + r#" + def f(a, b, c): + if not (a and not (b or c)): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, b, c): + if not a or b or c: + return 1 + "); + } + + /// a walrus taken out of its `not` keeps parentheses among the operands of an `or` + #[test] + fn de_morgan_parenthesises_a_walrus_operand() { + let test = RefactorTest::python( + r#" + def f(a, g): + if a and not (x := g()): + return x + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @" + Apply De Morgan's law + --- + + def f(a, g): + if not (not a or (x := g())): + return x + "); + } + + /// each operand of a flipped comparison keeps the parentheses it is written in + #[test] + fn invert_if_keeps_a_comparison_operand_s_parentheses() { + let test = RefactorTest::python( + r#" + def f(a, b): + if (a or b) is None: + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(a, b): + if (a or b) is not None: + return 2 + else: + return 1 + "); + let test = RefactorTest::python( + r#" + def f(g): + if (x := g()) is None: + return 1 + else: + return x + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(g): + if (x := g()) is not None: + return x + else: + return 1 + "); + } + + /// a condition that loses its `not` loses the parentheses it needed only under the `not`, + /// but keeps them where it spans lines + #[test] + fn invert_if_takes_off_a_not() { + let test = RefactorTest::python( + r#" + def f(a, b): + if not (a or b): + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(a, b): + if a or b: + return 2 + else: + return 1 + "); + let test = RefactorTest::python( + r#" + def f(a, b): + if not (a or + b): + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(a, b): + if (a or + b): + return 2 + else: + return 1 + "); + } + + /// `a and b` is `a` or `b`, which the rewrite would make a `bool` #[test] fn de_morgan_on_a_value_is_refused() { let test = RefactorTest::python( @@ -887,7 +1511,7 @@ mod tests { "); } - /// A comprehension leaves `item` unbound, which the loop left bound to the last item. + /// a comprehension leaves `item` unbound, which the loop left bound to the last item #[test] fn a_loop_whose_variable_is_read_after_it_is_refused() { let test = RefactorTest::python( @@ -916,8 +1540,222 @@ mod tests { ); assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (the loop reads `seen` while it is built)"); } - /// A later loop binds `item` afresh before reading it, so the first loop's `item` is not - /// read after it. + /// a closure made before the loop reads the `item` the loop leaves bound, which a + /// comprehension would not bind in the function at all + #[test] + fn a_loop_whose_variable_a_closure_reads_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + last = lambda: item + result = [] + for item in items: + result.append(item) + return result, last() + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (a function reads `item`, which a comprehension leaves unbound)"); + } + + /// inside another loop, a read of `item` above the loop reads what the loop bound on the + /// round before + #[test] + fn a_loop_whose_variable_is_read_before_it_on_the_next_round_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int], rounds: int): + for _ in range(rounds): + if rounds > 1: + print(item) + result = [] + for item in items: + result.append(item) + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (`item` is read above the loop, where the loop may have bound it, which a comprehension does not)"); + } + + /// a function that reads the collection may be called while the loop builds it, where a + /// comprehension has not bound it yet + #[test] + fn a_loop_whose_collection_a_closure_reads_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + def counted(item): + print(len(result)) + return item + result = [] + for item in items: + result.append(counted(item)) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (a function reads `result`, and may be called while the loop builds it)"); + } + + /// `yield` is not allowed inside a comprehension + #[test] + fn a_loop_that_adds_what_it_yields_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + result.append((yield item)) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (it adds what a `yield` evaluates to, which a comprehension cannot hold)"); + } + + /// a walrus may not rebind a comprehension's variable, nor be part of its iterable + #[test] + fn a_loop_with_a_walrus_a_comprehension_rejects_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + result.append(item := item + 1) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (a walrus assigns `item`, which a comprehension does not allow)"); + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in (chosen := items): + result.append(item) + return result, chosen + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (its iterable assigns with a walrus, which a comprehension does not allow)"); + } + + /// the loop's comments have no place in a comprehension, unless they are inside what it + /// keeps + #[test] + fn a_loop_with_a_comment_is_refused() { + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + # doubled, since the total is halved later + result.append(item * 2) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (the comment on line 5 has no place in a comprehension)"); + let test = RefactorTest::python( + r#" + def f(items: list[int]): + result = [] + for item in items: + result.append(min( + item, # never above the cap + 10, + )) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to list comprehension + --- + + def f(items: list[int]): + result = [min( + item, # never above the cap + 10, + ) for item in items] + return result + "); + } + + /// a set is built by `add`, and a dict by item assignment, whatever its key reads like + #[test] + fn a_loop_that_builds_a_set_becomes_a_set_comprehension() { + let test = RefactorTest::python( + r#" + def f(items: list[str]): + seen = set() + for item in items: + seen.add(item + ": ") + return seen + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @r#" + Convert loop to set comprehension + --- + + def f(items: list[str]): + seen = {item + ": " for item in items} + return seen + "#); + let test = RefactorTest::python( + r#" + def f(items: list[str]): + labels = {} + for item in items: + labels[item]=item + return labels + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to dict comprehension + --- + + def f(items: list[str]): + labels = {item: item for item in items} + return labels + "); + } + + /// what python would read as more of the comprehension keeps parentheses: a generator + /// passed bare, a condition with a walrus, an iterable that is a conditional expression + #[test] + fn a_comprehension_part_keeps_the_parentheses_it_needs() { + let test = RefactorTest::python( + r#" + def f(groups: list[list[int]], spare: list[int], g): + result = [] + for group in groups if groups else spare: + if (found := g(group)): + result.append(sum(x for x in group) for _ in found) + return result + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to list comprehension + --- + + def f(groups: list[list[int]], spare: list[int], g): + result = [(sum(x for x in group) for _ in found) for group in (groups if groups else spare) if (found := g(group))] + return result + "); + let test = RefactorTest::python( + r#" + def f(items: list[int]): + pairs = {} + for item in items: + pairs[item, item] = item, item + return pairs + "#, + ); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to dict comprehension + --- + + def f(items: list[int]): + pairs = {(item, item): (item, item) for item in items} + return pairs + "); + } + + /// a later loop binds `item` afresh before reading it, so the first loop's `item` is not + /// read after it #[test] fn a_variable_rebound_before_it_is_read_again_does_not_refuse() { let test = RefactorTest::python( @@ -944,4 +1782,108 @@ mod tests { return result, names "); } + + /// a comment between the operands has nowhere to go once they are rewritten one by one + #[test] + fn de_morgan_with_a_comment_between_the_operands_is_refused() { + let test = RefactorTest::python( + r#" + def f(a, b): + if not ( + a # the first + or b + ): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @"refused: Apply De Morgan's law (the comment on line 4 would be lost)"); + let test = RefactorTest::python( + r#" + def f(a, b): + if ( + a # the first + and b + ): + return 1 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::DeMorgan), @"refused: Apply De Morgan's law (the comment on line 4 would be lost)"); + } + + /// the `not (...)` the inversion takes off would take a comment inside it with it, while + /// one inside the operand it keeps stays + #[test] + fn invert_if_keeps_or_refuses_a_comment_in_its_condition() { + let test = RefactorTest::python( + r#" + def f(a, b): + if not ( # either + a or b + ): + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @"refused: Invert `if` condition (the comment on line 3 would be lost)"); + let test = RefactorTest::python( + r#" + def f(a, b): + if not ( + a # the first + or b + ): + return 1 + else: + return 2 + "#, + ); + assert_snapshot!(test.apply(RefactorKind::InvertIf), @" + Invert `if` condition + --- + + def f(a, b): + if (a # the first + or b): + return 2 + else: + return 1 + "); + } + + /// before 3.12 a comprehension runs as a function of its own, where a zero-argument + /// `super()` is passed the iterator rather than the method's receiver + #[test] + fn a_loop_calling_zero_argument_super_needs_python_3_12() { + let source = r#" + class Base: + def scale(self, x): + return x + + + class Child(Base): + def scaled(self, items): + result = [] + for item in items: + result.append(super().scale(item)) + return result + "#; + let test = RefactorTest::python(source).with_python_version(PythonVersion::PY311); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @"refused: Convert loop to list comprehension (it calls `super()` without arguments, which in a comprehension before python 3.12 is not passed the method's receiver)"); + let test = RefactorTest::python(source).with_python_version(PythonVersion::PY312); + assert_snapshot!(test.apply(RefactorKind::LoopToComprehension), @" + Convert loop to list comprehension + --- + + class Base: + def scale(self, x): + return x + + + class Child(Base): + def scaled(self, items): + result = [super().scale(item) for item in items] + return result + "); + } } diff --git a/crates/ty_ide/src/refactor/return_annotation.rs b/crates/ty_ide/src/refactor/return_annotation.rs index b4aab4554a..91efc9684c 100644 --- a/crates/ty_ide/src/refactor/return_annotation.rs +++ b/crates/ty_ide/src/refactor/return_annotation.rs @@ -69,6 +69,7 @@ pub(super) fn plan(context: &RefactorContext<'_>, range: TextRange) -> Result {annotation}`"), edits, + name: None, }) } diff --git a/crates/ty_ide/src/refactor/split_join.rs b/crates/ty_ide/src/refactor/split_join.rs index 8dd635bedf..6a53369dfd 100644 --- a/crates/ty_ide/src/refactor/split_join.rs +++ b/crates/ty_ide/src/refactor/split_join.rs @@ -125,6 +125,7 @@ pub(super) fn plan( return Ok(Plan { title: title.to_string(), edits: vec![Edit::range_replacement(text, inside)], + name: None, }); } Err(Refusal::NotApplicable) diff --git a/crates/ty_server/src/server/api/requests/code_action.rs b/crates/ty_server/src/server/api/requests/code_action.rs index 31e91084f4..cc236f6e5c 100644 --- a/crates/ty_server/src/server/api/requests/code_action.rs +++ b/crates/ty_server/src/server/api/requests/code_action.rs @@ -328,6 +328,11 @@ pub(crate) struct RefactorData { pub(crate) version: crate::document::DocumentVersion, pub(crate) refactor: String, pub(crate) range: types::Range, + /// set when the action is resolved, for a refactoring that makes up a name, the new + /// function's or variable's: where the name is in the document once the action's edit is + /// applied, at the place the rewritten code uses it, for an editor to start renaming it there + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) rename: Option, } impl RefactorData { @@ -337,6 +342,7 @@ impl RefactorData { version: snapshot.document().version(), refactor: kind.id().to_string(), range, + rename: None, } } } diff --git a/crates/ty_server/src/server/api/requests/code_action_resolve.rs b/crates/ty_server/src/server/api/requests/code_action_resolve.rs index b670e40a56..c8eefa3646 100644 --- a/crates/ty_server/src/server/api/requests/code_action_resolve.rs +++ b/crates/ty_server/src/server/api/requests/code_action_resolve.rs @@ -2,10 +2,12 @@ use std::borrow::Cow; use std::str::FromStr; use lsp_types::{self as types, CodeActionResolveRequest, Uri}; +use ruff_source_file::LineIndex; +use ruff_text_size::Ranged; use ty_ide::RefactorKind; use ty_project::{ProjectDatabase, SemanticDb as _}; -use crate::document::RangeExt; +use crate::document::{RangeExt, text_range_to_lsp_range}; use crate::server::api::RequestHandler; use crate::server::api::requests::code_action::{RefactorData, to_lsp_edits}; use crate::server::api::traits::{BackgroundDocumentRequestHandler, RetriableRequestHandler}; @@ -71,6 +73,38 @@ impl BackgroundDocumentRequestHandler for CodeActionResolveRequestHandler { match ty_ide::refactor(db, db.program_file(file), kind, range) { Ok(refactor) => { + if let Some(name) = refactor.name { + // the name is in the text the edits leave behind, so it is a range in that text + let source = ruff_db::source::source_text(db, file); + let mut edited = source.as_str().to_string(); + let mut applied: Vec<_> = refactor + .edits + .iter() + .filter(|edit| edit.file == file) + .map(|edit| &edit.edit) + .collect(); + applied.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + for edit in applied { + edited.replace_range( + edit.range().to_std_range(), + edit.content().unwrap_or_default(), + ); + } + let range = text_range_to_lsp_range( + name, + &edited, + &LineIndex::from_source_text(&edited), + snapshot.encoding(), + ); + let data = RefactorData { + rename: Some(types::Location { + uri: snapshot.document().uri().clone(), + range, + }), + ..data + }; + action.data = serde_json::to_value(data).ok(); + } action.edit = Some(types::WorkspaceEdit { changes: to_lsp_edits(db, snapshot.encoding(), refactor.edits), ..types::WorkspaceEdit::default() diff --git a/crates/ty_server/tests/e2e/refactors.rs b/crates/ty_server/tests/e2e/refactors.rs index 8fc6d7e605..d98ecd55f9 100644 --- a/crates/ty_server/tests/e2e/refactors.rs +++ b/crates/ty_server/tests/e2e/refactors.rs @@ -243,3 +243,46 @@ fn extraction_at_a_caret_offers_the_expressions_around_it() -> Result<()> { Ok(()) } + +/// an extraction, once resolved, says in its `data` where the name it made up is in the text its +/// edit leaves, where the rewritten code uses it, for an editor to rename it in place +#[test] +fn a_resolved_extraction_says_where_its_new_name_is() -> Result<()> { + let (mut server, file) = server(true)?; + + let selection = Range::new(Position::new(1, 8), Position::new(1, 17)); + let actions = request( + &mut server, + file, + selection, + Some(vec![CodeActionKind::new("refactor.extract.variable")]), + Some(CodeActionTriggerKind::Invoked), + ); + let [action] = actions.as_slice() else { + panic!("expected exactly the extraction, got {actions:#?}"); + }; + let resolved = server.send_request_await::(action.clone()); + let data = resolved + .data + .expect("a resolved refactoring keeps its data"); + // the new binding goes above, so line 1 becomes line 2, `x = `, and the name is the + // one the title gives + let range = &data["rename"]["range"]; + assert_eq!( + range["start"], + serde_json::json!({ "line": 2, "character": 8 }) + ); + assert_eq!(range["end"]["line"], 2); + let length = range["end"]["character"].as_u64().unwrap_or_default() - 8; + let name = resolved + .title + .split('`') + .nth(1) + .expect("the title quotes the name"); + assert_eq!(length, name.len() as u64, "{}", resolved.title); + assert_eq!( + data["rename"]["uri"], + serde_json::json!(server.file_uri(file)) + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 647b70fc4e..56c3880322 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -545,6 +545,17 @@ one that asks while the caret moves is only offered what applies | convert quotes | `refactor.rewrite.flipQuotes` | a string or bytes literal | | loop to comprehension | `refactor.rewrite.loopToComprehension` | a loop filling an empty collection | +an extraction makes up a name, which its user will want to choose. once its +action is resolved, its `data` says where the name is: `rename`, a `uri` and a +`range` in the document as the action's edit leaves it, at the place the +rewritten code uses the name, where the selection was. an editor applies the +edit and starts renaming there, as TypeScript's refactorings hand over a +rename location: + +```json +{ "data": { "rename": { "uri": "file:///src/shapes.by", "range": { "start": { "line": 13, "character": 21 }, "end": { "line": 13, "character": 30 } } } } } +``` + inlining is offered on the variable's assignment and on any of its reads alike, and replaces every read either way From 161c6c40c7a4f4ca0e7a1dc4f5640671f795d30f Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:03:39 +1000 Subject: [PATCH 052/110] parenthesise an inlined value as an operator's operand only where python's precedence and associativity would read it otherwise, writing math.pi * radius ** 2 and scale * math.pi * radius ** 2 + 1 bare and c - (a - b) with them --- crates/ty_ide/src/refactor/inline_variable.rs | 117 +++++++++++++++++- docs/basedpython/features/editor.md | 5 +- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/crates/ty_ide/src/refactor/inline_variable.rs b/crates/ty_ide/src/refactor/inline_variable.rs index cddd3771e5..f50dc6b43c 100644 --- a/crates/ty_ide/src/refactor/inline_variable.rs +++ b/crates/ty_ide/src/refactor/inline_variable.rs @@ -658,6 +658,9 @@ fn needs_parentheses( if atomic { return false; } + if let Some(needed) = operand_needs_parentheses(value, parent, read) { + return needed; + } if !is_free_position(parent, read) { return true; } @@ -671,6 +674,46 @@ fn needs_parentheses( } } +/// whether `value`, written as the operand `read` of the operator `parent`, needs parentheses +/// for the operator to take it whole, as python's precedence and associativity read it: none for +/// an operand that binds tighter, `math.pi * radius ** 2`, and for the left operand of an operator +/// as tight that groups from the left, `scale * math.pi * radius ** 2 + 1`; some for the right +/// one, `a - (b - c)`, which would otherwise group the other way, and for any comparison inside +/// another, which python would chain. `None` where `parent` is no operator +fn operand_needs_parentheses( + value: &Expr, + parent: AnyNodeRef<'_>, + read: &ast::ExprName, +) -> Option { + let precedence = OperatorPrecedence::from_expr(value); + let range = read.range(); + Some(match parent { + AnyNodeRef::ExprBinOp(operation) => { + let operator = OperatorPrecedence::from(operation.op); + let is_left = operation.left.range() == range; + match precedence.cmp(&operator) { + std::cmp::Ordering::Greater => false, + std::cmp::Ordering::Less => true, + // `**` groups from the right, everything else from the left + std::cmp::Ordering::Equal => is_left == operator.is_right_associative(), + } + } + AnyNodeRef::ExprBoolOp(operation) => { + let operator = OperatorPrecedence::from(operation.op); + let is_first = operation + .values + .first() + .is_some_and(|first| first.range() == range); + precedence < operator || (precedence == operator && !is_first) + } + AnyNodeRef::ExprUnaryOp(operation) => precedence < OperatorPrecedence::from(operation.op), + AnyNodeRef::ExprCompare(_) => { + precedence <= OperatorPrecedence::ComparisonsMembershipIdentity + } + _ => return None, + }) +} + /// Whether a read is a whole expression on its own — a statement's value, an /// argument, an element — rather than an operand of something that binds. fn is_free_position(parent: AnyNodeRef<'_>, read: &ast::ExprName) -> bool { @@ -785,23 +828,36 @@ mod tests { /// wherever it is evaluated, so it is inlined into both reads #[test] fn value_over_builtin_scalars_inlines_into_every_read() { - assert_snapshot!(inline( + assert_snapshot!(RefactorTest::basedpython( " - def f(name: str, count: int): + def f(name: final str, count: final int): greeting = 'hello ' + name print(greeting * count) return greeting ", - ), @" + ).apply(RefactorKind::InlineVariable), @" Inline variable `greeting` --- - def f(name: str, count: int): + def f(name: final str, count: final int): print(('hello ' + name) * count) return 'hello ' + name "); } + /// a `str` may be an instance of a subclass, whose `+` is its own and may have effects + #[test] + fn value_over_a_str_that_may_be_a_subclass_is_not_inlined_into_every_read() { + assert_snapshot!(inline( + " + def f(name: str): + greeting = 'hello ' + name + print(greeting) + return greeting + ", + ), @"refused: Inline variable `greeting` (its value would be evaluated once for each read, and evaluating it may have effects)"); + } + /// a module's attribute can be assigned between the assignment and a read #[test] fn value_reading_a_module_attribute_is_not_inlined_into_every_read() { @@ -851,6 +907,59 @@ mod tests { "); } + /// an operand that binds tighter than its operator, or as tightly on the side the operator + /// groups from, is written without parentheses + #[test] + fn operator_value_is_bare_where_precedence_keeps_it_whole() { + assert_snapshot!(RefactorTest::basedpython( + " + import math + + def f(radius: final float): + square = radius ** 2 + return math.pi * square + ", + ).apply(RefactorKind::InlineVariable), @" + Inline variable `square` + --- + + import math + + def f(radius: final float): + return math.pi * radius ** 2 + "); + assert_snapshot!(RefactorTest::basedpython( + " + import math + + def f(radius: final float, scale: final float): + area = scale * math.pi * radius ** 2 + return area + 1 + ", + ).apply(RefactorKind::InlineVariable), @" + Inline variable `area` + --- + + import math + + def f(radius: final float, scale: final float): + return scale * math.pi * radius ** 2 + 1 + "); + assert_snapshot!(RefactorTest::basedpython( + " + def f(a: final int, b: final int, c: final int): + x = a - b + return x - c, c - x, -x, x < c, x ** 2, 2 ** x + ", + ).apply(RefactorKind::InlineVariable), @" + Inline variable `x` + --- + + def f(a: final int, b: final int, c: final int): + return a - b - c, c - (a - b), -(a - b), a - b < c, (a - b) ** 2, 2 ** (a - b) + "); + } + #[test] fn value_with_effects_is_refused_for_two_reads() { assert_snapshot!(inline( diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 56c3880322..9054162b67 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -557,7 +557,10 @@ rename location: ``` inlining is offered on the variable's assignment and on any of its reads alike, and -replaces every read either way +replaces every read either way. the value is parenthesised only where python +would read it otherwise: inlining `square = radius ** 2` into `math.pi * square` +writes `math.pi * radius ** 2`, and `x = a - b` into `c - x` writes +`c - (a - b)` moving an expression can change when it runs, so that is what most refusals are about. a value made of names and literals, and of python's own operators over From 056b9b6bab64b4ba25c28247d8ddae914e99d4b9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:06:56 +1000 Subject: [PATCH 053/110] say in by/syntaxOutline what a method's first parameter receives, instance or class, class for __new__, which is static but passed its class, and nothing for another static method, so an editor writing self or cls need not know python's special names --- crates/ty_ide/src/syntax_outline.rs | 64 ++++++++++++------- .../src/server/api/requests/syntax_outline.rs | 10 ++- docs/basedpython/features/editor.md | 5 +- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index e8247ec2b0..a2d91175d4 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -67,8 +67,13 @@ pub struct OutlineStatement { pub parameters: Vec, /// for a `def` a class body makes, what its first parameter is passed pub method: Option, - /// For a simple statement, the expression Enter may break across lines by wrapping it in - /// parentheses; see [`wrappable`]. + /// for a `def` a class body makes, what its first parameter is, the receiver an editor writes + /// when the parameter list is opened: `Instance` (`self`) or `Class` (`cls`). `__new__` is a + /// static method that is passed its class explicitly, so its receiver is `Class`; any other + /// static method has none + pub receiver: Option, + /// for a simple statement, the expression Enter may break across lines by wrapping it in + /// parentheses; see [`wrappable`] pub wrap: Option, } @@ -86,8 +91,8 @@ pub struct OutlineClause { pub range: TextRange, /// The statements of the suite. pub body: Vec, - /// The expression of the header Enter may break across lines by wrapping it in parentheses; - /// see [`wrappable`]. + /// the expression of the header Enter may break across lines by wrapping it in parentheses; + /// see [`wrappable`] pub wrap: Option, } @@ -265,6 +270,15 @@ impl OutlineBuilder<'_> { } _ => None, }; + let receiver = match (method, stmt) { + (Some(MethodKind::Static), Stmt::FunctionDef(function)) + if function.name.as_str() == "__new__" => + { + Some(MethodKind::Class) + } + (Some(MethodKind::Static), _) => None, + (kind, _) => kind, + }; OutlineStatement { range: stmt.range(), clauses: self.clauses(stmt, in_class), @@ -273,6 +287,7 @@ impl OutlineBuilder<'_> { name: Self::name(stmt), parameters: Self::parameters(stmt), method, + receiver, wrap: statement_expression(stmt) .filter(|expr| wrappable(expr)) .map(Ranged::range), @@ -635,10 +650,10 @@ impl OutlineBuilder<'_> { } /// One clause of a compound statement, as [`OutlineBuilder::clause_heads`] finds it. -/// The one expression of a simple statement that Enter could break: an assignment's value, a +/// the one expression of a simple statement that Enter could break: an assignment's value, a /// `return`'s, an expression statement, a `raise` without `from`, an `assert` without a message -/// and a `type` alias's value. A statement with two expressions, such as `raise E from cause`, -/// has none, since one pair of parentheses cannot hold both. +/// and a `type` alias's value. a statement with two expressions, such as `raise E from cause`, +/// has none, since one pair of parentheses cannot hold both fn statement_expression(stmt: &Stmt) -> Option<&Expr> { match stmt { Stmt::Expr(stmt) => Some(&stmt.value), @@ -655,8 +670,8 @@ fn statement_expression(stmt: &Stmt) -> Option<&Expr> { } } -/// The expressions clause headers of `stmt` hold that Enter could break: the condition of an -/// `if`, an `elif` and a `while`, the iterable of a `for` and the subject of a `match`. +/// the expressions clause headers of `stmt` hold that Enter could break: the condition of an +/// `if`, an `elif` and a `while`, the iterable of a `for` and the subject of a `match` fn header_expressions(stmt: &Stmt) -> Vec<&Expr> { match stmt { Stmt::If(stmt) if stmt.pattern.is_none() => std::iter::once(&*stmt.test) @@ -673,14 +688,14 @@ fn header_expressions(stmt: &Stmt) -> Vec<&Expr> { } } -/// Whether `expr` means the same wrapped in parentheses, so that Enter inside it can break it -/// over lines by wrapping it rather than with a backslash. +/// whether `expr` means the same wrapped in parentheses, so that Enter inside it can break it +/// over lines by wrapping it rather than with a backslash /// -/// Every python expression in these positions does — `x = (a + b)`, `if (a and b):`, -/// `return (1, 2)`. What does not is basedpython's: a statement used as an expression, a type +/// every python expression in these positions does — `x = (a + b)`, `if (a and b):`, +/// `return (1, 2)`. what does not is basedpython's: a statement used as an expression, a type /// written in a form only an annotation reads, and a tuple whose parentheses are its syntax, /// such as an anonymous named tuple, all of which are left alone, along with a starred -/// expression on its own. +/// expression on its own fn wrappable(expr: &Expr) -> bool { struct Unwrappable(bool); impl<'a> SourceOrderVisitor<'a> for Unwrappable { @@ -1699,7 +1714,10 @@ else: let method = statement .method .map_or("not a method", |kind| kind.as_str()); - Some(format!("{name}: {method}")) + let receiver = statement.receiver.map_or(String::new(), |kind| { + format!(", receives the {}", kind.as_str()) + }); + Some(format!("{name}: {method}{receiver}")) }, &mut out, ); @@ -1731,16 +1749,16 @@ def free(): ... " ), @" Shape: not a method - area: instance - unit: class + area: instance, receives the instance + unit: class, receives the class origin: static zero: static - build: class - size: instance - __new__: static - __init_subclass__: class - maybe: instance - outer: instance + build: class, receives the class + size: instance, receives the instance + __new__: static, receives the class + __init_subclass__: class, receives the class + maybe: instance, receives the instance + outer: instance, receives the instance inner: not a method free: not a method "); diff --git a/crates/ty_server/src/server/api/requests/syntax_outline.rs b/crates/ty_server/src/server/api/requests/syntax_outline.rs index 47e9832393..f444388393 100644 --- a/crates/ty_server/src/server/api/requests/syntax_outline.rs +++ b/crates/ty_server/src/server/api/requests/syntax_outline.rs @@ -97,7 +97,12 @@ pub(crate) struct Statement { /// is passed #[serde(default, skip_serializing_if = "Option::is_none")] method: Option, - /// On a simple statement: the expression Enter may break by wrapping it in parentheses. + /// on a `def` a class body makes: `instance` or `class`, what its first parameter is, the + /// `self` or `cls` an editor writes when the parameter list is opened. `class` for `__new__`, + /// which is static but passed its class; absent for any other static method + #[serde(default, skip_serializing_if = "Option::is_none")] + receiver: Option, + /// on a simple statement: the expression Enter may break by wrapping it in parentheses #[serde(default, skip_serializing_if = "Option::is_none")] wrap: Option, } @@ -120,7 +125,7 @@ pub(crate) struct Clause { /// The suite's statements. #[serde(default, skip_serializing_if = "Vec::is_empty")] body: Vec, - /// The header's expression Enter may break by wrapping it in parentheses. + /// the header's expression Enter may break by wrapping it in parentheses #[serde(default, skip_serializing_if = "Option::is_none")] wrap: Option, } @@ -290,6 +295,7 @@ impl Converter<'_> { .filter_map(|range| self.range(*range)) .collect(), method: statement.method.map(|kind| kind.as_str().to_string()), + receiver: statement.receiver.map(|kind| kind.as_str().to_string()), wrap: statement.wrap.and_then(|range| self.range(range)), }) }) diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 9054162b67..e317cce179 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1599,7 +1599,10 @@ character of the fragment. the outline also carries the name each `def` and from `@staticmethod` and `@classmethod`, `static def` and `class def`, and the methods python makes so by name, `__new__` static (it is handed its class explicitly, as its first argument) and `__init_subclass__` and `__class_getitem__` -class methods; a plain function has none. +class methods; a plain function has none. beside it, `receiver` says what the +first parameter is, the `self` or `cls` an editor writes when the parameter +list is opened: `instance` or `class`, `class` for `__new__` too, and absent for +any other static method. a line broken inside an expression needs the expression in brackets, or a backslash, to go on parsing. `wrap` names the expression Enter may break by From 6aa1962f243335cab2c6dc7c83750e90d76ffc62 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:33:51 +1000 Subject: [PATCH 054/110] report a pytest run under by run to an IDE's test runner as TeamCity service messages when BY_TEST_REPORTER=teamcity, suites named by the .by files, each test with its node id and its .by line, a failing == with its actual and expected, and an error raised importing a test module at its .by line rather than the staged .py's --- crates/ty/docs/environment.md | 9 + crates/ty/src/by_commands.rs | 377 ++++++++++++++++++++- crates/ty/tests/by_e2e.rs | 206 +++++++++-- crates/ty_static/src/env_vars.rs | 8 + docs/basedpython/development/sourcemaps.md | 9 +- docs/basedpython/features/editor.md | 38 +++ 6 files changed, 613 insertions(+), 34 deletions(-) diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index fce846bba4..331937ee31 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -13,6 +13,15 @@ not open the socket that makes it reachable. Equivalent to the `--no-server` command-line argument. +### `BY_TEST_REPORTER` + +report the tests pytest runs under `by run` as service messages, `TeamCity`'s protocol + +set this to `teamcity` for an IDE's test runner to read the run: a suite for each test +file, named by the `.by` it was transpiled from, and one for each class in it, each test +with its pytest node id and where it is in the `.by`, and a failing `==` with what each +side was. any other value is refused + ### `TY_CONFIG_FILE` Path to a `ty.toml` configuration file to use. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 0c0a254dbe..d92c44e2eb 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1892,11 +1892,20 @@ fn write_traceback_runtime( /// transpiled from, so that a failure is reported at the `.by` line that failed const BY_PYTEST_PLUGIN_FILENAME: &str = "_by_pytest.py"; -const BY_PYTEST_PLUGIN_SRC: &str = r#"# generated by `by run` — reports a failing test at the `.by` line that failed, for pytest +const BY_PYTEST_PLUGIN_SRC: &str = r###"# generated by `by run` — reports a test at the `.by` line it failed at, for pytest +import os +import pprint +import sys +import traceback import types +import pytest + import _by_runner +# set to `teamcity` by an IDE that reads the run as TeamCity service messages +_REPORTER = "BY_TEST_REPORTER" + def pytest_pycollect_makeitem(collector): # called for each name of a test module or class pytest collects, once the module is @@ -1908,7 +1917,312 @@ def pytest_plugin_registered(plugin): # a `conftest.py` is a plugin, and its fixtures run in the tests if isinstance(plugin, types.ModuleType): _by_runner.reline_module(plugin) -"#; + + +def pytest_exception_interact(node, call, report): + # an error raised while a test module is imported comes from its top level, which + # ran as pytest compiled it, at lines of the generated python, before + # `reline_module` saw the module. it is reported as `by run` reports an error in + # a program, as python prints it with each frame at its `.by` line + if not isinstance(report, pytest.CollectReport) or call.excinfo is None: + return + error = call.excinfo.value + if isinstance(error, pytest.Collector.CollectError): + # pytest words the other errors it wraps, which are kept: a module that skips + # without `allow_module_level`, or a file imported under another's name + error = error.__cause__ + if not isinstance(error, ImportError): + return + if _by_runner.runs_generated_lines(error.__traceback__): + report.longrepr = _Traceback(error) + + +class _Traceback: + """An error as `by run` prints it, standing as a report's `longrepr`, which pytest + writes out with `toterminal` and summarises in a line by `reprcrash.message`.""" + + def __init__(self, error): + self._text = _by_runner.format_exception(type(error), error, error.__traceback__) + self.reprcrash = types.SimpleNamespace( + message="".join(traceback.format_exception_only(type(error), error)).strip() + ) + + def toterminal(self, out): + out.line(self._text.rstrip("\n")) + + def __str__(self): + return self._text + + +def pytest_configure(config): + reporter = os.environ.get(_REPORTER) + if not reporter: + return + if reporter != "teamcity": + raise pytest.UsageError(f"{_REPORTER} is {reporter!r}, and the one reporter is teamcity") + config.pluginmanager.register(_TeamCity(config), "_by_teamcity") + + +def _escape(value): + # https://www.jetbrains.com/help/teamcity/service-messages.html#Escaped+Values + return ( + value.replace("|", "||") + .replace("'", "|'") + .replace("\n", "|n") + .replace("\r", "|r") + .replace("[", "|[") + .replace("]", "|]") + .replace("\u0085", "|x") + .replace("
", "|l") + .replace("
", "|p") + ) + + +def _shown(left, right): + # two strings are compared as their text, anything else as python writes it + if isinstance(left, str) and isinstance(right, str): + return left, right + return pprint.pformat(left), pprint.pformat(right) + + +class _TeamCity: + """Reports the run as the TeamCity service messages an IDE's test runner reads. + + Each test file is a suite named by the project file it was transpiled from or + copied out of, relative to the project, and each class collected in it a suite + inside it, so the tree is the project's rather than the staged tree's. A test + carries its pytest node id as `metainfo`, which is how it is run again. Every + `locationHint` is `file://` and an absolute path, with `:` and a 1-based line + for a class or a test, of the project file. + + A failing `==` is reported with what each side was, `actual` the left and + `expected` the right, as PyCharm reports python's. + """ + + def __init__(self, config): + self._config = config + self._capture = config.pluginmanager.getplugin("capturemanager") + # the suites open, outermost first, each its collector's node id and name + self._open = [] + # the name of the test running, and its phases, each pytest's report of it + self._name = None + self._phases = [] + # each `==` an assertion failing in the phase running compared, which + # `_comparison` matches to the assertion it failed in + self._compared = [] + # the comparison each phase failed in, by node id and phase + self._comparisons = {} + + def _message(self, kind, **attributes): + text = "".join( + f" {key}='{_escape(value)}'" for key, value in attributes.items() if value is not None + ) + # pytest captures the program's output while a test runs, and a message is + # never written while it does: it would land in the test's output + if self._capture is None: + self._write(f"##teamcity[{kind}{text}]\n") + else: + with self._capture.global_and_fixture_disabled(): + self._write(f"##teamcity[{kind}{text}]\n") + + @staticmethod + def _write(text): + sys.stdout.write(text) + sys.stdout.flush() + + def _location(self, path, line=None): + path, line = _by_runner.project_location(os.fspath(path), line) + url = "file://" + os.path.abspath(path) + return url if line is None else f"{url}:{line}" + + def _file_name(self, path): + # the project file, relative to the project when it is in it + project_file, _ = _by_runner.project_location(os.fspath(path), None) + relative = os.path.relpath(os.path.abspath(project_file), _by_runner.ROOT) + if relative.split(os.sep)[0] == os.pardir: + return project_file + return relative.replace(os.sep, "/") + + def pytest_sessionstart(self, session): + self._message("enteredTheMatrix") + + def pytest_collectreport(self, report): + # a file or directory that failed to collect, or skipped as a whole, is a test + # of its own: there is no other node to put it on + if not (report.failed or report.skipped): + return + path = os.path.join(self._config.rootpath, report.fspath) + name = self._file_name(path) + self._message("testStarted", name=name, locationHint=self._location(path)) + if report.failed: + self._message( + "testFailed", + name=name, + message=_crash(report), + details=str(report.longrepr), + ) + else: + self._message("testIgnored", name=name, message=_skip_reason(report)) + self._message("testFinished", name=name) + + def pytest_collection_finish(self, session): + self._message("testCount", count=str(len(session.items))) + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_protocol(self, item, nextitem): + # the suites are the collectors from the file down: no directory or package + chain = item.listchain()[:-1] + files = [index for index, node in enumerate(chain) if isinstance(node, pytest.File)] + suites = chain[files[-1] :] if files else [] + shared = 0 + while ( + shared < len(suites) + and shared < len(self._open) + and suites[shared].nodeid == self._open[shared][0] + ): + shared += 1 + self._close(shared) + for suite in suites[shared:]: + if isinstance(suite, pytest.File): + name = self._file_name(suite.path) + location = self._location(suite.path) + else: + name = suite.name + path, line, _ = suite.reportinfo() + location = self._location(path, None if line is None else line + 1) + self._message("testSuiteStarted", name=name, locationHint=location) + self._open.append((suite.nodeid, name)) + path, line, _ = item.location + self._message( + "testStarted", + name=item.name, + locationHint=self._location( + os.path.join(self._config.rootpath, path), None if line is None else line + 1 + ), + metainfo=item.nodeid, + # with pytest capturing it, a test's output is sent with its result + captureStandardOutput="false" if self._captures() else "true", + ) + self._name = item.name + self._phases = [] + self._comparisons = {} + + def _captures(self): + return self._capture is not None and self._config.getoption("capture") != "no" + + def _close(self, depth): + while len(self._open) > depth: + _, name = self._open.pop() + self._message("testSuiteFinished", name=name) + + def pytest_assertrepr_compare(self, config, op, left, right): + if op != "==": + return None + # the frame of the assertion: the first that is neither pytest's nor pluggy's + frame = sys._getframe(1) + while frame is not None and frame.f_globals.get("__name__", "").startswith( + ("_pytest.", "pluggy.") + ): + frame = frame.f_back + if frame is None: + return None + record = (frame, frame.f_lineno, frame.f_lasti, left, right) + # an assertion explains each comparison it failed in, one call each; a + # comparison it has explained already is the assertion failing again + if ( + self._compared + and self._compared[-1][0] is frame + and self._compared[-1][1] == frame.f_lineno + and all(previous[2] != frame.f_lasti for previous in self._compared) + ): + self._compared.append(record) + else: + self._compared = [record] + return None + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_makereport(self, item, call): + compared, self._compared = self._compared, [] + if call.excinfo is None or not isinstance(call.excinfo.value, AssertionError): + return None + tb = call.excinfo.value.__traceback__ + while tb is not None and tb.tb_next is not None: + tb = tb.tb_next + # the comparison is the assertion's that failed only when it was the one + # comparison that assertion explained: of `a == b or c == d`, neither is + if ( + tb is not None + and len(compared) == 1 + and compared[0][0] is tb.tb_frame + and compared[0][1] == tb.tb_lineno + ): + _, _, _, left, right = compared[0] + self._comparisons[call.when] = _shown(left, right) + return None + + def pytest_runtest_logreport(self, report): + if report.when in ("setup", "call", "teardown"): + self._phases.append(report) + + @pytest.hookimpl(trylast=True) + def pytest_runtest_logfinish(self, nodeid, location): + name = self._name + for report in self._phases: + for title, content in report.sections: + if not content or not title.endswith(" " + report.when): + continue + if "stdout" in title: + self._message("testStdOut", name=name, out=content) + elif "stderr" in title: + self._message("testStdErr", name=name, out=content) + failed = [report for report in self._phases if report.failed] + skipped = [report for report in self._phases if report.skipped] + if failed: + first = failed[0] + comparison = self._comparisons.get(first.when) + message = _crash(first) + if first.when != "call": + message = f"error at {first.when}: {message}" + attributes = {} + if comparison is not None: + actual, expected = comparison + attributes = {"type": "comparisonFailure", "expected": expected, "actual": actual} + self._message( + "testFailed", + name=name, + message=message, + details="\n\n".join(str(report.longrepr) for report in failed), + **attributes, + ) + elif skipped: + self._message("testIgnored", name=name, message=_skip_reason(skipped[0])) + duration = sum(report.duration for report in self._phases) + self._message("testFinished", name=name, duration=str(int(duration * 1000))) + self._phases = [] + self._comparisons = {} + + @pytest.hookimpl(trylast=True) + def pytest_sessionfinish(self, session): + self._close(0) + + +def _crash(report): + # the error's own line, as pytest's summary shows it, or the last of what it wrote + crash = getattr(report.longrepr, "reprcrash", None) + if crash is not None: + return crash.message + lines = str(report.longrepr).strip().splitlines() + return lines[-1] if lines else "" + + +def _skip_reason(report): + if hasattr(report, "wasxfail"): + return f"expected failure: {report.wasxfail}" if report.wasxfail else "expected failure" + # pytest's skip is a (path, line, reason) triple + if isinstance(report.longrepr, tuple) and len(report.longrepr) == 3: + return report.longrepr[2] + return str(report.longrepr) +"###; /// the distribution that names `_by_pytest` as a pytest plugin. pytest loads the /// plugins every distribution on `sys.path` declares, and the tree is at the head @@ -2135,7 +2449,10 @@ def _rewrite(frames): return out -def _excepthook(etype, evalue, tb): +def format_exception(etype, evalue, tb): + """The traceback python prints for an exception, with every frame of a staged + module at the line of the project file it stands for, and none of the frames + that ran before the program's first.""" frames = _rewrite(traceback.extract_tb(tb)) if ( isinstance(evalue, SyntaxError) @@ -2150,9 +2467,8 @@ def _excepthook(etype, evalue, tb): # the program's own module, which the runner could not compile: every frame # is the runner's, so it is reported as python reports a script it cannot # compile, by where the error is and nothing above it - sys.stderr.write("".join(traceback.format_exception_only(etype, evalue))) - return - sys.stderr.write("Traceback (most recent call last):\n") + return "".join(traceback.format_exception_only(etype, evalue)) + out = ["Traceback (most recent call last):\n"] # a frame `_rewrite` could only name is already text. the others are formatted a # run at a time, so python still folds a repeated one run = [] @@ -2160,11 +2476,27 @@ def _excepthook(etype, evalue, tb): if isinstance(frame, traceback.FrameSummary): run.append(frame) continue - sys.stderr.write("".join(traceback.StackSummary.from_list(run).format())) + out.extend(traceback.StackSummary.from_list(run).format()) run = [] if frame is not None: - sys.stderr.write(frame) - sys.stderr.write("".join(traceback.format_exception_only(etype, evalue))) + out.append(frame) + out.extend(traceback.format_exception_only(etype, evalue)) + return "".join(out) + + +def runs_generated_lines(tb): + """Whether a frame of `tb` runs a transpiled module's code at the lines of the + generated python rather than of its `.by`: code a test runner compiled, run + before `reline_module` moved it — a test module's own top level.""" + while tb is not None: + if os.path.realpath(tb.tb_frame.f_code.co_filename) in _BY_MAP: + return True + tb = tb.tb_next + return False + + +def _excepthook(etype, evalue, tb): + sys.stderr.write(format_exception(etype, evalue, tb)) def _project_file_of(origin): @@ -2316,6 +2648,21 @@ def _by_lines(origin): return by_path, by_line, width +def project_location(path, line): + """The project file a location in the tree stands for and the line of it, 1-based: + a staged `.py` moves to the line of its `.by`, and a copy of a hand-written `.py` + to the same line of the original. A location in no staged module is its own. + + `line` is `None`, and so is the answer's, for the file as a whole, and for a line + the transpiler wrote on its own.""" + lines = _by_lines(path) + if lines is not None: + by_path, by_line, _ = lines + return by_path, (by_line(line) or None) if line is not None else None + carried = _CARRIED.get(os.path.realpath(path)) + return (path if carried is None else carried), line + + def reline_module(module): """Every function a staged module compiled by another loader defines, moved to the `.by` the module was transpiled from, as `_by_code` compiles the modules no other @@ -2364,10 +2711,14 @@ def reline_module(module): if accessor is not None: visit(accessor) # what `functools.wraps` wraps — a pytest fixture's function among them. read from - # the object's own `__dict__`, so no `__getattr__` of the program's is run - wrapped = getattr(value, "__dict__", {}).get("__wrapped__") - if wrapped is not None: - visit(wrapped) + # the object's own `__dict__` with `object.__getattribute__`, so no `__getattr__` or + # `__getattribute__` of the program's runs + try: + namespace = object.__getattribute__(value, "__dict__") + except AttributeError: + return + if type(namespace) is dict and namespace.get("__wrapped__") is not None: + visit(namespace["__wrapped__"]) for value in list(vars(module).values()): visit(value) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 0d91119d5b..91995889cd 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1611,17 +1611,15 @@ fn run_coverage_maps_a_module_a_test_runner_compiled() { ); } -/// pytest compiles a test module itself, to rewrite its assertions, so a failure -/// in a test was reported from the staged `.py`, at a line of the generated python -/// and quoting it. the plugin `by run` puts in the tree moves the module's functions -/// to the `.by`, so a failure — in a test, or in a fixture of a `conftest.by` — is -/// reported at the `.by` line that failed, quoting the `.by` -#[test] +/// A project of `.by` tests for pytest to run, with a module they test, checked +/// against the environment the tests run in: the `conftest` imports pytest, which +/// `--python` does not configure. `None`, having said why, when `$PYTHON` (or +/// `python3`) cannot import pytest. #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" )] -fn run_pytest_reports_a_failure_at_the_by_line() { +fn pytest_project(dir: &Path) -> Option { let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); let pytest = Command::new(&python) .args(["-c", "import pytest"]) @@ -1629,10 +1627,8 @@ fn run_pytest_reports_a_failure_at_the_by_line() { .is_ok_and(|status| status.success()); if !pytest { eprintln!("skipping: {python} cannot import pytest"); - return; + return None; } - // the conftest imports pytest, so the check has to see the environment the tests - // run in, which `--python` does not configure let executable = Command::new(&python) .args(["-c", "import sys; print(sys.executable)"]) .output() @@ -1640,35 +1636,53 @@ fn run_pytest_reports_a_failure_at_the_by_line() { let executable = String::from_utf8_lossy(&executable.stdout) .trim() .to_owned(); - let dir = tempfile::tempdir().expect("tempdir"); - project_with_partly_covered_modules(dir.path()); + project_with_partly_covered_modules(dir); fs::write( - dir.path().join("pyproject.toml"), + dir.join("pyproject.toml"), format!("[tool.basedpython.environment]\npython = {executable:?}\n"), ) .unwrap(); - fs::create_dir_all(dir.path().join("tests")).unwrap(); + fs::create_dir_all(dir.join("tests")).unwrap(); // `let` is basedpython's own, so the quoted source is the `.by`'s and not python fs::write( - dir.path().join("tests/test_calc.by"), + dir.join("tests/test_calc.by"), "from pkg.calc import add\n\n\ndef test_sum():\n let total = add(1, 2)\n assert total == 4\n\n\nclass TestGroup:\n def test_fixture(self, broken: int):\n assert broken\n", ) .unwrap(); fs::write( - dir.path().join("tests/conftest.by"), + dir.join("tests/conftest.by"), "import pytest\n\n\n@pytest.fixture\ndef broken() -> int:\n let value = 0\n assert value == 1\n return value\n", ) .unwrap(); + Some(python) +} +/// `by run --in-build pytest` in `dir`, with `env` set, answering its standard output +fn run_pytest(dir: &Path, python: &str, env: &[(&str, &str)]) -> String { let output = Command::new(env!("CARGO_BIN_EXE_by")) .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .envs(env.iter().copied()) .arg("run") - .args(["--python", &python, "--in-build"]) + .args(["--python", python, "--in-build"]) .args(["pytest", "-p", "no:cacheprovider"]) - .current_dir(dir.path()) + .current_dir(dir) .output() .expect("failed to spawn by"); - let stdout = String::from_utf8_lossy(&output.stdout); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +/// pytest compiles a test module itself, to rewrite its assertions, so a failure +/// in a test was reported from the staged `.py`, at a line of the generated python +/// and quoting it. the plugin `by run` puts in the tree moves the module's functions +/// to the `.by`, so a failure — in a test, or in a fixture of a `conftest.by` — is +/// reported at the `.by` line that failed, quoting the `.by` +#[test] +fn run_pytest_reports_a_failure_at_the_by_line() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + let stdout = run_pytest(dir.path(), &python, &[]); assert!( stdout.contains(&format!( "{}:6: AssertionError", @@ -1693,6 +1707,160 @@ fn run_pytest_reports_a_failure_at_the_by_line() { ); } +/// the plugin looks through a test module's values for the functions it moves to the `.by` +/// without running any of the program's own code: an object whose `__getattr__` answers every +/// name, as a proxy's does, is not asked for its `__dict__` +#[test] +fn run_pytest_runs_none_of_a_test_module_s_code_to_reline_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + fs::write( + dir.path().join("tests/test_proxy.by"), + "class Proxy:\n __slots__ = ()\n\n def __getattr__(self, name: str) -> object:\n raise RuntimeError(f\"the plugin read {name}\")\n\n\nproxy = Proxy()\n\n\ndef test_proxy():\n assert True\n", + ) + .unwrap(); + let stdout = run_pytest(dir.path(), &python, &[]); + let module = Path::new("tests") + .join("test_proxy.py") + .display() + .to_string(); + assert!( + !stdout.contains("the plugin read") && stdout.contains(&format!("{module} .")), + "{stdout}" + ); +} + +/// a test module's top level runs as pytest compiled it, at the lines of the +/// generated python, before the plugin sees the module. an error it raises, or an +/// import of it fails with, is reported as `by run` reports one in a program, as +/// python prints it with each frame at its `.by` line and quoting the `.by` +#[test] +fn run_pytest_reports_an_error_importing_a_test_module_at_the_by_line() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + fs::write( + dir.path().join("tests/test_broken.by"), + "from pkg.calc import add\n\nlet total = add(1, 2)\nraise RuntimeError(f\"broken at {total}\")\n\n\ndef test_never():\n pass\n", + ) + .unwrap(); + fs::write( + dir.path().join("tests/test_missing.by"), + "import importlib\n\nlet name = \"missing_module\"\nlet _ = importlib.import_module(name)\n\n\ndef test_never():\n pass\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + let stdout = run_pytest(dir.path(), &python, &[]); + for (file, line, quoted, error) in [ + ( + "test_broken.by", + 4, + "raise RuntimeError(f\"broken at {total}\")", + "RuntimeError: broken at 3", + ), + ( + "test_missing.by", + 4, + "let _ = importlib.import_module(name)", + "ModuleNotFoundError: No module named", + ), + ] { + let at = format!( + " File \"{}\", line {line}, in \n {quoted}\n", + project.join("tests").join(file).display() + ); + assert!( + stdout.contains(&at), + "{file}'s error is not reported at its `.by` line:\n{stdout}" + ); + // the summary's line names the error, as it does for one pytest reports + assert!( + stdout.contains(&format!(" - {error}")), + "{file}'s error is not summarised:\n{stdout}" + ); + } +} + +/// with `BY_TEST_REPORTER=teamcity` the run is reported as the `TeamCity` service +/// messages an IDE's test runner reads: suites named by the `.by` files and the +/// classes in them, each test with its node id and its line in the `.by`, a failing +/// `==` with what each side was and the failure's traceback at the `.by` line, and +/// a fixture's failure as the test's error at setup +#[test] +fn run_pytest_reports_teamcity_messages() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + let project = dir.path().canonicalize().unwrap(); + let test_file = project.join("tests").join("test_calc.by"); + let url = format!("file://{}", test_file.display()); + + let stdout = run_pytest( + dir.path(), + &python, + &[(EnvVars::BY_TEST_REPORTER, "teamcity")], + ); + let messages: Vec<&str> = stdout + .match_indices("##teamcity[") + .map(|(at, _)| { + let message = &stdout[at..]; + &message[..message.find("]\n").map_or(message.len(), |end| end + 1)] + }) + .collect(); + let named = |kind: &str, name: &str| { + messages + .iter() + .copied() + .find(|message| message.starts_with(&format!("##teamcity[{kind} name='{name}'"))) + .unwrap_or_else(|| panic!("no {kind} for {name}:\n{stdout}")) + }; + + assert_eq!(messages.first(), Some(&"##teamcity[enteredTheMatrix]")); + assert_eq!( + named("testSuiteStarted", "tests/test_calc.by"), + format!("##teamcity[testSuiteStarted name='tests/test_calc.by' locationHint='{url}']") + ); + assert_eq!( + named("testStarted", "test_sum"), + format!( + "##teamcity[testStarted name='test_sum' locationHint='{url}:4' metainfo='tests/test_calc.py::test_sum' captureStandardOutput='false']" + ) + ); + let failed = named("testFailed", "test_sum"); + assert!( + failed + .starts_with("##teamcity[testFailed name='test_sum' message='assert 3 == 4' details='") + && failed.contains("> assert total == 4|n") + && failed.contains("tests/test_calc.by:6: AssertionError'") + && failed.ends_with(" type='comparisonFailure' expected='4' actual='3']"), + "{failed}" + ); + assert_eq!( + named("testSuiteStarted", "TestGroup"), + format!("##teamcity[testSuiteStarted name='TestGroup' locationHint='{url}:9']") + ); + let errored = named("testFailed", "test_fixture"); + assert!( + errored.starts_with( + "##teamcity[testFailed name='test_fixture' message='error at setup: assert 0 == 1' details='" + ) && errored.contains("conftest.by:7: AssertionError") + && errored.ends_with(" type='comparisonFailure' expected='1' actual='0']"), + "{errored}" + ); + assert!( + messages.ends_with(&[ + "##teamcity[testSuiteFinished name='TestGroup']", + "##teamcity[testSuiteFinished name='tests/test_calc.by']", + ]), + "{stdout}" + ); +} + /// `multiprocessing`'s spawn start method — the default on macos and windows — /// reads `__main__.__spec__.name` to tell the child what to re-import, and falls /// back to running `__file__` as a *path* when there is none. `__file__` is a diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index e67a9723e5..668f7caf20 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -71,6 +71,14 @@ impl EnvVars { /// Equivalent to the `--no-server` command-line argument. pub const BY_NO_PROJECT_SERVER: &'static str = "BY_NO_PROJECT_SERVER"; + /// Report the tests pytest runs under `by run` as service messages, `TeamCity`'s protocol. + /// + /// Set this to `teamcity` for an IDE's test runner to read the run: a suite for each test + /// file, named by the `.by` it was transpiled from, and one for each class in it, each test + /// with its pytest node id and where it is in the `.by`, and a failing `==` with what each + /// side was. Any other value is refused. + pub const BY_TEST_REPORTER: &'static str = "BY_TEST_REPORTER"; + /// Enable uv integration. /// /// When set to `"1"` or `"true"`, ty invokes `uv workspace metadata` to discover the workspace diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index aa79ddf3fc..8c524a8877 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -128,8 +128,13 @@ of its tests runs, the plugin moves the code of every function the module define — the methods of its classes, what a decorator wraps, a fixture's function, and the functions nested in each — to the `.by` lines, as the shim's own loader does. a failure in a test or a fixture is then reported at the `.by` line, quoting the -`.by`. the module's top level ran as it was imported, before the plugin saw it, so -an error raised while importing a test module still names the staged `.py` +`.by`. the module's top level ran as it was imported, before the plugin saw it, at +the lines of the generated python: an error it raises, or an import in it fails +with, is reported in place of pytest's report of it as `by run` reports an error in +a program, python's traceback with each frame at its `.by` line. pytest's node ids +still name the staged `.py`, `tests/test_calc.py::test_sum`: they are what pytest +is given back to run a test again, and what its cache and its `--deselect` read, +and they have to name a file in the tree pytest runs in both tables are keyed by the generated path exactly as the map spells it. the shim resolves symlinks before matching a frame's filename against it — a diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index e317cce179..18248a202f 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1433,6 +1433,44 @@ project's `src/shapes/circle.by` is `shapes.circle`, and `src/shapes` and `src/shapes/__init__.by` are both `shapes`. it is `null` for a path no module is imported from, and for a file that resolution would never reach +### running tests in an IDE + +`by run pytest` reports to an IDE's test runner when `BY_TEST_REPORTER=teamcity` +is set: the pytest plugin `by run` puts in the tree writes the run to standard +output as TeamCity service messages, alongside pytest's own output, which is left +as the arguments ask for. what it writes: + +- `enteredTheMatrix` as the session starts, so a client knows the plugin is + reporting before the first test, and `testCount` with how many tests were + collected +- a file that failed to collect, a test module whose import raised, as a test of + its own at the top level, named by its file: `testStarted`, `testFailed` with the + error's line as `message` and the whole report as `details`, and `testFinished`; + a file skipped as a whole has `testIgnored` instead +- for each test, the suites it is in: its file, named by the project file it was + transpiled from or copied out of relative to the project, `tests/test_calc.by`, + and each class it is collected under, `TestGroup`, each opened before the first + of its tests and closed after the last +- `testStarted` with the test's `name` as pytest gives it, `test_sum` or + `test_add[1-2]`, its node id as `metainfo`, `tests/test_calc.py::test_sum`, which + is what pytest is given to run it again, and `captureStandardOutput` `false` + while pytest captures its output, which follows as `testStdOut` and + `testStdErr` +- `testFailed` when any of its setup, call or teardown failed: the error's line as + `message`, prefixed `error at setup:` or `error at teardown:` for those, and + pytest's report of the failure as `details`, every frame at its `.by` line. when + the failure is an assertion that failed comparing two values with `==`, and that + comparison is the one it made, it carries `type='comparisonFailure'`, `actual`, + the left side, and `expected`, the right side, each a string as it is and + anything else as `pprint` writes it. `assert a == b or c == d` carries neither +- `testIgnored` for a test skipped, with the reason, or failing as expected, with + `expected failure:` and the reason, then `testFinished` with its `duration` in + milliseconds + +every `locationHint` is `file://` and the absolute path of the project file, with +`:` and the 1-based line of the `def` or `class` for a test or a class. any other +value of `BY_TEST_REPORTER` is refused as a usage error + ## override and implement members an editor's *override methods* and *implement methods* ask twice: once for what From 2dce7fd8b01c931de9fe11fc2bc0ddc653ce453d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:47:02 +1000 Subject: [PATCH 055/110] report an invalid regex pattern written where it is passed at the character python names the position of, the ( left open or the ) that closes nothing, reading the literal as python does so an escape is the whole escape and adjacent literals one pattern, and a {2,1} or a redefined group name as all of it, rather than at the whole literal --- .../resources/mdtest/basedpython_regex.md | 92 +++++++ .../src/types/infer/builder.rs | 20 +- crates/ty_python_semantic/src/types/regex.rs | 2 + .../src/types/regex/parse.rs | 34 ++- .../src/types/regex/source.rs | 259 ++++++++++++++++++ docs/basedpython/features/regex-groups.md | 7 + 6 files changed, 405 insertions(+), 9 deletions(-) create mode 100644 crates/ty_python_semantic/src/types/regex/source.rs diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md index 9489339d0b..e362848da7 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md @@ -326,6 +326,98 @@ re.findall("(", "") # error: [invalid-regex] "missing ), unterminated subpatter re.sub("(", "", "") # error: [invalid-regex] "missing ), unterminated subpattern at position 0" ``` +## an invalid pattern's error points at the character it is about + +python names the position in the pattern where it gave up, which is the character the error is +about: the parenthesis left open, the one that closes nothing, the quantifier with nothing to +repeat. + +```py +import re + +re.compile("ab(c") # snapshot +``` + +```snapshot +error[invalid-regex]: missing ), unterminated subpattern at position 2 + --> src/mdtest_snippet.py:3:15 + | +3 | re.compile("ab(c") # snapshot + | ^ +``` + +the position is a character of the pattern the program holds, which is not always one character of +the literal: an escape writes one character in several, and the error is about all of them. here +`\x28` writes the parenthesis left open, after a tab written as `\t`. + +```py +import re + +re.compile("\t\x28") # snapshot +``` + +```snapshot +error[invalid-regex]: missing ), unterminated subpattern at position 1 + --> src/mdtest_snippet.py:6:15 + | +6 | re.compile("\t/x28") # snapshot + | ^^^^ +``` + +literals written next to each other are one pattern, so the error is in whichever of them wrote the +character. + +```py +import re + +re.compile( + r"\d+" + "x{2,1}" # snapshot +) +``` + +```snapshot +error[invalid-regex]: min repeat greater than max repeat at position 5 + --> src/mdtest_snippet.py:11:7 + | +11 | "x{2,1}" # snapshot + | ^^^^^ +``` + +an error about a construct as a whole, a repeat whose minimum is above its maximum or a group's +name, is about all of it. + +```py +import re + +re.compile("(?Pa)(?Pb)") # snapshot +``` + +```snapshot +error[invalid-regex]: redefinition of group name 'name' as group 2; was group 1 at position 15 + --> src/mdtest_snippet.py:15:28 + | +15 | re.compile("(?Pa)(?Pb)") # snapshot + | ^^^^ +``` + +a pattern that is not written where it is passed is pointed at as what was passed. + +```py +import re + +pattern = "a(" +re.compile(pattern) # snapshot +``` + +```snapshot +error[invalid-regex]: missing ), unterminated subpattern at position 1 + --> src/mdtest_snippet.py:19:12 + | +19 | re.compile(pattern) # snapshot + | ^^^^^^^ +``` + ## a group the pattern has not got ```py diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 6e9894eb88..83f8c9a16b 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -12418,11 +12418,21 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { (regex::RegexGroups::from_parsed(db, &parsed), any_str) } regex::PatternAnalysis::Invalid(error) => { - let anchor = call_expression - .arguments - .find_argument_value("pattern", 0) - .map_or(AnyNodeRef::from(call_expression), AnyNodeRef::from); - if let Some(builder) = self.context.report_lint(&INVALID_REGEX, anchor) { + let pattern = call_expression.arguments.find_argument_value("pattern", 0); + // the characters the error is about, where the pattern is a literal + // here, and otherwise what was passed as the pattern + let range = pattern + .and_then(|pattern| { + regex::pattern_range( + pattern, + &source_text(db, self.file()), + &text, + error.span(), + ) + }) + .or_else(|| pattern.map(Ranged::range)) + .unwrap_or_else(|| call_expression.range()); + if let Some(builder) = self.context.report_lint(&INVALID_REGEX, range) { builder.into_diagnostic(error.message()); } overload.set_return_type(Type::unknown()); diff --git a/crates/ty_python_semantic/src/types/regex.rs b/crates/ty_python_semantic/src/types/regex.rs index cf53204d26..248976a246 100644 --- a/crates/ty_python_semantic/src/types/regex.rs +++ b/crates/ty_python_semantic/src/types/regex.rs @@ -11,6 +11,7 @@ //! round trip through `re.compile` mod parse; +mod source; use std::fmt; @@ -27,6 +28,7 @@ use crate::types::{ }; pub(crate) use parse::{PatternAnalysis, analyze}; +pub(crate) use source::pattern_range; /// one capture group of a statically-known pattern #[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] diff --git a/crates/ty_python_semantic/src/types/regex/parse.rs b/crates/ty_python_semantic/src/types/regex/parse.rs index cb0bd8739d..4cdaa2d740 100644 --- a/crates/ty_python_semantic/src/types/regex/parse.rs +++ b/crates/ty_python_semantic/src/types/regex/parse.rs @@ -10,6 +10,8 @@ //! error: reporting a valid pattern as invalid is much worse than declining to //! refine it +use std::ops::Range; + use ruff_python_ast::name::Name; use rustc_hash::FxHashMap; @@ -38,18 +40,33 @@ pub(crate) struct ParsedGroup { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct RegexError { message: String, + /// the characters of the pattern the error is about: the one at the position + /// python names, or the whole of a construct that is wrong as a whole, a + /// `{2,1}` or a group's name + span: Range, } impl RegexError { fn new(message: impl std::fmt::Display, position: usize) -> Self { + Self::spanning(message, position, position..position + 1) + } + + fn spanning(message: impl std::fmt::Display, position: usize, span: Range) -> Self { Self { message: format!("{message} at position {position}"), + span, } } pub(crate) fn message(&self) -> &str { &self.message } + + /// the characters of the pattern the error is about, which end past the + /// pattern for an error at its end, as `unexpected end of pattern` is + pub(crate) fn span(&self) -> Range { + self.span.clone() + } } /// what we were able to learn about a pattern @@ -271,9 +288,10 @@ impl Parser { }; self.pos += quantifier.length; if quantifier.min > quantifier.max { - return Err(Failure::Invalid(RegexError::new( + return Err(Failure::Invalid(RegexError::spanning( "min repeat greater than max repeat", open + 1, + open..self.pos, ))); } quantifier.min == 0 @@ -536,9 +554,10 @@ impl Parser { // python resolves a backreference against the groups opened so // far, so a forward reference is an error there too if !self.names.contains_key(&name) { - return Err(Failure::Invalid(RegexError::new( + return Err(Failure::Invalid(RegexError::spanning( format_args!("unknown group name '{name}'"), start, + name_span(&name, start), ))); } Ok(Some(Node::Atom)) @@ -589,9 +608,10 @@ impl Parser { ))); } if !is_identifier(name) { - return Err(Failure::Invalid(RegexError::new( + return Err(Failure::Invalid(RegexError::spanning( format_args!("bad character in group name '{name}'"), start, + name_span(name, start), ))); } Ok(Name::new(name)) @@ -695,11 +715,12 @@ impl Parser { if let Some(name) = &name && let Some(previous) = self.names.insert(name.clone(), index) { - return Err(Failure::Invalid(RegexError::new( + return Err(Failure::Invalid(RegexError::spanning( format_args!( "redefinition of group name '{name}' as group {index}; was group {previous}" ), start, + name_span(name, start), ))); } self.group_names.push(name); @@ -715,6 +736,11 @@ struct BracedQuantifier { max: u32, } +/// the characters of a group's name that starts at `start` +fn name_span(name: &str, start: usize) -> Range { + start..start + name.chars().count() +} + fn uncertain_if(node: Node, uncertain: bool) -> Node { if uncertain { Node::Uncertain(Box::new(node)) diff --git a/crates/ty_python_semantic/src/types/regex/source.rs b/crates/ty_python_semantic/src/types/regex/source.rs new file mode 100644 index 0000000000..26bf02cb6d --- /dev/null +++ b/crates/ty_python_semantic/src/types/regex/source.rs @@ -0,0 +1,259 @@ +//! where the characters of a literal pattern were written +//! +//! a pattern's errors are positions in the text the program holds, which is +//! not the text of the literal that wrote it: `"\t("` holds two characters +//! written as four, and `"a(" "b"` one pattern written as two literals. to +//! point at the character an error is about, the literal is read again the way +//! python reads it, keeping where each character it holds came from + +use std::ops::Range; + +use ruff_python_ast as ast; +use ruff_text_size::{TextRange, TextSize}; + +/// where in `source` the characters `span` of `pattern` were written, when +/// `pattern` is the value of the string or bytes literal `expr` +/// +/// a bytes pattern is `pattern` read as latin-1, one character a byte. `span` +/// is clamped to the pattern, so a span past its end — an error at the end — +/// is its last character. `None` for an expression that is not a literal, or a +/// literal whose value does not read as `pattern`, which leaves the caller to +/// point at the expression as a whole +pub(crate) fn pattern_range( + expr: &ast::Expr, + source: &str, + pattern: &str, + span: Range, +) -> Option { + let mut written = Vec::new(); + match expr { + ast::Expr::StringLiteral(string) => { + for part in &string.value { + read_part( + source, + part.content_range(), + part.flags.prefix().is_raw(), + false, + &mut written, + )?; + } + } + ast::Expr::BytesLiteral(bytes) => { + for part in &bytes.value { + read_part( + source, + part.content_range(), + part.flags.prefix().is_raw(), + true, + &mut written, + )?; + } + } + _ => return None, + } + + // the literal has to read as the pattern, character for character, or the + // reading is not the one python made, and no character of it can be trusted + let mut held = pattern.chars(); + for character in &written { + let held = held.next()?; + if character.value.is_some_and(|value| value != held) { + return None; + } + } + if held.next().is_some() { + return None; + } + + let last = written.len().checked_sub(1)?; + let start = span.start.min(last); + let end = span.end.clamp(start + 1, written.len()) - 1; + Some(TextRange::new( + written[start].range.start(), + written[end].range.end(), + )) +} + +/// a character a literal holds and the text that wrote it +struct Written { + /// the character, or `None` for one written by a name, `\N{BULLET}`, which + /// is not looked up + value: Option, + range: TextRange, +} + +/// read the text between one literal's quotes, `content`, as python does +fn read_part( + source: &str, + content: TextRange, + raw: bool, + bytes: bool, + written: &mut Vec, +) -> Option<()> { + let text = source.get(content.start().to_usize()..content.end().to_usize())?; + let at = |index: usize| content.start() + TextSize::of(&text[..index]); + let mut characters = text.char_indices().peekable(); + while let Some((start, character)) = characters.next() { + if raw || character != '\\' { + written.push(Written { + value: Some(character), + range: TextRange::at(at(start), TextSize::of(character)), + }); + continue; + } + let &(_, escaped) = characters.peek()?; + let value = match escaped { + // a backslash ending a line joins it to the next, and holds nothing + '\n' => { + characters.next(); + continue; + } + '\r' => { + characters.next(); + characters.next_if(|&(_, next)| next == '\n'); + continue; + } + '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => { + characters.next(); + Some(match escaped { + 'a' => '\x07', + 'b' => '\x08', + 'f' => '\x0c', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'v' => '\x0b', + quoted => quoted, + }) + } + '0'..='7' => { + let mut code = 0; + while let Some((_, digit)) = + characters.next_if(|&(index, next)| index - start <= 3 && next.is_digit(8)) + { + code = code * 8 + digit.to_digit(8)?; + } + // a bytes literal keeps the low byte of an octal escape past 0o377 + Some(char::from_u32(if bytes { code & 0xff } else { code })?) + } + 'x' => { + characters.next(); + Some(hex(&mut characters, 2)?) + } + 'u' if !bytes => { + characters.next(); + Some(hex(&mut characters, 4)?) + } + 'U' if !bytes => { + characters.next(); + Some(hex(&mut characters, 8)?) + } + 'N' if !bytes => { + characters.next(); + characters.next_if(|&(_, next)| next == '{')?; + characters.find(|&(_, next)| next == '}')?; + None + } + // any other backslash is the character it is, and so is what follows + _ => Some('\\'), + }; + let end = if value == Some('\\') && escaped != '\\' { + start + 1 + } else { + characters.peek().map_or(text.len(), |&(index, _)| index) + }; + written.push(Written { + value, + range: TextRange::new(at(start), at(end)), + }); + } + Some(()) +} + +/// the character `digits` hex digits write, consuming them +fn hex( + characters: &mut std::iter::Peekable>, + digits: usize, +) -> Option { + let mut code = 0; + for _ in 0..digits { + let (_, digit) = characters.next()?; + code = code * 16 + digit.to_digit(16)?; + } + char::from_u32(code) +} + +#[cfg(test)] +mod tests { + use ruff_python_ast as ast; + use ruff_python_parser::parse_expression; + + use super::pattern_range; + + /// the text of `literal` that wrote the characters `span` of the pattern it holds + fn written(literal: &str, pattern: &str, span: std::ops::Range) -> Option { + let parsed = parse_expression(literal).expect("a literal"); + let expr: &ast::Expr = parsed.expr(); + pattern_range(expr, literal, pattern, span).map(|range| literal[range].to_string()) + } + + #[test] + fn a_plain_character_is_itself() { + assert_eq!(written(r#""ab(c""#, "ab(c", 2..3).as_deref(), Some("(")); + } + + #[test] + fn an_escape_is_the_text_that_wrote_it() { + assert_eq!( + written(r#""\t\x41é\N{BULLET}(""#, "\tAé\u{2022}(", 4..5).as_deref(), + Some("(") + ); + assert_eq!( + written(r#""\t\x41é\N{BULLET}(""#, "\tAé\u{2022}(", 1..4).as_deref(), + Some(r"\x41é\N{BULLET}") + ); + assert_eq!(written(r#""\101(""#, "A(", 0..1).as_deref(), Some(r"\101")); + } + + #[test] + fn a_backslash_that_escapes_nothing_is_itself() { + assert_eq!(written(r#""\d+(""#, r"\d+(", 0..2).as_deref(), Some(r"\d")); + } + + #[test] + fn a_raw_literal_holds_its_text() { + assert_eq!(written(r#"r"\t(""#, r"\t(", 2..3).as_deref(), Some("(")); + } + + #[test] + fn concatenated_literals_are_one_pattern() { + assert_eq!( + written(r#"("\t" r"\d" "x{2,1}")"#, "\t\\dx{2,1}", 4..9).as_deref(), + Some("{2,1}") + ); + } + + #[test] + fn a_line_continuation_holds_nothing() { + assert_eq!(written("\"a\\\n(\"", "a(", 1..2).as_deref(), Some("(")); + } + + #[test] + fn a_bytes_literal_is_read_a_byte_a_character() { + assert_eq!( + written(r#"b"\xff(""#, "\u{ff}(", 1..2).as_deref(), + Some("(") + ); + } + + #[test] + fn an_error_at_the_end_is_the_last_character() { + assert_eq!(written(r#""a\\""#, r"a\", 2..3).as_deref(), Some(r"\\")); + } + + #[test] + fn a_literal_that_does_not_hold_the_pattern_is_not_read() { + assert_eq!(written(r#""ab""#, "abc", 0..1), None); + assert_eq!(written("name", "abc", 0..1), None); + } +} diff --git a/docs/basedpython/features/regex-groups.md b/docs/basedpython/features/regex-groups.md index e68e86af6f..64a31e340f 100644 --- a/docs/basedpython/features/regex-groups.md +++ b/docs/basedpython/features/regex-groups.md @@ -113,6 +113,13 @@ if m := re.match("(a)(b)", text): the group check also covers `m[3]`, `m.start(3)`, `m.end(3)` and `m.span(3)`. +an invalid pattern written where it is passed is reported at the character python +names the position of, not the whole literal: the `(` left open, the `)` that +closes nothing. a character written as an escape, `\x28`, is reported as the whole +escape, a pattern written as adjacent literals at the literal that has the +character, and an error about a construct as a whole, `{2,1}` or a group's +redefined name, at all of it. a pattern passed by name is reported at the name. + a construct the pattern reader does not model produces neither a refinement nor a diagnostic: a pattern reported invalid when `re.compile` would have accepted it is much worse than one left alone. From e4b91a17f4406763ebff8f79a7a377840ad40305 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:05:03 +1000 Subject: [PATCH 056/110] answer by/endpoints, every HTTP route a project serves for an editor's endpoints view, FastAPI's decorators, add_api_route and include_router and django's urlpatterns from ROOT_URLCONF through include(), each resolved to what it calls rather than read by name, with its methods, its whole path under every prefix, its handler, where it is declared and the application that serves it --- crates/ty_ide/src/django_template.rs | 1 + crates/ty_ide/src/django_template/project.rs | 2 +- crates/ty_ide/src/endpoints.rs | 1092 +++++++++++++++++ crates/ty_ide/src/lib.rs | 2 + .../src/types/ide_support.rs | 17 + crates/ty_server/src/server/api.rs | 4 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/endpoints.rs | 138 +++ crates/ty_server/tests/e2e/endpoints.rs | 71 ++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 60 + 11 files changed, 1389 insertions(+), 1 deletion(-) create mode 100644 crates/ty_ide/src/endpoints.rs create mode 100644 crates/ty_server/src/server/api/requests/endpoints.rs create mode 100644 crates/ty_server/tests/e2e/endpoints.rs diff --git a/crates/ty_ide/src/django_template.rs b/crates/ty_ide/src/django_template.rs index 182315b671..5d0d3fa617 100644 --- a/crates/ty_ide/src/django_template.rs +++ b/crates/ty_ide/src/django_template.rs @@ -38,6 +38,7 @@ pub use code_lens::{DjangoCodeLens, DjangoLensAction, DjangoLensTarget}; pub use completion::{TemplateCompletion, TemplateEdit}; pub use hover::{DisplayTemplateHover, TemplateHover}; pub use inlay_hints::{TemplateInlayHint, TemplateInlayHintKind}; +pub(crate) use project::root_urlconf as django_root_urlconf; pub(crate) use python::{ string_completions as django_string_completions, string_definition as django_string_definition, }; diff --git a/crates/ty_ide/src/django_template/project.rs b/crates/ty_ide/src/django_template/project.rs index dd23b85f88..787e7d3a3b 100644 --- a/crates/ty_ide/src/django_template/project.rs +++ b/crates/ty_ide/src/django_template/project.rs @@ -1091,7 +1091,7 @@ fn walk_urls(db: &dyn Db, project: Project) -> Option> { /// it is a dotted module name like any other, so an installed package may be the /// one that holds it. #[salsa::tracked] -fn root_urlconf(db: &dyn Db, project: Project) -> Option { +pub(crate) fn root_urlconf(db: &dyn Db, project: Project) -> Option { let importing = (*settings_file(db, project))?; let root = django_settings(db, project).root_urlconf.as_ref()?; diff --git a/crates/ty_ide/src/endpoints.rs b/crates/ty_ide/src/endpoints.rs new file mode 100644 index 0000000000..7097f52560 --- /dev/null +++ b/crates/ty_ide/src/endpoints.rs @@ -0,0 +1,1092 @@ +//! the HTTP endpoints a project serves: what an editor's *endpoints* view lists +//! +//! ```py +//! # app/items.py +//! router = APIRouter(prefix="/items") +//! +//! @router.get("/{item_id}") +//! def read_item(item_id: int) -> Item: ... +//! +//! # app/main.py +//! app = FastAPI() +//! app.include_router(items.router, prefix="/v1") +//! ``` +//! +//! `read_item` serves `GET /v1/items/{item_id}` for the application `app.main.app`. every part of +//! that is read through what the code means rather than how it is spelled: `router.get` is a route +//! because it resolves to FastAPI's `APIRouter.get`, whatever the router is called and however it +//! was imported, the router `include_router` is handed is the one the route was declared on because +//! both names resolve to the same assignment, and the prefixes are what ty infers them to be, so a +//! prefix held in a constant is read as well as one written out +//! +//! django's routes are read the same way from the urlconf module `ROOT_URLCONF` names, or, in a +//! project whose settings do not name one, from every module with `urlpatterns` that no other +//! includes: an entry is a route when it is a call of `django.urls.path` or `re_path`, an +//! `include()` is followed to the module, the list or the module object it names, and a view built +//! by `as_view()` is handled by its class +//! +//! a route whose path, or a prefix it is mounted under, is not a string ty knows exactly is left +//! out rather than listed at a path it may not have + +use ruff_db::files::FileRange; +use ruff_db::parsed::parsed_module; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::{self as ast}; +use ruff_text_size::Ranged; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_module_resolver::{ImportingFile, ModuleName, resolve_real_module}; +use ty_python_core::ProgramFile; +use ty_python_core::definition::{Definition, DefinitionKind}; +use ty_python_semantic::types::ide_support::{module_value_file, string_literal_value}; +use ty_python_semantic::{HasDefinition, ImportAliasResolution, ResolvedDefinition, SemanticModel}; + +use crate::declaration::declaration; +use crate::goto::{GotoTarget, navigation_target}; +use crate::{Db, NavigationTarget}; + +/// the framework that serves an endpoint +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Framework { + FastApi, + Django, +} + +impl Framework { + /// the name a client sees + pub const fn as_str(self) -> &'static str { + match self { + Framework::FastApi => "fastapi", + Framework::Django => "django", + } + } +} + +/// one route a project declares +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Endpoint { + pub framework: Framework, + /// the HTTP methods the route serves, upper case: `["GET"]`. empty for a route that serves + /// every method, as each of django's does + pub methods: Vec, + /// the path it is served at, with every prefix it is mounted under, and its parameters as the + /// route writes them: `/v1/items/{item_id}` + pub path: String, + /// what handles a request to it + pub handler: EndpointTarget, + /// where the route is declared: the decorator, or the call that adds it, `path(...)` among them + pub declaration: FileRange, + /// the application that serves it: a FastAPI application, or the urlconf module django starts + /// at. `None` for a route on a router no application includes + pub application: Option, +} + +/// a declaration an endpoint names: its qualified name, where it has one, and where it is +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EndpointTarget { + pub name: Option, + pub target: NavigationTarget, +} + +/// every endpoint the project's files declare, ordered by path and then by method +pub fn endpoints(db: &dyn Db) -> Vec { + let mut scan = FastApiScan::default(); + for file in db.project().files(db).iter() { + let file = db.program_file(file); + scan.scan_file(db, file); + } + let mut found = scan.endpoints(db); + found.extend(django_endpoints(db)); + found.sort_by(|a, b| { + ( + &a.path, + &a.methods, + a.declaration.file(), + a.declaration.range().start(), + ) + .cmp(&( + &b.path, + &b.methods, + b.declaration.file(), + b.declaration.range().start(), + )) + }); + found +} + +/// the FastAPI class that holds the methods declaring routes on an application +const FASTAPI_APPLICATION: &str = "fastapi.applications.FastAPI"; + +/// the FastAPI class that holds the methods declaring routes on a router +const FASTAPI_ROUTER: &str = "fastapi.routing.APIRouter"; + +/// the decorators that declare a route for one method, by the method they name +const METHOD_DECORATORS: [&str; 8] = [ + "get", "post", "put", "patch", "delete", "options", "head", "trace", +]; + +/// the decorator that declares a route for the methods it is given +const API_ROUTE_DECORATOR: &str = "api_route"; + +/// the call that adds a route for a function it is handed rather than decorates +const ADD_API_ROUTE: &str = "add_api_route"; + +/// the call that mounts a router's routes on an application or another router +const INCLUDE_ROUTER: &str = "include_router"; + +/// what FastAPI serves a route declared without `methods` for +const DEFAULT_METHOD: &str = "GET"; + +/// whether routes are declared on an application or on a router +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Owner { + Application, + Router, +} + +/// a route as declared, on the application or router `owner` +struct Route<'db> { + owner: Definition<'db>, + owner_kind: Owner, + methods: Vec, + path: String, + handler: ResolvedDefinition<'db>, + declaration: FileRange, +} + +/// a router mounted on an application or another router +struct Include<'db> { + parent: Definition<'db>, + parent_kind: Owner, + router: Definition<'db>, + prefix: String, +} + +#[derive(Default)] +struct FastApiScan<'db> { + routes: Vec>, + includes: Vec>, +} + +impl<'db> FastApiScan<'db> { + fn scan_file(&mut self, db: &'db dyn Db, file: ProgramFile<'db>) { + let parsed = parsed_module(db, file.python_file(db)).load(db); + let mut visitor = FastApiVisitor { + db, + model: SemanticModel::new(db, file), + file, + scan: self, + }; + visitor.visit_body(parsed.suite()); + } + + /// each route at every path an application serves it at, and a route on a router no + /// application includes at the path the router gives it + fn endpoints(&self, db: &'db dyn Db) -> Vec { + let mut found = Vec::new(); + let mut prefixes = FxHashMap::default(); + for route in &self.routes { + let handler = target(db, &route.handler); + let mounted = match route.owner_kind { + Owner::Application => vec![(Some(route.owner), route.path.clone())], + Owner::Router => { + let Some(own) = router_prefix(db, &mut prefixes, route.owner) else { + continue; + }; + self.mounted( + db, + &mut prefixes, + route.owner, + format!("{own}{}", route.path), + &mut FxHashSet::default(), + ) + } + }; + for (application, path) in mounted { + found.push(Endpoint { + framework: Framework::FastApi, + methods: route.methods.clone(), + path, + handler: handler.clone(), + declaration: route.declaration, + application: application.map(|application| { + target(db, &ResolvedDefinition::Definition(application)) + }), + }); + } + } + found + } + + /// every application `router` is mounted on, directly or through other routers, with the + /// path that serves `path` there: FastAPI copies a router's routes into what includes it, + /// under the include's prefix and the including router's own. a router no application + /// includes is served nowhere, and answers itself with no application + fn mounted( + &self, + db: &'db dyn Db, + prefixes: &mut FxHashMap, Option>, + router: Definition<'db>, + path: String, + visiting: &mut FxHashSet>, + ) -> Vec<(Option>, String)> { + // a router included, around a loop, in itself is mounted nowhere new + if !visiting.insert(router) { + return Vec::new(); + } + let mut found = Vec::new(); + let mut included = false; + for include in self + .includes + .iter() + .filter(|include| include.router == router) + { + included = true; + let path = format!("{}{path}", include.prefix); + match include.parent_kind { + Owner::Application => found.push((Some(include.parent), path)), + Owner::Router => { + let Some(own) = router_prefix(db, prefixes, include.parent) else { + continue; + }; + found.extend(self.mounted( + db, + prefixes, + include.parent, + format!("{own}{path}"), + visiting, + )); + } + } + } + visiting.remove(&router); + if !included { + found.push((None, path)); + } + found + } +} + +struct FastApiVisitor<'a, 'db> { + db: &'db dyn Db, + model: SemanticModel<'db>, + file: ProgramFile<'db>, + scan: &'a mut FastApiScan<'db>, +} + +impl<'db> FastApiVisitor<'_, 'db> { + /// the FastAPI method `function` resolves to, with whether it is an application's or a + /// router's: `router.get` is the router's `get` for an `APIRouter` however it is spelled + fn fastapi_method(&self, function: &ast::ExprAttribute) -> Option { + let db = self.db; + let definitions = GotoTarget::Expression(ast::ExprRef::Attribute(function)) + .expression_definitions(&self.model, ImportAliasResolution::ResolveAliases)?; + let mut owners = definitions.iter().map(|resolved| { + let qualified = declaration(db, resolved)?.qualified_name?; + let (class, method) = qualified.rsplit_once('.')?; + if method != function.attr.as_str() { + return None; + } + match class { + FASTAPI_APPLICATION => Some(Owner::Application), + FASTAPI_ROUTER => Some(Owner::Router), + _ => None, + } + }); + let owner = owners.next()??; + owners.all(|other| other == Some(owner)).then_some(owner) + } + + /// the one assignment `receiver` names, which is how two mentions of an application or a + /// router are known to be the same one + fn assignment(&self, receiver: &ast::Expr) -> Option> { + let definitions = GotoTarget::Expression(receiver.into()) + .expression_definitions(&self.model, ImportAliasResolution::ResolveAliases)?; + let mut assignments = definitions.iter().map(|resolved| match resolved { + ResolvedDefinition::Definition(definition) => Some(*definition), + _ => None, + }); + let first = assignments.next()??; + assignments + .all(|other| other == Some(first)) + .then_some(first) + } + + /// the methods a route declared with `methods=` serves, or GET without it. `None` where + /// `methods` is something other than a display of strings ty knows + fn methods(&self, arguments: &ast::Arguments) -> Option> { + let Some(methods) = arguments.find_keyword("methods") else { + return Some(vec![DEFAULT_METHOD.to_string()]); + }; + let elements = match &methods.value { + ast::Expr::List(list) => &list.elts, + ast::Expr::Tuple(tuple) => &tuple.elts, + ast::Expr::Set(set) => &set.elts, + _ => return None, + }; + elements + .iter() + .map(|element| { + string_literal_value(&self.model, element).map(|method| method.to_uppercase()) + }) + .collect() + } + + /// the route a decorator declares on `function` + fn decorator(&mut self, decorator: &ast::Decorator, function: &ast::StmtFunctionDef) { + let ast::Expr::Call(call) = &decorator.expression else { + return; + }; + let ast::Expr::Attribute(callee) = call.func.as_ref() else { + return; + }; + let name = callee.attr.as_str(); + let methods = if METHOD_DECORATORS.contains(&name) { + vec![name.to_uppercase()] + } else if name == API_ROUTE_DECORATOR { + let Some(methods) = self.methods(&call.arguments) else { + return; + }; + methods + } else { + return; + }; + let handler = ResolvedDefinition::Definition(function.definition(&self.model)); + self.route(call, callee, methods, handler); + } + + /// `add_api_route(path, endpoint, methods=...)`, which declares a route for a function + /// without decorating it + fn add_api_route(&mut self, call: &ast::ExprCall, callee: &ast::ExprAttribute) { + let Some(endpoint) = call.arguments.find_argument_value("endpoint", 1) else { + return; + }; + let Some(handler) = self.assignment(endpoint) else { + return; + }; + let Some(methods) = self.methods(&call.arguments) else { + return; + }; + self.route( + call, + callee, + methods, + ResolvedDefinition::Definition(handler), + ); + } + + fn route( + &mut self, + call: &ast::ExprCall, + callee: &ast::ExprAttribute, + methods: Vec, + handler: ResolvedDefinition<'db>, + ) { + let Some(owner_kind) = self.fastapi_method(callee) else { + return; + }; + let Some(owner) = self.assignment(&callee.value) else { + return; + }; + let Some(path) = call + .arguments + .find_argument_value("path", 0) + .and_then(|path| string_literal_value(&self.model, path)) + else { + return; + }; + self.scan.routes.push(Route { + owner, + owner_kind, + methods, + path, + handler, + declaration: FileRange::new(self.file.file(self.db), call.range()), + }); + } + + /// `include_router(router, prefix=...)`, which mounts a router on what it is called on + fn include_router(&mut self, call: &ast::ExprCall, callee: &ast::ExprAttribute) { + let Some(parent_kind) = self.fastapi_method(callee) else { + return; + }; + let (Some(parent), Some(router)) = ( + self.assignment(&callee.value), + call.arguments + .find_argument_value("router", 0) + .and_then(|router| self.assignment(router)), + ) else { + return; + }; + let prefix = match call.arguments.find_keyword("prefix") { + None => String::new(), + Some(prefix) => match string_literal_value(&self.model, &prefix.value) { + Some(prefix) => prefix, + None => return, + }, + }; + self.scan.includes.push(Include { + parent, + parent_kind, + router, + prefix, + }); + } +} + +impl<'a> SourceOrderVisitor<'a> for FastApiVisitor<'_, '_> { + fn visit_stmt(&mut self, statement: &'a ast::Stmt) { + if let ast::Stmt::FunctionDef(function) = statement { + for decorator in &function.decorator_list { + self.decorator(decorator, function); + } + } + source_order::walk_stmt(self, statement); + } + + fn visit_expr(&mut self, expression: &'a ast::Expr) { + if let ast::Expr::Call(call) = expression + && let ast::Expr::Attribute(callee) = call.func.as_ref() + { + match callee.attr.as_str() { + ADD_API_ROUTE => self.add_api_route(call, callee), + INCLUDE_ROUTER => self.include_router(call, callee), + _ => {} + } + } + source_order::walk_expr(self, expression); + } +} + +/// the prefix a router declares its routes under, `APIRouter(prefix="/items")`, read once per +/// router. empty for a router built without one, and `None` for one whose prefix ty does not know +/// exactly, or which is not built where it is assigned +fn router_prefix<'db>( + db: &'db dyn Db, + prefixes: &mut FxHashMap, Option>, + router: Definition<'db>, +) -> Option { + prefixes + .entry(router) + .or_insert_with(|| { + let file = router.program_file(db); + let parsed = parsed_module(db, file.python_file(db)).load(db); + let value = match router.kind(db) { + DefinitionKind::Assignment(assignment) => assignment.value(&parsed), + DefinitionKind::AnnotatedAssignment(assignment) => assignment.value(&parsed)?, + _ => return None, + }; + let ast::Expr::Call(call) = value else { + return None; + }; + match call.arguments.find_keyword("prefix") { + None => Some(String::new()), + Some(prefix) => string_literal_value(&SemanticModel::new(db, file), &prefix.value), + } + }) + .clone() +} + +/// the functions an entry of `urlpatterns` is built with, by what they resolve to +const DJANGO_PATH: &str = "django.urls.conf.path"; +const DJANGO_RE_PATH: &str = "django.urls.conf.re_path"; +const DJANGO_INCLUDE: &str = "django.urls.conf.include"; + +/// the method a class-based view is turned into a view function with +const DJANGO_AS_VIEW: &str = "django.views.generic.base.View.as_view"; + +/// what a urlconf module holds its routes in +const URLPATTERNS: &str = "urlpatterns"; + +/// the routes django serves, from the urlconf module `ROOT_URLCONF` names or, without one, from +/// every module of the project with `urlpatterns` that no other includes +fn django_endpoints(db: &dyn Db) -> Vec { + let roots = match *crate::django_template::django_root_urlconf(db, db.project()) { + Some(root) => vec![db.program_file(root)], + None => db + .project() + .files(db) + .iter() + .map(|file| db.program_file(file)) + .filter(|file| binds_urlpatterns(db, *file)) + .collect(), + }; + let mut walks: Vec<(ProgramFile<'_>, DjangoWalk<'_>)> = roots + .into_iter() + .map(|root| { + let mut walk = DjangoWalk { + db, + application: target(db, &ResolvedDefinition::Module(root)), + found: Vec::new(), + included: FxHashSet::default(), + walking: Vec::new(), + }; + walk.module(root, &[]); + (root, walk) + }) + .collect(); + let included: FxHashSet> = walks + .iter() + .flat_map(|(_, walk)| walk.included.iter().copied()) + .collect(); + if walks.len() > 1 { + walks.retain(|(root, _)| !included.contains(root)); + } + walks.into_iter().flat_map(|(_, walk)| walk.found).collect() +} + +/// whether `file` assigns `urlpatterns` at its top level +fn binds_urlpatterns(db: &dyn Db, file: ProgramFile<'_>) -> bool { + let parsed = parsed_module(db, file.python_file(db)).load(db); + parsed + .suite() + .iter() + .any(|statement| urlpatterns_value(statement).is_some()) +} + +/// the display a top-level statement assigns or adds to `urlpatterns` +fn urlpatterns_value(statement: &ast::Stmt) -> Option<&ast::Expr> { + let (target, value) = match statement { + ast::Stmt::Assign(assign) => match assign.targets.as_slice() { + [target] => (target, assign.value.as_ref()), + _ => return None, + }, + ast::Stmt::AnnAssign(assign) => (assign.target.as_ref(), assign.value.as_deref()?), + ast::Stmt::AugAssign(assign) if assign.op == ast::Operator::Add => { + (assign.target.as_ref(), assign.value.as_ref()) + } + _ => return None, + }; + (target.as_name_expr()?.id.as_str() == URLPATTERNS).then_some(value) +} + +/// a part of a route's pattern, as `path()` or `re_path()` writes it +#[derive(Clone)] +struct RoutePart { + text: String, + regex: bool, +} + +/// the url tree walked from one urlconf module +struct DjangoWalk<'db> { + db: &'db dyn Db, + application: EndpointTarget, + found: Vec, + /// every module an `include()` reached + included: FxHashSet>, + /// the modules whose patterns are being walked, outermost first: one that includes a module + /// among them includes itself, whose routes would nest under themselves without end + walking: Vec>, +} + +impl<'db> DjangoWalk<'db> { + fn module(&mut self, file: ProgramFile<'db>, prefix: &[RoutePart]) { + if self.walking.contains(&file) { + return; + } + self.walking.push(file); + let db = self.db; + let parsed = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + for statement in parsed.suite() { + if let Some(patterns) = urlpatterns_value(statement) { + self.patterns(&model, file, patterns, prefix); + } + } + self.walking.pop(); + } + + /// the entries of a list or tuple of patterns + fn patterns( + &mut self, + model: &SemanticModel<'db>, + file: ProgramFile<'db>, + patterns: &ast::Expr, + prefix: &[RoutePart], + ) { + let entries = match patterns { + ast::Expr::List(list) => &list.elts, + ast::Expr::Tuple(tuple) => &tuple.elts, + _ => return, + }; + for entry in entries { + self.entry(model, file, entry, prefix); + } + } + + fn entry( + &mut self, + model: &SemanticModel<'db>, + file: ProgramFile<'db>, + entry: &ast::Expr, + prefix: &[RoutePart], + ) { + let ast::Expr::Call(call) = entry else { + return; + }; + let regex = match callee(self.db, model, call).as_deref() { + Some(DJANGO_PATH) => false, + Some(DJANGO_RE_PATH) => true, + _ => return, + }; + let (Some(route), Some(view)) = ( + call.arguments + .find_argument_value("route", 0) + .and_then(|route| string_literal_value(model, route)), + call.arguments.find_argument_value("view", 1), + ) else { + return; + }; + let mut parts = prefix.to_vec(); + parts.push(RoutePart { text: route, regex }); + + if let ast::Expr::Call(view_call) = view { + match callee(self.db, model, view_call).as_deref() { + Some(DJANGO_INCLUDE) => { + if let Some(included) = view_call.arguments.find_argument_value("arg", 0) { + self.include(model, file, included, &parts); + } + return; + } + Some(DJANGO_AS_VIEW) => { + if let ast::Expr::Attribute(as_view) = view_call.func.as_ref() + && let Some(class) = single_definition(model, &as_view.value) + { + self.route(file, call, &parts, &class); + } + return; + } + _ => return, + } + } + if let Some(handler) = single_definition(model, view) { + self.route(file, call, &parts, &handler); + } + } + + /// what `include()` mounts: a module named by a string or by the module itself, a list of + /// patterns, or a `(patterns, app_name)` pair + fn include( + &mut self, + model: &SemanticModel<'db>, + file: ProgramFile<'db>, + included: &ast::Expr, + prefix: &[RoutePart], + ) { + let db = self.db; + let included = match included { + ast::Expr::Tuple(pair) => match pair.elts.first() { + Some(patterns) => patterns, + None => return, + }, + other => other, + }; + if matches!(included, ast::Expr::List(_) | ast::Expr::Tuple(_)) { + self.patterns(model, file, included, prefix); + return; + } + let module = if let Some(name) = string_literal_value(model, included) { + ModuleName::new(&name) + .and_then(|name| { + resolve_real_module( + db, + ImportingFile::File(file.file(db), file.resolver_environment(db)), + &name, + ) + }) + .and_then(|module| module.file(db)) + .map(|module| db.program_file(module)) + } else { + module_value_file(model, included).map(|module| db.program_file(module)) + }; + if let Some(module) = module { + self.included.insert(module); + self.module(module, prefix); + } + } + + fn route( + &mut self, + file: ProgramFile<'db>, + call: &ast::ExprCall, + parts: &[RoutePart], + handler: &ResolvedDefinition<'db>, + ) { + // django matches a route against the path without its leading slash, and a regular + // expression's anchors say where the part it matches starts and stops, which joined + // into one path they no longer do + let path = parts + .iter() + .map(|part| { + if part.regex { + let text = part.text.strip_prefix('^').unwrap_or(&part.text); + text.strip_suffix('$').unwrap_or(text) + } else { + &part.text + } + }) + .collect::(); + self.found.push(Endpoint { + framework: Framework::Django, + methods: Vec::new(), + path: format!("/{path}"), + handler: target(self.db, handler), + declaration: FileRange::new(file.file(self.db), call.range()), + application: Some(self.application.clone()), + }); + } +} + +/// the qualified name of what `call` calls, where every definition of it agrees +fn callee(db: &dyn Db, model: &SemanticModel<'_>, call: &ast::ExprCall) -> Option { + let definitions = GotoTarget::Expression(call.func.as_ref().into()) + .expression_definitions(model, ImportAliasResolution::ResolveAliases)?; + let mut names = definitions.iter().map(|resolved| { + declaration(db, resolved).and_then(|declaration| declaration.qualified_name) + }); + let first = names.next()??; + names + .all(|other| other.as_ref() == Some(&first)) + .then_some(first) +} + +/// the one thing `expr` names +fn single_definition<'db>( + model: &SemanticModel<'db>, + expr: &ast::Expr, +) -> Option> { + let definitions = GotoTarget::Expression(expr.into()) + .expression_definitions(model, ImportAliasResolution::ResolveAliases)?; + match definitions.iter().as_slice() { + [single] => Some(single.clone()), + _ => None, + } +} + +fn target(db: &dyn Db, definition: &ResolvedDefinition<'_>) -> EndpointTarget { + EndpointTarget { + name: declaration(db, definition).and_then(|declaration| declaration.qualified_name), + target: navigation_target(db, definition), + } +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + use ruff_text_size::Ranged; + + use crate::tests::CursorTest; + + const FASTAPI_ROUTING: &str = "\ +from typing import Any, Callable + +Decorator = Callable[[Callable[..., Any]], Callable[..., Any]] + +class APIRouter: + def __init__(self, *, prefix: str = '') -> None: ... + def get(self, path: str) -> Decorator: ... + def post(self, path: str) -> Decorator: ... + def delete(self, path: str) -> Decorator: ... + def api_route(self, path: str, *, methods: list[str] | None = None) -> Decorator: ... + def add_api_route( + self, path: str, endpoint: Callable[..., Any], *, methods: list[str] | None = None + ) -> None: ... + def include_router(self, router: APIRouter, *, prefix: str = '') -> None: ... +"; + + const FASTAPI_APPLICATIONS: &str = "\ +from typing import Any, Callable + +from fastapi.routing import APIRouter, Decorator + +class FastAPI: + def get(self, path: str) -> Decorator: ... + def post(self, path: str) -> Decorator: ... + def add_api_route( + self, path: str, endpoint: Callable[..., Any], *, methods: list[str] | None = None + ) -> None: ... + def include_router(self, router: APIRouter, *, prefix: str = '') -> None: ... +"; + + fn test(sources: &[(&str, &str)]) -> CursorTest { + let mut builder = CursorTest::builder(); + builder + .source( + "fastapi/__init__.py", + "from .applications import FastAPI as FastAPI\nfrom .routing import APIRouter as APIRouter\n", + ) + .source("fastapi/routing.py", FASTAPI_ROUTING) + .source("fastapi/applications.py", FASTAPI_APPLICATIONS); + for (path, source) in sources { + builder.source(*path, *source); + } + builder.build() + } + + impl CursorTest { + fn endpoints(&self) -> String { + super::endpoints(&self.db) + .iter() + .map(|endpoint| { + let file = endpoint.declaration.file(); + let source = source_text(&self.db, file); + let line = LineIndex::from_source_text(&source) + .line_index(endpoint.declaration.range().start()); + format!( + "{} {} {} -> {} declared at {}:{line} for {}", + endpoint.framework.as_str(), + endpoint.methods.join("|"), + endpoint.path, + endpoint.handler.name.as_deref().unwrap_or("-"), + file.path(&self.db).as_str(), + endpoint + .application + .as_ref() + .map_or("nothing", |application| application + .name + .as_deref() + .unwrap_or("-")), + ) + }) + .collect::>() + .join("\n") + } + } + + #[test] + fn a_route_is_served_at_every_prefix_it_is_mounted_under() { + let test = test(&[ + ( + "app/items.py", + "\ +from fastapi import APIRouter + +PREFIX = '/items' +router = APIRouter(prefix=PREFIX) + +@router.get('/{item_id}') +def read_item(item_id: int) -> int: + return item_id + +@router.api_route('/', methods=['get', 'POST']) +def items() -> None: ... +", + ), + ( + "app/users.py", + "\ +from fastapi import APIRouter + +users_router = APIRouter(prefix='/users') +admin = APIRouter() + +@admin.delete('/{user_id}') +def remove(user_id: int) -> None: ... + +users_router.include_router(admin, prefix='/admin') +", + ), + ( + "app/main.py", + "\ +from fastapi import FastAPI as Application +from app import items +from app.users import users_router + +app = Application() +app.include_router(items.router, prefix='/v1') +app.include_router(users_router) + +@app.get('/health') +def health() -> str: ... + +def ping() -> str: ... + +app.add_api_route('/ping', ping, methods=['HEAD']) +", + ), + ]); + assert_snapshot!(test.endpoints(), @" + fastapi GET /health -> app.main.health declared at /app/main.py:9 for app.main.app + fastapi HEAD /ping -> app.main.ping declared at /app/main.py:14 for app.main.app + fastapi DELETE /users/admin/{user_id} -> app.users.remove declared at /app/users.py:6 for app.main.app + fastapi GET|POST /v1/items/ -> app.items.items declared at /app/items.py:10 for app.main.app + fastapi GET /v1/items/{item_id} -> app.items.read_item declared at /app/items.py:6 for app.main.app + "); + } + + #[test] + fn a_route_on_a_router_nothing_includes_is_served_by_nothing() { + let test = test(&[( + "main.py", + "\ +from fastapi import APIRouter + +orphan = APIRouter(prefix='/orphan') + +@orphan.post('/x') +def x() -> None: ... +", + )]); + assert_snapshot!(test.endpoints(), @"fastapi POST /orphan/x -> main.x declared at /main.py:5 for nothing"); + } + + #[test] + fn a_method_named_like_a_route_is_not_one() { + let test = test(&[( + "main.py", + "\ +from typing import Any, Callable + +class Registry: + def get(self, path: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: ... + +registry = Registry() + +@registry.get('/nope') +def nope() -> None: ... +", + )]); + assert_snapshot!(test.endpoints(), @""); + } + + #[test] + fn a_path_that_is_not_known_exactly_is_left_out() { + let test = test(&[( + "main.py", + "\ +from fastapi import FastAPI + +app = FastAPI() + +def route(path: str) -> None: + @app.get(path) + def handler() -> None: ... +", + )]); + assert_snapshot!(test.endpoints(), @""); + } + + fn django_test(sources: &[(&str, &str)]) -> CursorTest { + let mut builder = CursorTest::builder(); + builder + .source( + "django/urls/__init__.py", + "from .conf import include as include, path as path, re_path as re_path\n", + ) + .source( + "django/urls/conf.py", + "\ +from typing import Any + +def include(arg: Any) -> Any: ... +def path(route: str, view: Any, name: str | None = None) -> Any: ... +def re_path(route: str, view: Any, name: str | None = None) -> Any: ... +", + ) + .source( + "django/views/generic/base.py", + "\ +from typing import Any, Callable + +class View: + @classmethod + def as_view(cls) -> Callable[..., Any]: ... +", + ); + for (path, source) in sources { + builder.source(*path, *source); + } + builder.build() + } + + #[test] + fn a_django_route_is_served_behind_every_include_it_is_mounted_under() { + let test = django_test(&[ + ( + "blog/views.py", + "\ +from django.views.generic.base import View + +def index(request: object) -> None: ... +def archive(request: object, year: str) -> None: ... + +class PostDetail(View): ... +", + ), + ( + "blog/urls.py", + "\ +from django.urls import path, re_path + +from . import views + +urlpatterns = [ + path('', views.index), + path('/', views.PostDetail.as_view()), +] +urlpatterns += [re_path(r'^archive/(?P[0-9]{4})/$', views.archive)] +", + ), + ( + "mysite/urls.py", + "\ +from django.urls import include, path + +import blog.urls +from blog import views + +urlpatterns = [ + path('blog/', include('blog.urls')), + path('again/', include(blog.urls)), + path('inline/', include([path('index/', views.index)])), +] +", + ), + ]); + assert_snapshot!(test.endpoints(), @" + django /again/ -> blog.views.index declared at /blog/urls.py:6 for mysite.urls + django /again// -> blog.views.PostDetail declared at /blog/urls.py:7 for mysite.urls + django /again/archive/(?P[0-9]{4})/ -> blog.views.archive declared at /blog/urls.py:9 for mysite.urls + django /blog/ -> blog.views.index declared at /blog/urls.py:6 for mysite.urls + django /blog// -> blog.views.PostDetail declared at /blog/urls.py:7 for mysite.urls + django /blog/archive/(?P[0-9]{4})/ -> blog.views.archive declared at /blog/urls.py:9 for mysite.urls + django /inline/index/ -> blog.views.index declared at /mysite/urls.py:9 for mysite.urls + "); + } + + /// a urlconf that includes itself mounts its own routes under itself without end, which no + /// request can reach the bottom of, so the include that closes the cycle is left out + #[test] + fn a_urlconf_that_includes_itself_is_walked_once() { + let test = django_test(&[( + "mysite/urls.py", + "\ +from django.urls import include, path + +def index(request: object) -> None: ... + +urlpatterns = [ + path('again/', include('mysite.urls')), + path('a/', include([path('b/', include([path('c/', index)]))])), +] +", + )]); + assert_snapshot!(test.endpoints(), @"django /a/b/c/ -> mysite.urls.index declared at /mysite/urls.py:7 for mysite.urls"); + } + + #[test] + fn a_call_named_like_path_is_not_a_route() { + let test = django_test(&[( + "urls.py", + "\ +def path(route: str, view: object) -> object: ... +def index() -> None: ... + +urlpatterns = [path('nope/', index)] +", + )]); + assert_snapshot!(test.endpoints(), @""); + } +} diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index a86e396255..6dd1223ddd 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -19,6 +19,7 @@ mod doc_highlights; mod docstring; mod docstring_stub; mod document_symbols; +mod endpoints; mod evaluatable_expression; mod expression_types; mod find_references; @@ -88,6 +89,7 @@ pub use django_template::{ pub use doc_highlights::document_highlights; pub use docstring_stub::{DocstringFormat, DocstringStub, docstring_stub}; pub use document_symbols::{document_symbols, symbol_detail}; +pub use endpoints::{Endpoint, EndpointTarget, Framework, endpoints}; pub use evaluatable_expression::evaluatable_expression; pub use expression_types::{ExpressionType, expression_types}; pub use find_references::find_references; diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 7919b082a9..3eba1692bc 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -2984,6 +2984,15 @@ pub fn folds_to_a_literal(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { ) } +/// the string `expr` evaluates to, where ty knows it exactly: a string literal, or a name or an +/// expression whose type is one +pub fn string_literal_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> Option { + match expr.inferred_type(model)?.as_literal_value_kind()? { + LiteralValueTypeKind::String(literal) => Some(literal.value(model.db()).to_string()), + _ => None, + } +} + /// One of the builtin types whose operators are python's own: none of them runs code of the /// program's. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -3126,6 +3135,14 @@ pub fn is_module_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { matches!(expr.inferred_type(model), Some(Type::ModuleLiteral(_))) } +/// the file of the module `expr` evaluates to: `blog.urls` after `import blog.urls` +pub fn module_value_file(model: &SemanticModel<'_>, expr: &ast::Expr) -> Option { + match expr.inferred_type(model)? { + Type::ModuleLiteral(module) => module.module(model.db()).file(model.db()), + _ => None, + } +} + pub fn inferred_raises<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index d23b12418a..c113d23167 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -159,6 +159,10 @@ pub(super) fn request(req: server::Request) -> Task { requests::RunModulesHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } + // every route the project serves, which an endpoints view lists off its own schedule + requests::EndpointsHandler::METHOD => { + background_request_task::(req, BackgroundSchedule::Worker) + } requests::TranspileRequestHandler::METHOD => background_document_request_task::< requests::TranspileRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index d2d59ecd67..65b98a1e4f 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -35,6 +35,7 @@ mod document_inherited_members; mod document_super_members; mod document_symbols; mod document_usage_counts; +mod endpoints; mod entry_point; mod evaluatable_expression; mod execute_command; @@ -108,6 +109,7 @@ pub(super) use document_inherited_members::DocumentInheritedMembersRequestHandle pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use document_usage_counts::DocumentUsageCountsRequestHandler; +pub(super) use endpoints::EndpointsHandler; pub(super) use entry_point::EntryPointHandler; pub(super) use evaluatable_expression::EvaluatableExpressionRequestHandler; pub(super) use execute_command::ExecuteCommand; diff --git a/crates/ty_server/src/server/api/requests/endpoints.rs b/crates/ty_server/src/server/api/requests/endpoints.rs new file mode 100644 index 0000000000..3748acdde7 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/endpoints.rs @@ -0,0 +1,138 @@ +//! `by/endpoints` — the HTTP routes a project serves, for an editor's endpoints view +//! +//! each route with the methods it serves, the whole path it is served at, its handler, where the +//! route is declared and the application that serves it, read through what the code means: see +//! [`ty_ide::endpoints`]. asked of every project in the session, as `workspace/symbol` is + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, Uri}; +use ty_ide::{EndpointTarget, endpoints}; + +use crate::Db; +use crate::PositionEncoding; +use crate::document::{FileRangeExt, ToLink}; +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum EndpointsRequest {} + +impl Request for EndpointsRequest { + type Params = Option; + type Result = Vec; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/endpoints"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// nothing: the answer is every project's +#[derive(Debug, Default, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +#[expect( + clippy::empty_structs_with_brackets, + reason = "the brackets make it the empty object a client sends, which a unit struct is not" +)] +pub(crate) struct EndpointsParams {} + +/// one route +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Endpoint { + /// `fastapi` or `django` + framework: String, + /// the HTTP methods, upper case. empty for a route that serves every method, as django's do + methods: Vec, + /// the path the route is served at, every prefix it is mounted under included: `/v1/items/{item_id}` + path: String, + /// what handles a request to it + handler: Target, + /// where the route is declared: the decorator, `add_api_route(...)` or `path(...)` + declaration: Location, + /// the FastAPI application, or the urlconf module django starts at. absent for a route on a + /// router no application includes + #[serde(skip_serializing_if = "Option::is_none")] + application: Option, +} + +/// a declaration an endpoint names +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Target { + /// the dotted name it is reached by, `app.items.read_item`; absent where it has none + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + uri: Uri, + /// the whole declaration + range: Range, + /// its name + selection_range: Range, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct Location { + uri: Uri, + range: Range, +} + +pub(crate) struct EndpointsHandler; + +impl RequestHandler for EndpointsHandler { + type RequestType = EndpointsRequest; +} + +impl BackgroundRequestHandler for EndpointsHandler { + fn run( + snapshot: &SessionSnapshot, + _client: &Client, + _params: Option, + ) -> crate::server::Result> { + let encoding = snapshot.position_encoding(); + let mut found = Vec::new(); + for db in snapshot.projects() { + for endpoint in endpoints(db) { + // an application that cannot be linked is left out with the rest of the route, + // as a handler is, rather than sent as absent, which says no application serves it + let application = match &endpoint.application { + Some(application) => match target(db, application, encoding) { + Some(application) => Some(application), + None => continue, + }, + None => None, + }; + let (Some(handler), Some(declaration)) = ( + target(db, &endpoint.handler, encoding), + endpoint + .declaration + .to_lsp_range(db, encoding) + .and_then(|range| range.to_location()), + ) else { + continue; + }; + found.push(Endpoint { + framework: endpoint.framework.as_str().to_string(), + methods: endpoint.methods, + path: endpoint.path, + handler, + declaration: Location { + uri: declaration.uri, + range: declaration.range, + }, + application, + }); + } + } + Ok(found) + } +} + +fn target(db: &dyn Db, target: &EndpointTarget, encoding: PositionEncoding) -> Option { + let link = target.target.to_link(db, None, encoding)?; + Some(Target { + name: target.name.clone(), + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + }) +} + +impl RetriableRequestHandler for EndpointsHandler {} diff --git a/crates/ty_server/tests/e2e/endpoints.rs b/crates/ty_server/tests/e2e/endpoints.rs new file mode 100644 index 0000000000..f7ce35ef09 --- /dev/null +++ b/crates/ty_server/tests/e2e/endpoints.rs @@ -0,0 +1,71 @@ +//! `by/endpoints`: the HTTP routes a project serves, for an editor's endpoints view + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use serde_json::{Value, json}; + +use crate::TestServerBuilder; + +enum Endpoints {} + +impl Request for Endpoints { + type Params = Value; + type Result = Value; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/endpoints"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[test] +fn a_route_is_listed_with_its_whole_path_handler_and_application() -> Result<()> { + let main = SystemPath::new("project/app/main.by"); + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file( + SystemPath::new("project/fastapi/__init__.py"), + "from .applications import FastAPI as FastAPI\nfrom .routing import APIRouter as APIRouter\n", + )? + .with_file( + SystemPath::new("project/fastapi/routing.py"), + "from typing import Any, Callable\n\nclass APIRouter:\n def __init__(self, *, prefix: str = '') -> None: ...\n def get(self, path: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: ...\n", + )? + .with_file( + SystemPath::new("project/fastapi/applications.py"), + "from fastapi.routing import APIRouter\n\nclass FastAPI:\n def include_router(self, router: APIRouter, *, prefix: str = '') -> None: ...\n", + )? + .with_file(SystemPath::new("project/app/__init__.by"), "")? + .with_file( + main, + "from fastapi import APIRouter, FastAPI\n\nrouter = APIRouter(prefix='/items')\n\n@router.get('/{item_id}')\ndef read_item(item_id: int) -> int:\n return item_id\n\napp = FastAPI()\napp.include_router(router, prefix='/v1')\n", + )? + .build() + .wait_until_workspaces_are_initialized(); + + let response = server.send_request_await::(json!({})); + let uri = server.file_uri(main); + assert_eq!( + response, + json!([{ + "framework": "fastapi", + "methods": ["GET"], + "path": "/v1/items/{item_id}", + "handler": { + "name": "app.main.read_item", + "uri": uri, + "range": {"start": {"line": 4, "character": 0}, "end": {"line": 6, "character": 18}}, + "selectionRange": {"start": {"line": 5, "character": 4}, "end": {"line": 5, "character": 13}}, + }, + "declaration": { + "uri": uri, + "range": {"start": {"line": 4, "character": 1}, "end": {"line": 4, "character": 25}}, + }, + "application": { + "name": "app.main.app", + "uri": uri, + "range": {"start": {"line": 8, "character": 0}, "end": {"line": 8, "character": 15}}, + "selectionRange": {"start": {"line": 8, "character": 0}, "end": {"line": 8, "character": 3}}, + }, + }]) + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index b6ac9e4f5f..c310687e9c 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -45,6 +45,7 @@ mod declaration; mod diagnostic_snapshots; mod django_templates; mod docstring_stub; +mod endpoints; mod evaluatable_expression; mod expression_types; mod file_watching; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 18248a202f..11b89f0e65 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1471,6 +1471,66 @@ every `locationHint` is `file://` and the absolute path of the project file, wit `:` and the 1-based line of the `def` or `class` for a test or a class. any other value of `BY_TEST_REPORTER` is refused as a usage error +## endpoints + +`by/endpoints` answers an editor's endpoints view: every HTTP route the project +serves, from FastAPI and from django. it takes `{}` and answers a list, from every +project in the session, ordered by path and then by method, each + +```json +{ + "framework": "fastapi", + "methods": ["GET"], + "path": "/v1/items/{item_id}", + "handler": { + "name": "app.items.read_item", + "uri": "file:///project/app/items.by", + "range": {"start": {"line": 6, "character": 0}, "end": {"line": 8, "character": 22}}, + "selectionRange": {"start": {"line": 7, "character": 4}, "end": {"line": 7, "character": 13}} + }, + "declaration": { + "uri": "file:///project/app/items.by", + "range": {"start": {"line": 6, "character": 1}, "end": {"line": 6, "character": 25}} + }, + "application": { + "name": "app.main.app", + "uri": "file:///project/app/main.by", + "range": {"start": {"line": 4, "character": 0}, "end": {"line": 4, "character": 15}}, + "selectionRange": {"start": {"line": 4, "character": 0}, "end": {"line": 4, "character": 3}} + } +} +``` + +`framework` is `fastapi` or `django`. `methods` are upper case, and empty for a +route that serves every method, as each of django's does. `path` is the whole +path, every prefix the route is mounted under included, with its parameters as +the route writes them, `{item_id}` or ``; a `re_path()` pattern is +joined to its prefixes with each part's `^` and `$` anchors dropped. `handler` is +the function, or for a django view built by `as_view()` the class, with its +qualified name, its whole declaration as `range` and its name as +`selectionRange`; `declaration` is where the route is declared, the decorator +without its `@`, the `add_api_route(...)` or the `path(...)`; `application` is the +FastAPI application that serves it, or the urlconf module django starts at, and +is absent for a route on a router no application includes. a route served under +two applications, or mounted twice, is listed once for each path. + +every part is read through what the code means. a FastAPI route is a decorator, +or an `add_api_route` call, that resolves to a method of `fastapi.FastAPI` or +`fastapi.APIRouter`, `get` to `trace` for that method and `api_route` for its +`methods=` (GET without it), however the application or router is named or +imported; a router's routes are mounted by an `include_router` handed the same +router, under the include's `prefix` and the router's own `APIRouter(prefix=)`, +through any number of routers between it and an application. a django route is +an entry of a urlconf module's `urlpatterns` that calls `django.urls.path` or +`re_path`, read from the module `ROOT_URLCONF` names, or, in a project whose +settings name none, from every module with `urlpatterns` that no other includes; +an `include()` is followed to the module a string names, a module object, a list +of patterns, or the patterns of a `(patterns, app_name)` pair. a path, a prefix +or a method that is not a string ty knows exactly — a literal, or a constant +holding one — leaves its route out rather than listing it at a path it may not +have, and so does a FastAPI router built somewhere other than where it is +assigned, and a route whose handler, declaration or application cannot be opened + ## override and implement members an editor's *override methods* and *implement methods* ask twice: once for what From b13864200071e6361bd7dbc885988b0222b55181 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:11:25 +1000 Subject: [PATCH 057/110] answer by/typeDeclaration, the class a dotted type name names, shapes.base.Shape.Meta, resolved as an import of it would be through submodules and re-exports to its source or its stub, for a debugger's jump to type source, which knows a value's type only by its module and qualified name --- crates/ty_ide/src/goto.rs | 2 +- crates/ty_ide/src/lib.rs | 2 + crates/ty_ide/src/type_name.rs | 117 ++++++++++++++++++ crates/ty_server/src/server/api.rs | 4 + crates/ty_server/src/server/api/requests.rs | 2 + .../server/api/requests/type_declaration.rs | 95 ++++++++++++++ crates/ty_server/tests/e2e/declaration.rs | 41 ++++++ docs/basedpython/features/editor.md | 17 +++ 8 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 crates/ty_ide/src/type_name.rs create mode 100644 crates/ty_server/src/server/api/requests/type_declaration.rs diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index 46b16709ba..e82a354bc6 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -451,7 +451,7 @@ impl<'db> Definitions<'db> { } /// Map definitions from stub files to corresponding source implementations. - fn map_stubs(self, db: &'db dyn ty_python_semantic::Db) -> Definitions<'db> { + pub(crate) fn map_stubs(self, db: &'db dyn ty_python_semantic::Db) -> Definitions<'db> { let resolved = StubMapper::new(db).map_definitions(self.0); Self::new(resolved) } diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 6dd1223ddd..ef9701ac6d 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -57,6 +57,7 @@ mod symbols; mod syntax_outline; mod test_links; mod type_hierarchy; +mod type_name; mod unwrap; mod usage_counts; mod usage_kinds; @@ -143,6 +144,7 @@ pub use type_hierarchy::{ TypeHierarchyItem, prepare_type_hierarchy, type_hierarchy_mro, type_hierarchy_subtypes, type_hierarchy_supertypes, }; +pub use type_name::type_declaration; pub use unwrap::{UnwrapOption, unwrap_options}; pub use usage_counts::{ DeclarationKind, DeclarationUsages, Implementor, UsageCountKinds, document_usage_counts, diff --git a/crates/ty_ide/src/type_name.rs b/crates/ty_ide/src/type_name.rs new file mode 100644 index 0000000000..f7c38730a1 --- /dev/null +++ b/crates/ty_ide/src/type_name.rs @@ -0,0 +1,117 @@ +//! The class a dotted type name names: what a debugger's *jump to type source* asks. +//! +//! A debugger knows a value's type by name, `type(value).__module__` and its `__qualname__`: +//! `shapes.base.Circle`, or `shapes.base.Shape.Meta` for a class nested in another. The name is +//! resolved as an import of it would be — its module from the search paths, then each name after it +//! as a member of what came before, through re-exports and submodules — so `builtins.list` is the +//! class typeshed declares and `fastapi.FastAPI` the one its package does. A name that reaches +//! something other than a class, or nothing, as `make..Local` does, is not answered. + +use ty_module_resolver::ModuleName; +use ty_python_core::ProgramFile; +use ty_python_semantic::SemanticModel; +use ty_python_semantic::types::Type; + +use crate::Db; +use crate::declaration::{DeclarationAt, single_declaration}; +use crate::goto::Definitions; + +/// The class `type_name` names, as a module of the program `context` is in would import it, with +/// its declaration where its source is, or its stub for a class that has no other. +pub fn type_declaration( + db: &dyn Db, + context: ProgramFile<'_>, + type_name: &str, +) -> Option { + let model = SemanticModel::new(db, context); + let mut parts = type_name.split('.'); + let module = model.resolve_module_name(&ModuleName::new(parts.next()?)?)?; + let path: Vec<&str> = parts.collect(); + let ty = model.path_type(module, &path)?; + if !matches!(ty, Type::ClassLiteral(_)) { + return None; + } + let definitions = Definitions::from_ty(db, &model.program_environment(), ty)?.map_stubs(db); + let declaration = single_declaration(db, &definitions)?; + let target = definitions.into_navigation_targets(db).into_iter().next()?; + Some(DeclarationAt { + declaration, + target, + }) +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + + use crate::tests::CursorTest; + + impl CursorTest { + fn type_declaration(&self, type_name: &str) -> String { + let Some(found) = + super::type_declaration(&self.db, self.program_file(self.cursor.file), type_name) + else { + return "None".to_string(); + }; + let file = found.target.file(); + let source = source_text(&self.db, file); + let line = + LineIndex::from_source_text(&source).line_index(found.target.focus_range().start()); + format!( + "{} {} at {}:{line}", + found.declaration.kind.as_str(), + found.declaration.qualified_name.as_deref().unwrap_or("-"), + file.path(&self.db).as_str(), + ) + } + } + + fn test() -> CursorTest { + CursorTest::builder() + .source("shapes/__init__.py", "from .base import Circle as Circle\n") + .source( + "shapes/base.py", + "\ +class Shape: + class Meta: ... + +class Circle(Shape): ... + +RADIUS = 1.0 + +def make() -> None: + class Local: ... +", + ) + .source("main.py", "") + .build() + } + + #[test] + fn a_class_is_found_by_its_module_and_qualified_name() { + let test = test(); + assert_snapshot!(test.type_declaration("shapes.base.Circle"), @"class shapes.base.Circle at /shapes/base.py:4"); + assert_snapshot!(test.type_declaration("shapes.base.Shape.Meta"), @"class shapes.base.Shape.Meta at /shapes/base.py:2"); + } + + #[test] + fn a_class_is_found_through_a_re_export() { + assert_snapshot!(test().type_declaration("shapes.Circle"), @"class shapes.base.Circle at /shapes/base.py:4"); + } + + #[test] + fn a_builtin_is_its_stub() { + assert_snapshot!(test().type_declaration("builtins.list"), @"class builtins.list at stdlib/builtins.byi:2659"); + } + + #[test] + fn what_is_not_a_class_is_not_answered() { + let test = test(); + assert_snapshot!(test.type_declaration("shapes.base.RADIUS"), @"None"); + assert_snapshot!(test.type_declaration("shapes.base"), @"None"); + assert_snapshot!(test.type_declaration("shapes.base.make..Local"), @"None"); + assert_snapshot!(test.type_declaration("nowhere.Thing"), @"None"); + } +} diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index c113d23167..160aedf628 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -163,6 +163,10 @@ pub(super) fn request(req: server::Request) -> Task { requests::EndpointsHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } + // The class a debugger names a value's type by, which is in no document it has open. + requests::TypeDeclarationHandler::METHOD => background_request_task::< + requests::TypeDeclarationHandler, + >(req, BackgroundSchedule::Worker), requests::TranspileRequestHandler::METHOD => background_document_request_task::< requests::TranspileRequestHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 65b98a1e4f..8c9bb166fb 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -77,6 +77,7 @@ mod test_items; mod test_subjects; mod transpile; mod transpile_for_build; +mod type_declaration; mod type_hierarchy_subtypes; mod type_hierarchy_supertypes; mod unwrap; @@ -154,6 +155,7 @@ pub(super) use test_items::TestItemsHandler; pub(super) use test_subjects::{SubjectTestsRequestHandler, TestSubjectsRequestHandler}; pub(super) use transpile::TranspileRequestHandler; pub(super) use transpile_for_build::TranspileForBuildRequestHandler; +pub(super) use type_declaration::TypeDeclarationHandler; pub(super) use type_hierarchy_subtypes::TypeHierarchySubtypesRequestHandler; pub(super) use type_hierarchy_supertypes::TypeHierarchySupertypesRequestHandler; pub(super) use unwrap::UnwrapRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/type_declaration.rs b/crates/ty_server/src/server/api/requests/type_declaration.rs new file mode 100644 index 0000000000..1d01198547 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/type_declaration.rs @@ -0,0 +1,95 @@ +//! `by/typeDeclaration` — the class a dotted type name names, for a debugger's *jump to type +//! source*. +//! +//! A debugger knows a value's type by its module and qualified name, `shapes.base.Circle`, and +//! has no position in a document to ask `by/declaration` about; `workspace/symbol` matches a +//! query loosely and only among the project's own files, where `builtins.list` or a package's +//! class is not. The name is resolved here as an import of it would be: see +//! [`ty_ide::type_declaration`]. + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, Uri}; +use ty_ide::type_declaration; +use ty_project::{Db as _, SemanticDb as _}; + +use crate::document::ToLink; +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum TypeDeclarationRequest {} + +impl Request for TypeDeclarationRequest { + type Params = TypeDeclarationParams; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/typeDeclaration"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct TypeDeclarationParams { + /// The type's module and qualified name, dot-separated: `shapes.base.Shape.Meta`. + type_name: String, +} + +/// The class, as `by/declaration` describes a declaration. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TypeDeclaration { + /// Always `class`. + kind: String, + name: String, + /// The class's own module and qualified name, which differs from the name asked about for a + /// class reached through a re-export. + qualified_name: Option, + uri: Uri, + /// The whole class. + range: Range, + /// Its name. + selection_range: Range, +} + +pub(crate) struct TypeDeclarationHandler; + +impl RequestHandler for TypeDeclarationHandler { + type RequestType = TypeDeclarationRequest; +} + +impl BackgroundRequestHandler for TypeDeclarationHandler { + /// The first project in the session whose environment the name resolves in; `null` for a name + /// that names no class. + fn run( + snapshot: &SessionSnapshot, + _client: &Client, + params: TypeDeclarationParams, + ) -> crate::server::Result> { + let encoding = snapshot.position_encoding(); + for db in snapshot.projects() { + // the project's environment is what resolves a module, and any file of it stands in + // for the module an import of the name would be written in + let Some(context) = db.project().files(db).iter().next() else { + continue; + }; + let Some(found) = type_declaration(db, db.program_file(context), ¶ms.type_name) + else { + continue; + }; + let Some(link) = found.target.to_link(db, None, encoding) else { + continue; + }; + return Ok(Some(TypeDeclaration { + kind: found.declaration.kind.as_str().to_string(), + name: found.declaration.name, + qualified_name: found.declaration.qualified_name, + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + })); + } + Ok(None) + } +} + +impl RetriableRequestHandler for TypeDeclarationHandler {} diff --git a/crates/ty_server/tests/e2e/declaration.rs b/crates/ty_server/tests/e2e/declaration.rs index 6d8c6b0332..d8d62b66d1 100644 --- a/crates/ty_server/tests/e2e/declaration.rs +++ b/crates/ty_server/tests/e2e/declaration.rs @@ -125,3 +125,44 @@ fn no_name_answers_null() -> Result<()> { assert_eq!(ask(&mut server, 1, 0), None); Ok(()) } + +/// `by/typeDeclaration`: a type name in, the class it names out, as `by/declaration` describes one +enum TypeDeclaration {} + +impl Request for TypeDeclaration { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/typeDeclaration"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[test] +fn a_type_name_is_answered_with_the_class_it_names() -> Result<()> { + let base = SystemPath::new("project/shapes/base.by"); + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(SystemPath::new("project/shapes/__init__.by"), "")? + .with_file(base, BASE)? + .build() + .wait_until_workspaces_are_initialized(); + + let found = server.send_request_await::( + serde_json::json!({ "typeName": "shapes.base.Circle" }), + ); + assert_eq!( + found, + Some(serde_json::json!({ + "kind": "class", + "name": "Circle", + "qualifiedName": "shapes.base.Circle", + "uri": server.file_uri(base), + "range": {"start": {"line": 0, "character": 0}, "end": {"line": 5, "character": 26}}, + "selectionRange": {"start": {"line": 0, "character": 6}, "end": {"line": 0, "character": 12}}, + })) + ); + let local = server.send_request_await::( + serde_json::json!({ "typeName": "shapes.base.make..Local" }), + ); + assert_eq!(local, None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 11b89f0e65..39291d3e15 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1805,6 +1805,23 @@ between: bind — a global it only reads, an attribute it never assigns — is a value nothing in that scope can vouch for +### a value's type + +a debugger names a value's type by its module and qualified name, +`type(value).__module__` and `__qualname__`: `shapes.base.Circle`, or +`shapes.base.Shape.Meta` for a class nested in another. `by/typeDeclaration` +takes `{"typeName": "shapes.base.Circle"}` and answers the class that names, as +`by/declaration` describes a declaration — `kind` (always `class`), `name`, its +`qualifiedName`, and `uri`, `range` and `selectionRange` — for an editor's *jump +to type source*. the name is resolved as an import of it would be: its first part +as a module of the project's environment, and each part after it as a member of +what came before, through submodules and re-exports, so `shapes.Circle`, where +`shapes/__init__` re-exports it, answers `shapes.base.Circle`. a class with a +source is answered at its source, and one with only a stub, as `builtins.list`, +at its stub. the answer is `null` for a name that reaches anything other than a +class, or nothing, as `make..Local` does. a session of several projects +is answered from the first whose environment resolves it + ## the expression under the cursor a debugger's value tooltip, and *Evaluate Expression* or *Quick Evaluate* with From ca43b015d0bab71d45d071dc1e647e69575d8be5 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:27:38 +1000 Subject: [PATCH 058/110] answer by/resolveMergeConflicts, resolving a merge's conflict where the sides' changes touch nothing in common, imports either side added or removed sorted by buff's isort configuration, functions, classes and methods each side added, elements each added to a display or a decorator list, and sides differing only in formatting, and by/diffIgnoredRanges, a text's import statements and the whitespace its tokenizer discards, for a diff ignoring imports and formatting --- .../ruff_linter/src/rules/isort/placement.rs | 52 + crates/ty_ide/src/diff_ignored.rs | 134 + crates/ty_ide/src/import_order.rs | 48 +- crates/ty_ide/src/lib.rs | 4 + crates/ty_ide/src/merge.rs | 2368 +++++++++++++++++ crates/ty_ide/src/refactor.rs | 2 +- crates/ty_ide/src/type_name.rs | 14 +- crates/ty_server/src/server/api.rs | 9 +- crates/ty_server/src/server/api/requests.rs | 2 + .../server/api/requests/merge_conflicts.rs | 197 ++ .../server/api/requests/type_declaration.rs | 26 +- crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/merge_conflicts.rs | 91 + docs/basedpython/features/editor.md | 62 + 14 files changed, 2982 insertions(+), 28 deletions(-) create mode 100644 crates/ty_ide/src/diff_ignored.rs create mode 100644 crates/ty_ide/src/merge.rs create mode 100644 crates/ty_server/src/server/api/requests/merge_conflicts.rs create mode 100644 crates/ty_server/tests/e2e/merge_conflicts.rs diff --git a/crates/ruff_linter/src/rules/isort/placement.rs b/crates/ruff_linter/src/rules/isort/placement.rs index 70f0b1c8c4..a06e151ae9 100644 --- a/crates/ruff_linter/src/rules/isort/placement.rs +++ b/crates/ruff_linter/src/rules/isort/placement.rs @@ -206,6 +206,58 @@ impl<'a> ImportOrder<'a> { } } +/// `source`, a run of import statements and nothing else, sorted and written as `I001`'s fix +/// writes an import block with the same settings: one block, deduplicated, `from` imports of one +/// module combined, in sections. `None` for source with anything other than import statements in +/// it, or a comment, which isort keeps beside the import it is on and this cannot place. +pub fn sort_import_statements( + source: &str, + settings: &crate::settings::LinterSettings, + package: Option>, + source_type: ruff_python_ast::PySourceType, + target_version: PythonVersion, +) -> Option { + use ruff_python_codegen::Stylist; + use ruff_python_index::Indexer; + + let parsed = ruff_python_parser::parse_unchecked_source(source, source_type); + if parsed.has_syntax_errors() { + return None; + } + let indexer = Indexer::from_tokens(parsed.tokens(), source); + if !indexer.comment_ranges().is_empty() { + return None; + } + let imports: Vec<&ruff_python_ast::Stmt> = parsed.suite().iter().collect(); + if imports + .iter() + .any(|statement| !(statement.is_import_stmt() || statement.is_import_from_stmt())) + { + return None; + } + let block = super::block::Block { + nested: false, + imports, + trailer: None, + }; + let locator = crate::Locator::new(source); + let stylist = Stylist::from_tokens(parsed.tokens(), source); + Some(super::format_imports( + &block, + Vec::new(), + &locator, + settings.line_length, + crate::line_width::LineWidthBuilder::new(settings.tab_size), + &stylist, + &settings.src, + package, + source_type, + target_version, + &settings.isort, + parsed.tokens(), + )) +} + #[cfg(test)] mod tests { use ruff_python_ast::PythonVersion; diff --git a/crates/ty_ide/src/diff_ignored.rs b/crates/ty_ide/src/diff_ignored.rs new file mode 100644 index 0000000000..9f136ea8d3 --- /dev/null +++ b/crates/ty_ide/src/diff_ignored.rs @@ -0,0 +1,134 @@ +//! The parts of a file a diff may ignore when asked to ignore imports and formatting. +//! +//! Imports are every import statement, wherever it is. Formatting is the text python's tokenizer +//! discards without its being significant: whitespace between two tokens of a logical line, the +//! line breaks and indentation of a line continued inside brackets or after a backslash, and blank +//! lines. The indentation a logical line starts with is not formatting, since it says which block +//! the line is in, and nor is anything inside an f-string or a t-string, where `f"{x = }"` prints +//! the spaces around its `=`. Two versions that differ only in what is ignored here tokenize to +//! the same tokens. + +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; +use ruff_python_ast::token::TokenKind; +use ruff_python_ast::{PySourceType, Stmt}; +use ruff_python_parser::parse_unchecked_source; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +/// What a diff ignoring imports and formatting ignores in one text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiffIgnoredRanges { + /// Each import statement. + pub imports: Vec, + /// Each run of text that is only formatting, in order. + pub formatting: Vec, +} + +pub fn diff_ignored_ranges(text: &str, source_type: PySourceType) -> DiffIgnoredRanges { + let parsed = parse_unchecked_source(text, source_type); + + let mut imports = Imports(Vec::new()); + imports.visit_body(parsed.suite()); + + let mut formatting: Vec = Vec::new(); + let mut push = |range: TextRange| { + if range.is_empty() { + return; + } + match formatting.last_mut() { + Some(last) if last.end() == range.start() => *last = last.cover(range), + _ => formatting.push(range), + } + }; + let mut brackets = 0u32; + let mut strings = 0u32; + let mut previous: Option<(TokenKind, TextSize)> = None; + for token in parsed.tokens() { + let kind = token.kind(); + let gap_start = previous.map_or(TextSize::default(), |(_, end)| end); + // the whitespace before a token starts its line's indentation where the line before it + // ended a logical line, or the file starts + let starts_logical_line = match previous { + None => true, + Some((TokenKind::Newline | TokenKind::Indent | TokenKind::Dedent, _)) => true, + Some((TokenKind::NonLogicalNewline, _)) => brackets == 0, + _ => false, + }; + if strings == 0 && !starts_logical_line { + push(TextRange::new(gap_start, token.start())); + } + match kind { + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => brackets += 1, + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => { + brackets = brackets.saturating_sub(1); + } + TokenKind::FStringStart | TokenKind::TStringStart => strings += 1, + TokenKind::FStringEnd | TokenKind::TStringEnd => strings = strings.saturating_sub(1), + TokenKind::NonLogicalNewline if strings == 0 => { + // a line break inside brackets, or a line with nothing on it + let blank = matches!( + previous, + None | Some((TokenKind::Newline | TokenKind::NonLogicalNewline, _)) + ); + if brackets > 0 || blank { + push(token.range()); + } + } + _ => {} + } + previous = Some((kind, token.end())); + } + + DiffIgnoredRanges { + imports: imports.0, + formatting, + } +} + +struct Imports(Vec); + +impl<'a> StatementVisitor<'a> for Imports { + fn visit_stmt(&mut self, statement: &'a Stmt) { + if statement.is_import_stmt() || statement.is_import_from_stmt() { + self.0.push(statement.range()); + } + walk_stmt(self, statement); + } +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::PySourceType; + + use super::diff_ignored_ranges; + + /// `text` with what is ignored as formatting written as `~`, and imports as `I`. + fn marked(text: &str) -> String { + let ignored = diff_ignored_ranges(text, PySourceType::BasedPython); + let mut out: Vec = text.chars().collect(); + let index = |offset: usize| text[..offset].chars().count(); + for range in ignored.imports { + let marked = index(range.start().to_usize())..index(range.end().to_usize()); + for character in &mut out[marked] { + *character = 'I'; + } + } + for range in ignored.formatting { + let marked = index(range.start().to_usize())..index(range.end().to_usize()); + for character in &mut out[marked] { + *character = if *character == '\n' { '$' } else { '~' }; + } + } + out.into_iter().collect() + } + + #[test] + fn whitespace_inside_a_line_and_brackets_is_formatting_and_indentation_is_not() { + insta::assert_snapshot!(marked("import os\nfrom a import (\n b,\n)\n\n\ndef f( x ):\n y = [\n 1 , 2,\n ]\n return f\"{ x = }\"\n"), @r#" + IIIIII~II + IIII~I~IIIIII~I~~~~~II~I + $$def~f(~x~): + y~=~[$~~~~~~~~1~,~~2,$~~~~] + return~f"{ x = }" + "#); + } +} diff --git a/crates/ty_ide/src/import_order.rs b/crates/ty_ide/src/import_order.rs index 93720bf87e..736d03a842 100644 --- a/crates/ty_ide/src/import_order.rs +++ b/crates/ty_ide/src/import_order.rs @@ -22,8 +22,8 @@ use ruff_db::source::source_text; use ruff_db::system::{System, SystemPath}; use ruff_linter::package::PackageRoot; use ruff_linter::packaging::detect_package_root; -use ruff_linter::rules::isort::placement::{ImportOrder, ImportPosition}; -use ruff_python_ast::PythonVersion; +use ruff_linter::rules::isort::placement::{ImportOrder, ImportPosition, sort_import_statements}; +use ruff_python_ast::{PySourceType, PythonVersion}; use ruff_python_importer::{ImportOrdering, SortedImport}; use ruff_workspace::configuration::Configuration; use ruff_workspace::pyproject::{ @@ -74,13 +74,37 @@ impl IsortConfig { /// The configuration that applies to the file at `path`. `project_root` stands in for the /// directory buff resolves `src` against when no configuration file is found. pub(crate) fn for_file(system: &dyn System, path: &Path, project_root: &Path) -> Self { - let directory = path.parent().unwrap_or(path); + Self::new( + system, + path.parent().unwrap_or(path), + Some(path), + project_root, + ) + } + + /// the configuration that applies to a file in `directory`, without a per-file target + /// version, which only a file's path can match + pub(crate) fn for_directory( + system: &dyn System, + directory: &Path, + project_root: &Path, + ) -> Self { + Self::new(system, directory, None, project_root) + } + + fn new( + system: &dyn System, + directory: &Path, + file: Option<&Path>, + project_root: &Path, + ) -> Self { let settings = settings_for(system, directory, project_root); let package_root = detect_package_root(directory, &settings.linter.namespace_packages) .map(Path::to_path_buf); - let target_version = settings - .linter - .resolve_target_version(path) + let target_version = file + .map_or(settings.linter.unresolved_target_version, |file| { + settings.linter.resolve_target_version(file) + }) .linter_version(); Self { settings, @@ -89,6 +113,18 @@ impl IsortConfig { } } + /// `source`, a run of import statements, as buff's `I001` fix sorts and writes them, or + /// `None` for source with anything else in it, a comment among them. + pub(crate) fn sort(&self, source: &str, source_type: PySourceType) -> Option { + sort_import_statements( + source, + &self.settings.linter, + self.package_root.as_deref().map(PackageRoot::root), + source_type, + self.target_version, + ) + } + pub(crate) fn order(&self) -> ImportOrder<'_> { ImportOrder::new( &self.settings.linter.isort, diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index ef9701ac6d..9e18a0cdc5 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -14,6 +14,7 @@ mod complete_statement; mod completion; mod data_flow; mod declaration; +mod diff_ignored; mod django_template; mod doc_highlights; mod docstring; @@ -38,6 +39,7 @@ mod inherited_members; mod injection; mod inlay_hints; mod markup; +mod merge; mod method_kind; mod module_rename; mod on_type_formatting; @@ -77,6 +79,7 @@ pub use completion::{ }; pub use data_flow::{Finding, FindingKind, data_flow_at}; pub use declaration::{ClassName, Declaration, DeclarationAt, DeclaredKind, declaration_at}; +pub use diff_ignored::{DiffIgnoredRanges, diff_ignored_ranges}; pub use django_template::{ DisplayTemplateHover, DjangoChecker, DjangoCodeLens, DjangoLensAction, DjangoLensTarget, PreparedTemplateRename, TemplateCompletion, TemplateEdit, TemplateHover, TemplateInlayHint, @@ -105,6 +108,7 @@ pub use inlay_hints::{ InlayHintKind, InlayHintLabel, InlayHintSettings, InlayHintTextEdit, inlay_hints, }; pub use markup::MarkupKind; +pub use merge::{ConflictLines, MergeTexts, resolve_file_merge_conflicts, resolve_merge_conflicts}; pub use method_kind::MethodKind; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, diff --git a/crates/ty_ide/src/merge.rs b/crates/ty_ide/src/merge.rs new file mode 100644 index 0000000000..0e2ffa19c6 --- /dev/null +++ b/crates/ty_ide/src/merge.rs @@ -0,0 +1,2368 @@ +//! resolving a merge's conflicts by what the code means: what an editor's *resolve simple +//! conflicts* asks a language for +//! +//! a version control system merges line by line, and reports a conflict wherever both sides +//! changed the same lines, whether or not the changes have anything to do with each other: +//! +//! ```py +//! import os +//! <<<<<<< ours +//! import json +//! ======= +//! import sys +//! >>>>>>> theirs +//! ``` +//! +//! each conflict is read here as the statements, or the elements of a display, that the lines +//! of each of the three versions hold, located in that version's whole syntax tree, and is +//! resolved only where the lines are in the same suite or display in all three, and the two +//! sides' changes provably touch nothing in common: +//! +//! - imports that each side added or removed, which are united, and written as buff's isort +//! configuration sorts them; +//! - functions and classes each side added, methods among them, under names neither side binds +//! otherwise, which are kept at the place each was added; +//! - elements each side added to a list, set, tuple or dict display, or to a decorator list, +//! which are kept at the place each was added, a dict's only under keys the other side does not +//! add; +//! - two sides whose statements or elements are the same syntax tree, with the same comments, +//! which differ only in how they are formatted, and which resolve to ours +//! +//! what both sides added first or last in one place is kept once. where both sides added +//! something else in one place, their order is one neither side chose, and only definitions +//! whose running binds their name and does nothing else are kept, ours first: a display's +//! elements are evaluated and kept in order, and decorators apply in order +//! +//! anything else is not resolved: a conflict that is not whole statements or whole elements, a +//! statement changed rather than added, and two sides binding one name differently + +use std::ops::Range; + +use ruff_db::system::SystemPath; +use ruff_python_ast::comparable::{ComparableDecorator, ComparableExpr, ComparableStmt}; +use ruff_python_ast::token::{TokenKind, parenthesized_range}; +use ruff_python_ast::visitor::{self, Visitor}; +use ruff_python_ast::{self as ast, AnyNodeRef, ExprContext, ModModule, PySourceType, Stmt}; +use ruff_python_parser::{Parsed, parse_unchecked_source}; +use ruff_python_trivia::leading_indentation; +use ruff_source_file::LineIndex; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashSet; + +use crate::Db; +use crate::import_order::IsortConfig; +use crate::refactor::evaluation::is_pure; + +/// the lines a conflict spans in each version, 0-based and end-exclusive +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConflictLines { + pub base: Range, + pub ours: Range, + pub theirs: Range, +} + +/// the three versions a merge combines, each a whole file +#[derive(Debug, Clone, Copy)] +pub struct MergeTexts<'a> { + pub base: &'a str, + pub ours: &'a str, + pub theirs: &'a str, +} + +/// the text that resolves each of `conflicts`, in order, or `None` for one that is not +/// resolved, lines that end before they start or lie outside their version among them. the text +/// replaces the conflict's lines, and ends with a line ending wherever they did +/// +/// `sort_imports` writes a run of import statements as the project's isort configuration sorts +/// them, and answers `None` for anything else +pub fn resolve_merge_conflicts( + texts: MergeTexts<'_>, + conflicts: &[ConflictLines], + source_type: PySourceType, + sort_imports: &dyn Fn(&str) -> Option, +) -> Vec> { + let base = Version::new(texts.base, source_type); + let ours = Version::new(texts.ours, source_type); + let theirs = Version::new(texts.theirs, source_type); + conflicts + .iter() + .map(|conflict| { + let (Some(base), Some(ours), Some(theirs)) = ( + base.as_ref().and_then(|base| base.region(&conflict.base)), + ours.as_ref().and_then(|ours| ours.region(&conflict.ours)), + theirs + .as_ref() + .and_then(|theirs| theirs.region(&conflict.theirs)), + ) else { + return None; + }; + resolve(&base, &ours, &theirs, sort_imports) + }) + .collect() +} + +/// [`resolve_merge_conflicts`] for the file at `path` in `db`'s project, whose extension says +/// how it is parsed, and whose imports are sorted with the isort configuration buff reads for it: +/// a project file's, or with no `path` the project root directory's +pub fn resolve_file_merge_conflicts( + db: &dyn Db, + path: Option<&SystemPath>, + texts: MergeTexts<'_>, + conflicts: &[ConflictLines], +) -> Vec> { + let root = db.project().root(db); + let source_type = path + .and_then(|path| path.extension()) + .and_then(PySourceType::try_from_extension) + .unwrap_or_default(); + let isort = match path { + Some(path) => IsortConfig::for_file(db.system(), path.as_std_path(), root.as_std_path()), + None => IsortConfig::for_directory(db.system(), root.as_std_path(), root.as_std_path()), + }; + resolve_merge_conflicts(texts, conflicts, source_type, &|source| { + isort.sort(source, source_type) + }) +} + +/// one version of the file, parsed +struct Version<'a> { + text: &'a str, + parsed: Parsed, + lines: LineIndex, +} + +impl<'a> Version<'a> { + /// `None` for a version that does not parse: what its lines hold cannot be told + fn new(text: &'a str, source_type: PySourceType) -> Option { + let parsed = parse_unchecked_source(text, source_type); + if parsed.has_syntax_errors() { + return None; + } + Some(Self { + text, + parsed, + lines: LineIndex::from_source_text(text), + }) + } + + /// where line `line` starts, or the end of the text for the line after the last + fn line_start(&self, line: usize) -> Option { + match self.lines.line_starts().get(line) { + Some(start) => Some(*start), + None if line == self.lines.line_starts().len() => Some(TextSize::of(self.text)), + None => None, + } + } + + /// where the line after the one `offset` is on starts: the end of `offset`'s line, its line + /// ending included + fn next_line_start(&self, offset: TextSize) -> TextSize { + let line = self.lines.line_index(offset).to_zero_indexed(); + self.line_start(line + 1).unwrap_or(TextSize::of(self.text)) + } + + /// whether `offset` is not inside a token, as the start of a line in a multi-line string is + fn between_tokens(&self, offset: TextSize) -> bool { + let tokens = self.parsed.tokens(); + let after = tokens.partition_point(|token| token.end() <= offset); + tokens + .get(after) + .is_none_or(|token| offset <= token.start()) + } + + fn module(&self) -> &[Stmt] { + self.parsed.suite() + } + + fn region(&self, lines: &Range) -> Option> { + if lines.start > lines.end { + return None; + } + let range = TextRange::new(self.line_start(lines.start)?, self.line_start(lines.end)?); + if !self.between_tokens(range.start()) || !self.between_tokens(range.end()) { + return None; + } + let module = Location { + path: Vec::new(), + scope: Scope::Module(self.module()), + }; + let (held, locations) = in_suite(self.module(), range, module)?; + let comments = self + .parsed + .tokens() + .in_range(range) + .iter() + .filter(|token| token.kind() == TokenKind::Comment) + .map(|token| self.text[token.range()].trim().to_string()) + .collect(); + Some(Region { + version: self, + range, + held, + comments, + locations, + }) + } +} + +/// what the lines of a conflict hold in one version +struct Region<'a> { + version: &'a Version<'a>, + range: TextRange, + held: Held<'a>, + comments: Vec, + /// where the lines are. lines holding no statement, after the last statement of a function + /// or class, are at the end of its body as much as after it, and have a location for each + /// body they end + locations: Vec>, +} + +enum Held<'a> { + /// whole statements of a suite + Statements { statements: Vec<&'a Stmt> }, + /// whole elements of a display or a decorator list + Elements { + container: Container<'a>, + elements: Vec>, + }, +} + +/// the suite or display a conflict's lines are in: the path to it from the module, and the +/// scope its names are bound in +#[derive(Clone)] +struct Location<'a> { + path: Vec>, + scope: Scope<'a>, +} + +impl<'a> Location<'a> { + fn then(&self, steps: impl IntoIterator>) -> Self { + let mut path = self.path.clone(); + path.extend(steps); + Self { + path, + scope: self.scope, + } + } + + /// whether two versions' locations are the same place: the same steps, from the module down + fn same_place(&self, other: &Self) -> bool { + self.path.len() == other.path.len() + && self.path.iter().zip(&other.path).all(|(a, b)| a.same(*b)) + } +} + +/// one step from a suite or a display towards the lines of a conflict +#[derive(Clone, Copy)] +enum Step<'a> { + /// into a statement of the suite, told apart from the others by what it binds or assigns + /// and by how many statements before it in the suite do the same + Statement { statement: &'a Stmt, ordinal: usize }, + /// into the body of the function or class entered + Body, + /// into the decorators of the function or class entered + Decorators, + /// into the display that is the value of the statement entered + Value, + /// into the display that is, or is the value of, the `index`th element of the display + /// entered + Element(usize), +} + +impl Step<'_> { + fn same(self, other: Self) -> bool { + match (self, other) { + ( + Step::Statement { + statement: a, + ordinal: a_ordinal, + }, + Step::Statement { + statement: b, + ordinal: b_ordinal, + }, + ) => { + a_ordinal == b_ordinal + && statement_key(a).is_some_and(|a| statement_key(b) == Some(a)) + } + (Step::Body, Step::Body) + | (Step::Decorators, Step::Decorators) + | (Step::Value, Step::Value) => true, + (Step::Element(a), Step::Element(b)) => a == b, + _ => false, + } + } +} + +/// what tells a statement holding a conflict apart from the others of its suite +#[derive(PartialEq)] +enum StatementKey<'a> { + Function(&'a str), + Class(&'a str), + Assign(Vec>), + AnnAssign(ComparableExpr<'a>), + AugAssign(ComparableExpr<'a>, ast::Operator), + Expr, + Return, +} + +/// `None` for a statement whose insides a conflict is not looked for in +fn statement_key(statement: &Stmt) -> Option> { + Some(match statement { + Stmt::FunctionDef(function) => StatementKey::Function(function.name.as_str()), + Stmt::ClassDef(class) => StatementKey::Class(class.name.as_str()), + Stmt::Assign(assign) => { + StatementKey::Assign(assign.targets.iter().map(ComparableExpr::from).collect()) + } + Stmt::AnnAssign(assign) => StatementKey::AnnAssign(ComparableExpr::from(&*assign.target)), + Stmt::AugAssign(assign) => { + StatementKey::AugAssign(ComparableExpr::from(&*assign.target), assign.op) + } + Stmt::Expr(_) => StatementKey::Expr, + Stmt::Return(_) => StatementKey::Return, + _ => return None, + }) +} + +fn statement_step(suite: &[Stmt], index: usize) -> Option> { + let statement = &suite[index]; + let key = statement_key(statement)?; + let ordinal = suite[..index] + .iter() + .filter(|earlier| statement_key(earlier).as_ref() == Some(&key)) + .count(); + Some(Step::Statement { statement, ordinal }) +} + +/// a scope the names of a suite are bound in +#[derive(Clone, Copy)] +enum Scope<'a> { + Module(&'a [Stmt]), + Class(&'a ast::StmtClassDef), + Function(&'a ast::StmtFunctionDef), +} + +impl<'a> Scope<'a> { + fn statements(self) -> &'a [Stmt] { + match self { + Scope::Module(suite) => suite, + Scope::Class(class) => &class.body, + Scope::Function(function) => &function.body, + } + } +} + +#[derive(Clone, Copy)] +enum Container<'a> { + List(&'a ast::ExprList), + Set(&'a ast::ExprSet), + Tuple(&'a ast::ExprTuple), + Dict(&'a ast::ExprDict), + Decorators, +} + +impl<'a> Container<'a> { + fn same_kind(self, other: Self) -> bool { + matches!( + (self, other), + (Container::List(_), Container::List(_)) + | (Container::Set(_), Container::Set(_)) + | (Container::Tuple(_), Container::Tuple(_)) + | (Container::Dict(_), Container::Dict(_)) + | (Container::Decorators, Container::Decorators) + ) + } + + /// the display, which a decorator list is not + fn display(self) -> Option> { + match self { + Container::List(list) => Some(list.into()), + Container::Set(set) => Some(set.into()), + Container::Tuple(tuple) => Some(tuple.into()), + Container::Dict(dict) => Some(dict.into()), + Container::Decorators => None, + } + } +} + +#[derive(Clone, Copy)] +enum Element<'a> { + Expr(&'a ast::Expr), + DictItem(&'a ast::DictItem), + Decorator(&'a ast::Decorator), +} + +impl<'a> Element<'a> { + fn range(self) -> TextRange { + match self { + Element::Expr(expr) => expr.range(), + Element::DictItem(item) => match &item.key { + Some(key) => TextRange::new(key.start(), item.value.end()), + None => item.value.range(), + }, + Element::Decorator(decorator) => decorator.range(), + } + } + + fn same(self, other: Self) -> bool { + match (self, other) { + (Element::Expr(a), Element::Expr(b)) => { + ComparableExpr::from(a) == ComparableExpr::from(b) + } + (Element::DictItem(a), Element::DictItem(b)) => { + a.key.as_ref().map(ComparableExpr::from) == b.key.as_ref().map(ComparableExpr::from) + && ComparableExpr::from(&a.value) == ComparableExpr::from(&b.value) + } + (Element::Decorator(a), Element::Decorator(b)) => { + ComparableDecorator::from(a) == ComparableDecorator::from(b) + } + _ => false, + } + } + + /// the key a dict item binds, which two sides must not both add + fn key(self) -> Option> { + match self { + Element::DictItem(item) => item.key.as_ref().map(ComparableExpr::from), + _ => None, + } + } + + /// what evaluating the element binds where the display is + fn names(self) -> Names<'a> { + let mut names = Names::default(); + match self { + Element::Expr(expr) => names.visit_expr(expr), + Element::DictItem(item) => { + if let Some(key) = &item.key { + names.visit_expr(key); + } + names.visit_expr(&item.value); + } + Element::Decorator(decorator) => names.visit_decorator(decorator), + } + names + } + + /// whether a comma follows the element, and any parentheses around it, in `version` + fn has_trailing_comma(self, version: &Version, container: Container) -> bool { + let (expr, parent) = match (self, container.display()) { + (Element::Expr(expr), Some(display)) => (expr, display), + (Element::DictItem(item), Some(display)) => (&item.value, display), + _ => return false, + }; + let tokens = version.parsed.tokens(); + let end = + parenthesized_range(expr.into(), parent, tokens).map_or(expr.end(), TextRange::end); + tokens + .after(end) + .iter() + .find(|token| !token.kind().is_trivia()) + .is_some_and(|token| token.kind() == TokenKind::Comma) + } +} + +/// whether a statement or element shares text with a range of whole lines, or, for an empty range, +/// holds the place between two lines it stands for +fn overlaps(statement: TextRange, range: TextRange) -> bool { + statement.start() < range.end() && range.start() < statement.end() + || (range.is_empty() + && statement.start() < range.start() + && range.start() < statement.end()) +} + +/// what `range` holds in `suite`, which is at `at`: some of its statements, all of each, or +/// something inside the one statement it is in. with where that is +fn in_suite<'a>( + suite: &'a [Stmt], + range: TextRange, + at: Location<'a>, +) -> Option<(Held<'a>, Vec>)> { + let overlapping: Vec = (0..suite.len()) + .filter(|index| overlaps(suite[*index].range(), range)) + .collect(); + if overlapping + .iter() + .all(|index| range.contains_range(suite[*index].range())) + { + let mut locations = Vec::new(); + if overlapping.is_empty() { + ended_bodies(suite, range, &at, &mut locations); + } + locations.insert(0, at); + let statements = overlapping.iter().map(|index| &suite[*index]).collect(); + return Some((Held::Statements { statements }, locations)); + } + match overlapping.as_slice() { + [index] => in_statement(suite, *index, range, &at), + _ => None, + } +} + +/// the bodies that `range`, holding no statement of `suite`, comes right after the end of: the +/// body of the function or class it follows, then of the last statement of that body, and so on +fn ended_bodies<'a>( + suite: &'a [Stmt], + range: TextRange, + at: &Location<'a>, + locations: &mut Vec>, +) { + let Some(index) = suite + .iter() + .rposition(|statement| statement.end() <= range.start()) + else { + return; + }; + let (scope, body) = match &suite[index] { + Stmt::FunctionDef(function) => (Scope::Function(function), &function.body), + Stmt::ClassDef(class) => (Scope::Class(class), &class.body), + _ => return, + }; + let Some(step) = statement_step(suite, index) else { + return; + }; + let mut location = at.then([step, Step::Body]); + location.scope = scope; + ended_bodies(body, range, &location, locations); + locations.insert(0, location); +} + +fn in_statement<'a>( + suite: &'a [Stmt], + index: usize, + range: TextRange, + at: &Location<'a>, +) -> Option<(Held<'a>, Vec>)> { + let step = statement_step(suite, index)?; + let (decorators, name, header_end, body, scope) = match &suite[index] { + Stmt::FunctionDef(function) => ( + &function.decorator_list, + &function.name, + function + .returns + .as_deref() + .map_or(function.parameters.end(), Ranged::end), + &function.body, + Scope::Function(function), + ), + Stmt::ClassDef(class) => ( + &class.decorator_list, + &class.name, + class + .arguments + .as_deref() + .map_or(class.name.end(), Ranged::end), + &class.body, + Scope::Class(class), + ), + Stmt::Assign(assign) => return in_expr(&assign.value, range, at.then([step, Step::Value])), + Stmt::AnnAssign(assign) => { + return in_expr( + assign.value.as_deref()?, + range, + at.then([step, Step::Value]), + ); + } + Stmt::AugAssign(assign) => { + return in_expr(&assign.value, range, at.then([step, Step::Value])); + } + Stmt::Expr(expr) => return in_expr(&expr.value, range, at.then([step, Step::Value])), + Stmt::Return(ret) => { + return in_expr(ret.value.as_deref()?, range, at.then([step, Step::Value])); + } + _ => return None, + }; + if !decorators.is_empty() && range.end() <= name.start() { + // the lines above the header hold decorators, all of each + let overlapping: Vec = decorators + .iter() + .filter(|decorator| overlaps(decorator.range(), range)) + .map(Element::Decorator) + .collect(); + if !overlapping + .iter() + .all(|decorator| range.contains_range(decorator.range())) + { + return None; + } + let held = Held::Elements { + container: Container::Decorators, + elements: overlapping, + }; + return Some((held, vec![at.then([step, Step::Decorators])])); + } + // the lines after the header hold statements of the body + if range.start() > header_end { + let mut location = at.then([step, Step::Body]); + location.scope = scope; + return in_suite(body, range, location); + } + None +} + +fn in_expr<'a>( + expr: &'a ast::Expr, + range: TextRange, + at: Location<'a>, +) -> Option<(Held<'a>, Vec>)> { + // strictly between the brackets + if !(expr.start() < range.start() && range.end() < expr.end()) { + return None; + } + let (container, elements): (Container, Vec) = match expr { + ast::Expr::List(list) => ( + Container::List(list), + list.elts.iter().map(Element::Expr).collect(), + ), + ast::Expr::Set(set) => ( + Container::Set(set), + set.elts.iter().map(Element::Expr).collect(), + ), + ast::Expr::Tuple(tuple) if tuple.parenthesized => ( + Container::Tuple(tuple), + tuple.elts.iter().map(Element::Expr).collect(), + ), + ast::Expr::Dict(dict) => ( + Container::Dict(dict), + dict.items.iter().map(Element::DictItem).collect(), + ), + _ => return None, + }; + let overlapping: Vec<(usize, Element)> = elements + .into_iter() + .enumerate() + .filter(|(_, element)| overlaps(element.range(), range)) + .collect(); + if overlapping + .iter() + .all(|(_, element)| range.contains_range(element.range())) + { + let held = Held::Elements { + container, + elements: overlapping + .into_iter() + .map(|(_, element)| element) + .collect(), + }; + return Some((held, vec![at])); + } + match overlapping.as_slice() { + [(index, Element::Expr(inner))] => in_expr(inner, range, at.then([Step::Element(*index)])), + [(index, Element::DictItem(item))] => { + in_expr(&item.value, range, at.then([Step::Element(*index)])) + } + _ => None, + } +} + +/// the location the three regions share, in each version, or `None` where the lines of the +/// conflict are in different suites or displays in different versions +fn shared_location<'r, 'a>( + base: &'r Region<'a>, + ours: &'r Region<'a>, + theirs: &'r Region<'a>, +) -> Option<[&'r Location<'a>; 3]> { + ours.locations.iter().find_map(|ours_location| { + let at = |region: &'r Region<'a>| { + region + .locations + .iter() + .find(|location| location.same_place(ours_location)) + }; + Some([at(base)?, ours_location, at(theirs)?]) + }) +} + +fn resolve( + base: &Region, + ours: &Region, + theirs: &Region, + sort_imports: &dyn Fn(&str) -> Option, +) -> Option { + let [_, ours_location, theirs_location] = shared_location(base, ours, theirs)?; + let sides = Sides { + ours: Side { + region: ours, + scope: ours_location.scope, + }, + theirs: Side { + region: theirs, + scope: theirs_location.scope, + }, + }; + match (&base.held, &ours.held, &theirs.held) { + ( + Held::Statements { statements: b }, + Held::Statements { statements: o }, + Held::Statements { statements: t }, + ) => { + if same_statements(o, t) && ours.comments == theirs.comments { + return Some(ours.text().to_string()); + } + let all_imports = |statements: &[&Stmt]| { + statements + .iter() + .all(|statement| statement.is_import_stmt() || statement.is_import_from_stmt()) + }; + if all_imports(b) && all_imports(o) && all_imports(t) { + return merge_imports(b, o, t, &sides, sort_imports); + } + merge_definitions(b, o, t, &sides) + } + ( + Held::Elements { + container: base_container, + elements: b, + }, + Held::Elements { + container: ours_container, + elements: o, + }, + Held::Elements { + container: theirs_container, + elements: t, + }, + ) if base_container.same_kind(*ours_container) + && ours_container.same_kind(*theirs_container) => + { + if o.len() == t.len() + && o.iter().zip(t).all(|(a, b)| a.same(*b)) + && ours.comments == theirs.comments + { + return Some(ours.text().to_string()); + } + merge_elements(b, o, t, &sides, *ours_container, *theirs_container) + } + _ => None, + } +} + +/// one side's region, and the scope the names it binds are bound in +struct Side<'r, 'a> { + region: &'r Region<'a>, + scope: Scope<'a>, +} + +struct Sides<'r, 'a> { + ours: Side<'r, 'a>, + theirs: Side<'r, 'a>, +} + +impl<'a> Side<'_, 'a> { + /// what the side binds in its scope, leaving out the statements in `excluded` + fn bound(&self, excluded: &[&Stmt]) -> Bound<'a> { + let module = self.region.version.module(); + let statements = self.scope.statements().iter().filter(|statement| { + !excluded + .iter() + .any(|excluded| std::ptr::eq(*excluded, *statement)) + }); + let names = Names::of_statements(statements); + let mut bound: FxHashSet<&str> = names.binding_in(self.scope).collect(); + let mut attributes = FxHashSet::default(); + match self.scope { + Scope::Function(function) => { + bound.extend( + function + .parameters + .iter() + .map(|parameter| parameter.name().as_str()), + ); + } + Scope::Class(_) => { + // an instance's attribute shadows the class's, and can be set from anywhere + let mut stores = AttributeStores::default(); + stores.visit_body(module); + attributes = stores.attributes; + } + Scope::Module(_) => {} + } + Bound { + names: bound, + attributes, + star: names.star, + } + } +} + +impl Region<'_> { + fn text(&self) -> &str { + &self.version.text[self.range] + } +} + +fn same_statements(a: &[&Stmt], b: &[&Stmt]) -> bool { + a.len() == b.len() + && a.iter() + .zip(b) + .all(|(a, b)| ComparableStmt::from(*a) == ComparableStmt::from(*b)) +} + +// ── names ─────────────────────────────────────────────────────────────────────────────────────── + +/// the names some statements or expressions bind in the scope they are in. a function, class, +/// lambda or comprehension nested in them is a scope of its own, whose names are not among them, +/// but a name a nested function declares `global` or `nonlocal` binds in an enclosing scope +#[derive(Default)] +struct Names<'a> { + bound: Vec<&'a str>, + /// what nested functions declare `global`, which binds in the module + global: Vec<&'a str>, + /// what nested functions declare `nonlocal`, which binds in an enclosing function + nonlocal: Vec<&'a str>, + /// whether a star import binds names the source does not tell + star: bool, +} + +impl<'a> Names<'a> { + fn of_statements(statements: impl IntoIterator) -> Self { + let mut names = Self::default(); + for statement in statements { + names.visit_stmt(statement); + } + names + } + + /// the names bound in a scope like `scope` + fn binding_in(&self, scope: Scope) -> impl Iterator { + let (global, nonlocal) = match scope { + Scope::Module(_) => (self.global.as_slice(), [].as_slice()), + Scope::Class(_) => ([].as_slice(), [].as_slice()), + Scope::Function(_) => ([].as_slice(), self.nonlocal.as_slice()), + }; + self.bound.iter().chain(global).chain(nonlocal).copied() + } + + /// the `global` and `nonlocal` declarations in a nested scope's body, at any depth + fn declarations_in(&mut self, body: &'a [Stmt]) { + let mut declarations = Declarations::default(); + declarations.visit_body(body); + self.global.extend(declarations.global); + self.nonlocal.extend(declarations.nonlocal); + } +} + +impl<'a> Visitor<'a> for Names<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + self.bound.push(function.name.as_str()); + for decorator in &function.decorator_list { + self.visit_decorator(decorator); + } + self.visit_parameters(&function.parameters); + if let Some(returns) = &function.returns { + self.visit_annotation(returns); + } + self.declarations_in(&function.body); + } + Stmt::ClassDef(class) => { + self.bound.push(class.name.as_str()); + for decorator in &class.decorator_list { + self.visit_decorator(decorator); + } + if let Some(arguments) = &class.arguments { + self.visit_arguments(arguments); + } + self.declarations_in(&class.body); + } + Stmt::Import(import) => { + for alias in &import.names { + let bound = alias.asname.as_ref().unwrap_or(&alias.name).as_str(); + // `import a.b` binds `a` + self.bound.push(bound.split('.').next().unwrap_or(bound)); + } + } + Stmt::ImportFrom(import) => { + for alias in &import.names { + if alias.asname.is_none() && &alias.name == "*" { + self.star = true; + } else { + self.bound + .push(alias.asname.as_ref().unwrap_or(&alias.name).as_str()); + } + } + } + Stmt::Global(global) => self + .bound + .extend(global.names.iter().map(ast::Identifier::as_str)), + Stmt::Nonlocal(nonlocal) => { + self.bound + .extend(nonlocal.names.iter().map(ast::Identifier::as_str)); + } + _ => visitor::walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &'a ast::Expr) { + match expr { + ast::Expr::Name(name) => { + if matches!(name.ctx, ExprContext::Store | ExprContext::Del) { + self.bound.push(name.id.as_str()); + } + } + // only the defaults are evaluated where the lambda is + ast::Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.visit_parameters(parameters); + } + } + _ => visitor::walk_expr(self, expr), + } + } + + /// a comprehension's targets are its own; an assignment expression in it binds where the + /// comprehension is + fn visit_comprehension(&mut self, comprehension: &'a ast::Comprehension) { + self.visit_expr(&comprehension.iter); + for condition in &comprehension.ifs { + self.visit_expr(condition); + } + } + + fn visit_except_handler(&mut self, except_handler: &'a ast::ExceptHandler) { + let ast::ExceptHandler::ExceptHandler(handler) = except_handler; + if let Some(name) = &handler.name { + self.bound.push(name.as_str()); + } + visitor::walk_except_handler(self, except_handler); + } + + fn visit_pattern(&mut self, pattern: &'a ast::Pattern) { + let name = match pattern { + ast::Pattern::MatchAs(pattern) => pattern.name.as_ref(), + ast::Pattern::MatchStar(pattern) => pattern.name.as_ref(), + ast::Pattern::MatchMapping(pattern) => pattern.rest.as_ref(), + _ => None, + }; + if let Some(name) = name { + self.bound.push(name.as_str()); + } + visitor::walk_pattern(self, pattern); + } +} + +/// the names declared `global` or `nonlocal` anywhere in some statements +#[derive(Default)] +struct Declarations<'a> { + global: Vec<&'a str>, + nonlocal: Vec<&'a str>, +} + +impl<'a> Visitor<'a> for Declarations<'a> { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::Global(global) => self + .global + .extend(global.names.iter().map(ast::Identifier::as_str)), + Stmt::Nonlocal(nonlocal) => { + self.nonlocal + .extend(nonlocal.names.iter().map(ast::Identifier::as_str)); + } + _ => visitor::walk_stmt(self, stmt), + } + } +} + +/// the attributes some statements assign or delete, of whatever object +#[derive(Default)] +struct AttributeStores<'a> { + attributes: FxHashSet<&'a str>, +} + +impl<'a> Visitor<'a> for AttributeStores<'a> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if let ast::Expr::Attribute(attribute) = expr + && matches!(attribute.ctx, ExprContext::Store | ExprContext::Del) + { + self.attributes.insert(attribute.attr.as_str()); + } + visitor::walk_expr(self, expr); + } +} + +/// what one side binds in a scope +struct Bound<'a> { + names: FxHashSet<&'a str>, + /// for a class's scope, the attributes the side assigns anywhere, which an instance's + /// attribute of that name would shadow the class's with + attributes: FxHashSet<&'a str>, + star: bool, +} + +impl Bound<'_> { + fn binds(&self, name: &str) -> bool { + self.star || self.names.contains(name) || self.attributes.contains(name) + } +} + +// ── imports ───────────────────────────────────────────────────────────────────────────────────── + +/// one name an import statement binds +#[derive(Debug, Clone, PartialEq, Eq)] +struct Imported { + /// the module a `from` import imports from, with its leading dots, or `None` for `import` + from: Option, + name: String, + asname: Option, +} + +impl Imported { + /// the name the import binds in the module + fn binds(&self) -> &str { + match (&self.asname, &self.from) { + (Some(asname), _) => asname, + (None, Some(_)) => &self.name, + // `import a.b` binds `a` + (None, None) => self.name.split('.').next().unwrap_or(&self.name), + } + } + + /// what the bound name is bound to: two imports binding one name to one thing do not + /// conflict, as `import a.b` and `import a.c` both bind the package `a` + fn bound_to(&self) -> (Option<&str>, &str) { + match (&self.from, &self.asname) { + (Some(from), _) => (Some(from), &self.name), + (None, Some(_)) => (None, &self.name), + (None, None) => (None, self.binds()), + } + } + + fn statement(&self) -> String { + let alias = self + .asname + .as_ref() + .map(|asname| format!(" as {asname}")) + .unwrap_or_default(); + match &self.from { + Some(from) => format!("from {from} import {}{alias}", self.name), + None => format!("import {}{alias}", self.name), + } + } +} + +fn imported(statements: &[&Stmt]) -> Vec { + let mut found = Vec::new(); + for statement in statements { + match statement { + Stmt::Import(import) => { + for alias in &import.names { + found.push(Imported { + from: None, + name: alias.name.to_string(), + asname: alias.asname.as_ref().map(ToString::to_string), + }); + } + } + Stmt::ImportFrom(import) => { + let module = format!( + "{}{}", + ".".repeat(import.level as usize), + import.module.as_deref().unwrap_or_default() + ); + for alias in &import.names { + found.push(Imported { + from: Some(module.clone()), + name: alias.name.to_string(), + asname: alias.asname.as_ref().map(ToString::to_string), + }); + } + } + _ => {} + } + } + found +} + +/// every import either side added, without those either side removed, sorted. refused where both +/// sides changed what one name is bound to, differently, where the result binds one name to two +/// things, where a side adds an import of a name the other binds elsewhere, and where a star +/// import binds names nothing here can tell +fn merge_imports( + base: &[&Stmt], + ours: &[&Stmt], + theirs: &[&Stmt], + sides: &Sides, + sort_imports: &dyn Fn(&str) -> Option, +) -> Option { + let (ours_region, theirs_region) = (sides.ours.region, sides.theirs.region); + // a comment beside an import goes where isort puts the import, which this does not decide + if !ours_region.comments.is_empty() || !theirs_region.comments.is_empty() { + return None; + } + let (b, o, t) = (imported(base), imported(ours), imported(theirs)); + if o.iter().chain(&t).any(|import| import.name == "*") { + return None; + } + let changed = |side: &[Imported]| -> Vec { + side.iter() + .filter(|import| !b.contains(import)) + .chain(b.iter().filter(|import| !side.contains(import))) + .map(|import| import.binds().to_string()) + .collect() + }; + let (ours_changed, theirs_changed) = (changed(&o), changed(&t)); + for name in ours_changed + .iter() + .filter(|name| theirs_changed.contains(name)) + { + let bound_to = |side: &'_ [Imported]| { + let mut bound_to: Vec<(Option, String)> = side + .iter() + .filter(|import| import.binds() == name) + .map(|import| { + let (from, name) = import.bound_to(); + (from.map(str::to_string), name.to_string()) + }) + .collect(); + bound_to.sort(); + bound_to.dedup(); + bound_to + }; + if bound_to(&o) != bound_to(&t) { + return None; + } + } + // a name one side imports that the other binds outside the conflict is bound twice, unless + // by another import to the same thing + for (side, other, other_statements) in [(&o, &sides.theirs, theirs), (&t, &sides.ours, ours)] { + let imports: Vec<&Stmt> = other + .scope + .statements() + .iter() + .filter(|statement| { + (statement.is_import_stmt() || statement.is_import_from_stmt()) + && !other_statements + .iter() + .any(|held| std::ptr::eq(*held, *statement)) + }) + .collect(); + let outside = imported(&imports); + if outside.iter().any(|import| import.name == "*") { + return None; + } + let excluded: Vec<&Stmt> = other_statements.iter().chain(&imports).copied().collect(); + let bound = other.bound(&excluded); + if side + .iter() + .filter(|import| !b.contains(import)) + .any(|import| { + bound.binds(import.binds()) + || outside.iter().any(|outside| { + outside.binds() == import.binds() && outside.bound_to() != import.bound_to() + }) + }) + { + return None; + } + } + let merged: Vec<&Imported> = o + .iter() + .filter(|import| t.contains(import) || !b.contains(import)) + .chain( + t.iter() + .filter(|import| !o.contains(import) && !b.contains(import)), + ) + .collect(); + for (index, import) in merged.iter().enumerate() { + if merged[..index].iter().any(|earlier| { + earlier.binds() == import.binds() && earlier.bound_to() != import.bound_to() + }) { + return None; + } + } + + let tail = ours_region.tail(); + if merged.is_empty() { + return Some(tail.to_string()); + } + let indentation = + region_indentation(ours_region).or_else(|| region_indentation(theirs_region))?; + let statements: String = merged + .iter() + .map(|import| import.statement() + "\n") + .collect(); + let sorted = sort_imports(&statements)?; + let mut out = String::new(); + for line in sorted.split_inclusive('\n') { + if !line.trim().is_empty() { + out.push_str(indentation); + } + out.push_str(line); + } + out.push_str(tail); + Some(out) +} + +/// the indentation the statements of a region are written at, or `None` for a region with none +fn region_indentation<'a>(region: &Region<'a>) -> Option<&'a str> { + let Held::Statements { statements } = ®ion.held else { + return None; + }; + let first = statements.first()?; + let version = region.version; + let line_start = + version.line_start(version.lines.line_index(first.start()).to_zero_indexed())?; + Some(leading_indentation( + &version.text[TextRange::new(line_start, first.start())], + )) +} + +impl Region<'_> { + /// what follows the last statement or element of the region on lines of their own: the blank + /// lines before what comes after it + fn tail(&self) -> &str { + let last = match &self.held { + Held::Statements { statements } => statements.last().map(Ranged::end), + Held::Elements { elements, .. } => elements.last().map(|last| last.range().end()), + }; + let start = last.map_or(self.range.start(), |end| { + self.version.next_line_start(end).min(self.range.end()) + }); + &self.version.text[TextRange::new(start, self.range.end())] + } + + /// the text each statement or element of the region is written as: from the end of the line + /// the one before it ends on, or the start of the region, to the end of its own last line + fn pieces(&self) -> Vec<&str> { + let ends: Vec = match &self.held { + Held::Statements { statements } => statements.iter().map(Ranged::end).collect(), + Held::Elements { elements, .. } => elements.iter().map(|e| e.range().end()).collect(), + }; + let mut start = self.range.start(); + ends.into_iter() + .map(|end| { + let end = self.version.next_line_start(end).min(self.range.end()); + let piece = &self.version.text[TextRange::new(start, end)]; + start = end; + piece + }) + .collect() + } +} + +// ── added statements and elements ─────────────────────────────────────────────────────────────── + +/// where each statement of `side` is in `base`, which it must hold in order: `None` for one the +/// side added. `None` overall for a side that changed or removed one of `base`'s +fn alignment( + base: &[T], + side: &[T], + same: impl Fn(&T, &T) -> bool, +) -> Option>> { + let mut next = 0; + let aligned: Vec> = side + .iter() + .map(|item| { + if next < base.len() && same(&base[next], item) { + next += 1; + Some(next - 1) + } else { + None + } + }) + .collect(); + (next == base.len()).then_some(aligned) +} + +/// the items a side added in each place: before each of base's, and after the last +fn added_in_places(aligned: &[Option], base_len: usize) -> Vec> { + let mut places = vec![Vec::new(); base_len + 1]; + let mut place = 0; + for (index, position) in aligned.iter().enumerate() { + match position { + Some(position) => place = position + 1, + None => places[place].push(index), + } + } + places +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Origin { + Ours, + Theirs, +} + +/// how two sides' additions to base's items are kept +struct Merged { + /// each item kept, in order, by the side and the index in it of the copy kept + order: Vec<(Origin, usize)>, + /// what each side added that the other did not + ours_only: Vec, + theirs_only: Vec, + /// everything each side added + ours_added: Vec, + theirs_added: Vec, +} + +/// base's items, with what each side added in each place between two of them. where both sides +/// added in one place, what both start and end with there is kept once, ours, and the rest of +/// ours goes before the rest of theirs, which is an order neither side chose: only where every +/// one of them is `reorderable`. `None` for that, and for a side that changed or removed one of +/// base's +fn merge_sequences( + base: &[T], + ours: &[T], + theirs: &[T], + same: impl Fn(&T, &T) -> bool, + reorderable: impl Fn(&T) -> bool, +) -> Option { + let ours_aligned = alignment(base, ours, &same)?; + let theirs_aligned = alignment(base, theirs, &same)?; + let ours_places = added_in_places(&ours_aligned, base.len()); + let theirs_places = added_in_places(&theirs_aligned, base.len()); + let ours_base: Vec = (0..ours.len()) + .filter(|index| ours_aligned[*index].is_some()) + .collect(); + let mut merged = Merged { + order: Vec::new(), + ours_only: Vec::new(), + theirs_only: Vec::new(), + ours_added: ours_places.concat(), + theirs_added: theirs_places.concat(), + }; + for (place, (o, t)) in ours_places.iter().zip(&theirs_places).enumerate() { + let both = |(a, b): (&usize, &usize)| same(&ours[*a], &theirs[*b]); + let first = o.iter().zip(t).take_while(|pair| both(*pair)).count(); + let last = o[first..] + .iter() + .rev() + .zip(t[first..].iter().rev()) + .take_while(|pair| both(*pair)) + .count(); + let ours_only = &o[first..o.len() - last]; + let theirs_only = &t[first..t.len() - last]; + let interleaved = !ours_only.is_empty() && !theirs_only.is_empty(); + let all_reorderable = ours_only.iter().all(|index| reorderable(&ours[*index])) + && theirs_only.iter().all(|index| reorderable(&theirs[*index])); + if interleaved && !all_reorderable { + return None; + } + let ours_items = |items: &[usize]| -> Vec<(Origin, usize)> { + items.iter().map(|index| (Origin::Ours, *index)).collect() + }; + merged.order.extend(ours_items(&o[..o.len() - last])); + merged + .order + .extend(theirs_only.iter().map(|index| (Origin::Theirs, *index))); + merged.order.extend(ours_items(&o[o.len() - last..])); + if let Some(index) = ours_base.get(place) { + merged.order.push((Origin::Ours, *index)); + } + merged.ours_only.extend(ours_only); + merged.theirs_only.extend(theirs_only); + } + Some(merged) +} + +/// whether running a definition binds its name and does nothing else, so that two of them run +/// in either order bind the same things: no decorator, base class or class keyword runs code of +/// the program's, and every default and annotation evaluated with it is a name or a literal +fn definition_is_quiet(statement: &Stmt) -> bool { + match statement { + Stmt::FunctionDef(function) => { + function.decorator_list.is_empty() + && function.parameters.iter().all(|parameter| { + parameter.default().is_none_or(is_pure) + && parameter.annotation().is_none_or(is_pure) + }) + && function.returns.as_deref().is_none_or(is_pure) + } + Stmt::ClassDef(class) => { + class.decorator_list.is_empty() + && class.arguments.as_deref().is_none_or(|arguments| { + arguments.args.is_empty() && arguments.keywords.is_empty() + }) + && class.body.iter().all(|statement| match statement { + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => definition_is_quiet(statement), + Stmt::Pass(_) => true, + Stmt::Expr(expr) => is_pure(&expr.value), + Stmt::Assign(assign) => { + assign.targets.iter().all(ast::Expr::is_name_expr) && is_pure(&assign.value) + } + Stmt::AnnAssign(assign) => { + assign.target.is_name_expr() + && is_pure(&assign.annotation) + && assign.value.as_deref().is_none_or(is_pure) + } + _ => false, + }) + } + _ => false, + } +} + +/// the functions and classes each side added at the conflict, both kept. refused where a side +/// changed or removed a statement, added something else, or added a name the other side binds +fn merge_definitions( + base: &[&Stmt], + ours: &[&Stmt], + theirs: &[&Stmt], + sides: &Sides, +) -> Option { + let same = |a: &&Stmt, b: &&Stmt| ComparableStmt::from(*a) == ComparableStmt::from(*b); + let merged = merge_sequences(base, ours, theirs, same, |statement| { + definition_is_quiet(statement) + })?; + if !merged + .ours_added + .iter() + .map(|index| ours[*index]) + .chain(merged.theirs_added.iter().map(|index| theirs[*index])) + .all(|statement| statement.is_function_def_stmt() || statement.is_class_def_stmt()) + { + return None; + } + for (only, statements, side, other) in [ + (&merged.ours_only, ours, &sides.ours, &sides.theirs), + (&merged.theirs_only, theirs, &sides.theirs, &sides.ours), + ] { + let bound = other.bound(&[]); + for index in only { + let names = Names::of_statements([statements[*index]]); + if names.binding_in(side.scope).any(|name| bound.binds(name)) { + return None; + } + } + } + let (ours_region, theirs_region) = (sides.ours.region, sides.theirs.region); + if merged.theirs_only.is_empty() { + return Some(ours_region.text().to_string()); + } + + let ours_pieces = ours_region.pieces(); + let theirs_pieces = theirs_region.pieces(); + let blank_lines = if matches!(sides.ours.scope, Scope::Module(_)) { + 2 + } else { + 1 + }; + let mut out = String::new(); + let mut previous = None; + for (from, index) in merged.order { + let piece = match from { + Origin::Ours => ours_pieces[index], + Origin::Theirs => theirs_pieces[index], + }; + push_separated( + &mut out, + piece, + blank_lines, + previous.is_some_and(|previous| previous != from), + ); + previous = Some(from); + } + out.push_str(ours_region.tail()); + Some(out) +} + +/// append `piece`, after at least `blank_lines` blank lines when it follows a piece from the +/// other side, which the two sides' own blank lines may not have put there +fn push_separated(out: &mut String, piece: &str, blank_lines: usize, other_side: bool) { + if other_side && !out.is_empty() { + let trailing = out.trim_end_matches(['\n', '\r']).len(); + let had = out[trailing..].matches('\n').count().saturating_sub(1); + let leading = piece + .split_inclusive('\n') + .take_while(|line| line.trim().is_empty()) + .count(); + for _ in had + leading..blank_lines { + out.push('\n'); + } + } + out.push_str(piece); +} + +/// the elements each side added to a display or a decorator list, each kept in the place it was +/// added. refused where a side changed or removed one, where both added different ones in one +/// place, as the order they are evaluated and kept in, or applied in, would be one neither side +/// chose, where two elements share a line, where an element that would no longer be last has no +/// comma after it, or where both sides add one key of a dict +fn merge_elements<'a>( + base: &[Element<'a>], + ours: &[Element<'a>], + theirs: &[Element<'a>], + sides: &Sides<'_, 'a>, + ours_container: Container<'a>, + theirs_container: Container<'a>, +) -> Option { + let same = |a: &Element, b: &Element| a.same(*b); + let merged = merge_sequences(base, ours, theirs, same, |_| false)?; + if let (Container::Dict(ours_dict), Container::Dict(theirs_dict)) = + (ours_container, theirs_container) + { + let keys = |dict: &'a ast::ExprDict| -> Vec> { + dict.items + .iter() + .filter_map(|item| Element::DictItem(item).key()) + .collect() + }; + let (ours_keys, theirs_keys) = (keys(ours_dict), keys(theirs_dict)); + for (only, elements, other_keys) in [ + (&merged.ours_only, ours, &theirs_keys), + (&merged.theirs_only, theirs, &ours_keys), + ] { + if only.iter().any(|index| { + elements[*index] + .key() + .is_some_and(|key| other_keys.contains(&key)) + }) { + return None; + } + } + } + for (only, elements, side, other) in [ + (&merged.ours_only, ours, &sides.ours, &sides.theirs), + (&merged.theirs_only, theirs, &sides.theirs, &sides.ours), + ] { + let bound = other.bound(&[]); + for index in only { + if elements[*index] + .names() + .binding_in(side.scope) + .any(|name| bound.binds(name)) + { + return None; + } + } + } + let (ours_region, theirs_region) = (sides.ours.region, sides.theirs.region); + if merged.theirs_only.is_empty() { + return Some(ours_region.text().to_string()); + } + for (region, elements) in [(ours_region, ours), (theirs_region, theirs)] { + // each element on lines of its own, which is what lets them be moved by the line + for element in elements { + let line = region.version.lines.line_index(element.range().start()); + if elements.iter().any(|other| { + other.range() != element.range() + && region.version.lines.line_index(other.range().end()) == line + }) { + return None; + } + } + } + + let ours_pieces = ours_region.pieces(); + let theirs_pieces = theirs_region.pieces(); + let mut out = String::new(); + for (position, (from, index)) in merged.order.iter().enumerate() { + let (region, element, container, piece) = match from { + Origin::Ours => ( + ours_region, + ours[*index], + ours_container, + ours_pieces[*index], + ), + Origin::Theirs => ( + theirs_region, + theirs[*index], + theirs_container, + theirs_pieces[*index], + ), + }; + // the last may go without its comma only where it is ours' own last, before what + // follows the conflict + let ours_last = *from == Origin::Ours && *index + 1 == ours.len(); + if container.display().is_some() + && !element.has_trailing_comma(region.version, container) + && !(position + 1 == merged.order.len() && ours_last) + { + return None; + } + out.push_str(piece); + } + out.push_str(ours_region.tail()); + Some(out) +} + +#[cfg(test)] +mod tests { + use insta::assert_snapshot; + use ruff_linter::rules::isort::placement::sort_import_statements; + use ruff_linter::settings::LinterSettings; + use ruff_python_ast::{PySourceType, PythonVersion}; + + use std::ops::Range; + + use super::{ConflictLines, MergeTexts, resolve_merge_conflicts}; + + /// The file a merge left with git's diff3 conflict markers in it, each conflict replaced by + /// what resolves it, or left as git wrote it where nothing does. + fn resolved(conflicted: &str) -> String { + let (mut base, mut ours, mut theirs) = (String::new(), String::new(), String::new()); + let mut conflicts = Vec::new(); + let mut lines = conflicted.split_inclusive('\n'); + let mut outside = Vec::new(); + let count = |text: &str| text.matches('\n').count(); + while let Some(line) = lines.next() { + if !line.starts_with("<<<<<<<") { + base.push_str(line); + ours.push_str(line); + theirs.push_str(line); + outside.push(line.to_string()); + continue; + } + let start = (count(&base), count(&ours), count(&theirs)); + let mut target = &mut ours; + let mut raw = line.to_string(); + for line in lines.by_ref() { + raw.push_str(line); + if line.starts_with("|||||||") { + target = &mut base; + } else if line.starts_with("=======") { + target = &mut theirs; + } else if line.starts_with(">>>>>>>") { + break; + } else { + target.push_str(line); + } + } + conflicts.push(( + ConflictLines { + base: start.0..count(&base), + ours: start.1..count(&ours), + theirs: start.2..count(&theirs), + }, + raw, + )); + outside.push(String::new()); + } + let settings = LinterSettings::default(); + let sort = |source: &str| { + sort_import_statements( + source, + &settings, + None, + PySourceType::BasedPython, + PythonVersion::PY312, + ) + }; + let resolutions = resolve_merge_conflicts( + MergeTexts { + base: &base, + ours: &ours, + theirs: &theirs, + }, + &conflicts + .iter() + .map(|(lines, _)| lines.clone()) + .collect::>(), + PySourceType::BasedPython, + &sort, + ); + let mut resolutions = resolutions + .into_iter() + .zip(conflicts.iter().map(|(_, raw)| raw)); + outside + .iter() + .map(|line| { + if !line.is_empty() { + return line.clone(); + } + let (resolution, raw) = resolutions.next().expect("a conflict"); + resolution.unwrap_or_else(|| raw.clone()) + }) + .collect() + } + + /// what `git merge` wrote for a branch adding an import, an `__all__` entry, a method and a + /// new limit, merged into one adding others: the two `__all__` entries, added in one place, + /// would be kept in an order neither side chose + #[test] + fn independent_additions_are_kept_and_a_changed_line_is_not() { + assert_snapshot!(resolved(r#"import os +<<<<<<< HEAD +import json +||||||| dbcf8e7 +======= +import sys +>>>>>>> theirs +from typing import Any + +__all__ = [ + "Shape", + "area", +<<<<<<< HEAD + "scale", +||||||| dbcf8e7 +======= + "volume", +>>>>>>> theirs +] + + +class Shape: + def area(self) -> float: + return 0.0 + +<<<<<<< HEAD + def name(self) -> str: + return 'shape' + +||||||| dbcf8e7 +======= + def perimeter(self) -> float: + return 1.0 + +>>>>>>> theirs + +def area(shape: Shape) -> float: + return shape.area() + + +<<<<<<< HEAD +LIMIT = 30 +||||||| dbcf8e7 +LIMIT = 10 +======= +LIMIT = 20 +>>>>>>> theirs +"#), @r#" + import os + import json + import sys + from typing import Any + + __all__ = [ + "Shape", + "area", + <<<<<<< HEAD + "scale", + ||||||| dbcf8e7 + ======= + "volume", + >>>>>>> theirs + ] + + + class Shape: + def area(self) -> float: + return 0.0 + + def name(self) -> str: + return 'shape' + + def perimeter(self) -> float: + return 1.0 + + + def area(shape: Shape) -> float: + return shape.area() + + + <<<<<<< HEAD + LIMIT = 30 + ||||||| dbcf8e7 + LIMIT = 10 + ======= + LIMIT = 20 + >>>>>>> theirs + "#); + } + + #[test] + fn imports_binding_one_name_to_two_things_are_not_merged() { + assert_snapshot!(resolved("import os +<<<<<<< ours +from a import x +||||||| base +======= +from b import x +>>>>>>> theirs +"), @" + import os + <<<<<<< ours + from a import x + ||||||| base + ======= + from b import x + >>>>>>> theirs + "); + } + + #[test] + fn an_import_one_side_removes_is_removed() { + assert_snapshot!(resolved("import os +<<<<<<< ours +import json +||||||| base +import re +======= +import sys +import re +>>>>>>> theirs +"), @" + import os + import json + import sys + "); + } + + #[test] + fn definitions_of_one_name_are_not_merged() { + assert_snapshot!(resolved("X = 1 +<<<<<<< ours + + +def f(): + return 1 +||||||| base +======= + + +def f(): + return 2 +>>>>>>> theirs +"), @" + X = 1 + <<<<<<< ours + + + def f(): + return 1 + ||||||| base + ======= + + + def f(): + return 2 + >>>>>>> theirs + "); + } + + #[test] + fn top_level_definitions_are_kept_two_blank_lines_apart() { + assert_snapshot!(resolved("X = 1 +<<<<<<< ours + + +def f(): + return 1 +||||||| base +======= + + +def g(): + return 2 +>>>>>>> theirs +"), @" + X = 1 + + + def f(): + return 1 + + + def g(): + return 2 + "); + } + + #[test] + fn sides_that_differ_only_in_formatting_resolve_to_ours() { + assert_snapshot!(resolved("<<<<<<< ours +x = [1,2] +||||||| base +x = [] +======= +x = [ + 1, + 2, +] +>>>>>>> theirs +"), @"x = [1,2]"); + } + + #[test] + fn a_dict_key_both_sides_add_is_not_merged() { + assert_snapshot!(resolved(r#"D = { + "a": 1, +<<<<<<< ours + "b": 2, +||||||| base +======= + "b": 3, +>>>>>>> theirs +} +"#), @r#" + D = { + "a": 1, + <<<<<<< ours + "b": 2, + ||||||| base + ======= + "b": 3, + >>>>>>> theirs + } + "#); + } + + /// what is left unresolved is exactly what git wrote + fn assert_unresolved(conflicted: &str) { + assert_eq!(resolved(conflicted), conflicted, "resolved:\n{conflicted}"); + } + + /// decorators compose, so two added in one place apply in an order neither side chose + #[test] + fn decorators_both_sides_add_in_one_place_are_not_merged() { + assert_unresolved( + "class A: + @a +<<<<<<< ours + @abstractmethod +||||||| base +======= + @property +>>>>>>> theirs + def f(self): ... +", + ); + } + + #[test] + fn decorators_each_side_adds_in_its_own_place_are_kept() { + assert_snapshot!(resolved("<<<<<<< ours +@b +@a +||||||| base +@a +======= +@a +@c +>>>>>>> theirs +def f(): ... +"), @" + @b + @a + @c + def f(): ... + "); + } + + #[test] + fn elements_both_sides_add_in_one_place_are_not_merged() { + assert_unresolved( + r#"MIDDLEWARE = [ + "a", +<<<<<<< ours + "b", +||||||| base +======= + "c", +>>>>>>> theirs +] +"#, + ); + } + + #[test] + fn elements_each_side_adds_in_its_own_place_are_kept() { + assert_snapshot!(resolved(r#"MIDDLEWARE = [ +<<<<<<< ours + "b", + "a", +||||||| base + "a", +======= + "a", + "c", +>>>>>>> theirs +] +"#), @r#" + MIDDLEWARE = [ + "b", + "a", + "c", + ] + "#); + } + + /// `"p,#q"` ends with no comma, which a scan of its text for `#` misreads + #[test] + fn an_element_without_a_comma_that_would_not_be_last_is_not_merged() { + assert_unresolved( + r#"X = [ +<<<<<<< ours + 0, + 1, + "p,#q" +||||||| base + 1, + "p,#q" +======= + 1, + "p,#q", + 2, +>>>>>>> theirs +] +"#, + ); + } + + /// registering decorators run when the module does, in an order neither side chose + #[test] + fn decorated_definitions_both_sides_add_in_one_place_are_not_merged() { + assert_unresolved( + r#"app = App() +<<<<<<< ours + + +@app.get("/users/me") +def me(): + return 1 +||||||| base +======= + + +@app.get("/users/{id}") +def user(id): + return 2 +>>>>>>> theirs +"#, + ); + } + + /// ours' function ends the module and theirs' method ends the class: the method would land + /// in the function's body + #[test] + fn additions_to_two_suites_ending_in_one_place_are_not_merged() { + assert_unresolved( + "class Shape: + def area(self) -> float: + return 0.0 +<<<<<<< ours + + +def helper(): + return 1 +||||||| base +======= + + def perimeter(self) -> float: + return 1.0 +>>>>>>> theirs +", + ); + } + + /// the same statement, in two suites, means two different things + #[test] + fn the_same_statement_in_two_suites_is_not_merged() { + assert_unresolved( + "class Shape: + def area(self) -> float: + return 0.0 +<<<<<<< ours + + +def size(): + return 1 +||||||| base +======= + + def size(): + return 1 +>>>>>>> theirs +", + ); + } + + #[test] + fn a_definition_both_sides_add_first_is_kept_once() { + assert_snapshot!(resolved("X = 1 +<<<<<<< ours + + +def f(): + return 1 + + +def g(): + return 2 +||||||| base +======= + + +def f(): + return 1 + + +def h(): + return 3 +>>>>>>> theirs +"), @" + X = 1 + + + def f(): + return 1 + + + def g(): + return 2 + + + def h(): + return 3 + "); + } + + /// `A` stays after theirs' `Other`, as it is in theirs + #[test] + fn a_definition_both_sides_add_last_is_kept_once_after_both_sides_others() { + assert_snapshot!(resolved("class Base: + pass +<<<<<<< ours + + +class Mixin: + pass + + +class A(Base): + pass +||||||| base +======= + + +class Other: + pass + + +class A(Base): + pass +>>>>>>> theirs +"), @" + class Base: + pass + + + class Mixin: + pass + + + class Other: + pass + + + class A(Base): + pass + "); + } + + /// `f` is between additions of each side's own, in an order neither side chose + #[test] + fn a_definition_both_sides_add_between_others_is_not_merged() { + assert_unresolved( + "X = 1 +<<<<<<< ours + + +def g(): + return 2 + + +def f(): + return 1 +||||||| base +======= + + +def f(): + return 1 + + +def h(): + return 3 +>>>>>>> theirs +", + ); + } + + #[test] + fn a_dict_item_both_sides_add_first_is_kept_once() { + assert_snapshot!(resolved(r#"D = { + "a": 1, +<<<<<<< ours + "b": 2, + "c": 3, +||||||| base +======= + "b": 2, +>>>>>>> theirs +} +"#), @r#" + D = { + "a": 1, + "b": 2, + "c": 3, + } + "#); + } + + #[test] + fn submodule_imports_binding_one_package_are_merged() { + assert_snapshot!(resolved("import os +<<<<<<< ours +import a.b +||||||| base +======= +import a.c +>>>>>>> theirs +"), @" + import os + import a.b + import a.c + "); + } + + #[test] + fn an_import_of_a_name_the_other_side_binds_elsewhere_is_not_merged() { + assert_unresolved( + "import os +<<<<<<< ours +import json +||||||| base +======= +import sys +>>>>>>> theirs + + +def json(): + pass +", + ); + } + + /// `helper` is bound, outside the conflict, in each of the ways python binds a name; the + /// same conflict under another name is resolved + #[test] + fn a_definition_of_a_name_bound_in_any_form_is_not_merged() { + let bindings = [ + "for {} in range(3):\n pass\n", + "with open('f') as {}:\n pass\n", + "try:\n pass\nexcept ValueError as {}:\n pass\n", + "first, *{} = 1, 2\n", + "type {} = int\n", + "if flag:\n {} = 1\n", + "while ({} := next(it)):\n pass\n", + "print([({} := x) for x in range(3)])\n", + "match x:\n case [{}]:\n pass\n", + "match x:\n case {**{}}:\n pass\n", + "def bump():\n global {}\n {} = 2\n", + "del {}\n", + "import {}.path\n", + ]; + let conflicted = |binding: &str| { + format!( + "{binding} +<<<<<<< ours + + +def helper(): + return 1 +||||||| base +======= + + +def other(): + return 2 +>>>>>>> theirs +" + ) + }; + for binding in bindings { + assert_unresolved(&conflicted(&binding.replace("{}", "helper"))); + let elsewhere = conflicted(&binding.replace("{}", "unrelated")); + assert_ne!( + resolved(&elsewhere), + elsewhere, + "not resolved:\n{elsewhere}" + ); + } + } + + /// what a star import binds cannot be told + #[test] + fn a_definition_beside_a_star_import_is_not_merged() { + assert_unresolved( + "from m import * +<<<<<<< ours + + +def helper(): + return 1 +||||||| base +======= + + +def other(): + return 2 +>>>>>>> theirs +", + ); + } + + #[test] + fn a_method_named_like_an_attribute_the_other_side_sets_is_not_merged() { + assert_unresolved( + r#"class Shape: + def __init__(self): + self.name = "shape" +<<<<<<< ours + + def name(self): + return 1 +||||||| base +======= + + def perimeter(self): + return 2 +>>>>>>> theirs +"#, + ); + } + + #[test] + fn a_conflict_whose_lines_end_before_they_start_is_not_resolved() { + let text = "import os\nimport sys\n"; + let resolutions = resolve_merge_conflicts( + MergeTexts { + base: text, + ours: text, + theirs: text, + }, + &[ConflictLines { + base: Range { start: 2, end: 1 }, + ours: 0..1, + theirs: 0..1, + }], + PySourceType::Python, + &|_| None, + ); + assert_eq!(resolutions, [None]); + } + + /// a conflict cannot start inside a token, the string here, but a client can say it does + #[test] + fn a_conflict_starting_inside_a_string_is_not_resolved() { + let text = "x = '''\na\n'''\n"; + let resolutions = resolve_merge_conflicts( + MergeTexts { + base: text, + ours: text, + theirs: text, + }, + &[ConflictLines { + base: 1..2, + ours: 1..2, + theirs: 1..2, + }], + PySourceType::Python, + &|_| None, + ); + assert_eq!(resolutions, [None]); + } +} diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index c63f481436..7d26d198f0 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -18,7 +18,7 @@ mod change_signature; mod class_members; mod data_class; -mod evaluation; +pub(crate) mod evaluation; mod extract_function; mod extract_variable; mod flow; diff --git a/crates/ty_ide/src/type_name.rs b/crates/ty_ide/src/type_name.rs index f7c38730a1..12e1b59447 100644 --- a/crates/ty_ide/src/type_name.rs +++ b/crates/ty_ide/src/type_name.rs @@ -1,11 +1,11 @@ -//! The class a dotted type name names: what a debugger's *jump to type source* asks. +//! the class a dotted type name names: what a debugger's *jump to type source* asks //! -//! A debugger knows a value's type by name, `type(value).__module__` and its `__qualname__`: -//! `shapes.base.Circle`, or `shapes.base.Shape.Meta` for a class nested in another. The name is +//! a debugger knows a value's type by name, `type(value).__module__` and its `__qualname__`: +//! `shapes.base.Circle`, or `shapes.base.Shape.Meta` for a class nested in another. the name is //! resolved as an import of it would be — its module from the search paths, then each name after it //! as a member of what came before, through re-exports and submodules — so `builtins.list` is the -//! class typeshed declares and `fastapi.FastAPI` the one its package does. A name that reaches -//! something other than a class, or nothing, as `make..Local` does, is not answered. +//! class typeshed declares and `fastapi.FastAPI` the one its package does. a name that reaches +//! something other than a class, or nothing, as `make..Local` does, is not answered use ty_module_resolver::ModuleName; use ty_python_core::ProgramFile; @@ -16,8 +16,8 @@ use crate::Db; use crate::declaration::{DeclarationAt, single_declaration}; use crate::goto::Definitions; -/// The class `type_name` names, as a module of the program `context` is in would import it, with -/// its declaration where its source is, or its stub for a class that has no other. +/// the class `type_name` names, as a module of the program `context` is in would import it, with +/// its declaration where its source is, or its stub for a class that has no other pub fn type_declaration( db: &dyn Db, context: ProgramFile<'_>, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 160aedf628..6f8cc0e324 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -163,7 +163,14 @@ pub(super) fn request(req: server::Request) -> Task { requests::EndpointsHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } - // The class a debugger names a value's type by, which is in no document it has open. + // A merge's and a diff's texts, which are versions no document holds. + requests::ResolveMergeConflictsHandler::METHOD => background_request_task::< + requests::ResolveMergeConflictsHandler, + >(req, BackgroundSchedule::Worker), + requests::DiffIgnoredRangesHandler::METHOD => background_request_task::< + requests::DiffIgnoredRangesHandler, + >(req, BackgroundSchedule::Worker), + // the class a debugger names a value's type by, which is in no document it has open requests::TypeDeclarationHandler::METHOD => background_request_task::< requests::TypeDeclarationHandler, >(req, BackgroundSchedule::Worker), diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 8c9bb166fb..9f66cd7c96 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -50,6 +50,7 @@ mod goto_type_definition; mod hover; mod injections; mod inlay_hints; +mod merge_conflicts; mod module_name; mod move_members; mod move_symbol; @@ -125,6 +126,7 @@ pub(super) use goto_type_definition::GotoTypeDefinitionRequestHandler; pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; +pub(super) use merge_conflicts::{DiffIgnoredRangesHandler, ResolveMergeConflictsHandler}; pub(super) use module_name::ModuleNameHandler; pub(super) use move_members::{ ExtractSuperclassRequestHandler, PrepareMoveMembersRequestHandler, PullMembersUpRequestHandler, diff --git a/crates/ty_server/src/server/api/requests/merge_conflicts.rs b/crates/ty_server/src/server/api/requests/merge_conflicts.rs new file mode 100644 index 0000000000..63ddfb7aa2 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/merge_conflicts.rs @@ -0,0 +1,197 @@ +//! `by/resolveMergeConflicts` and `by/diffIgnoredRanges`: what an editor's merge and diff tools +//! ask a language about the texts they compare. +//! +//! Both take texts rather than documents: a merge's base, ours and theirs, and a diff's two sides, +//! are versions no file on disk holds. A `uri`, where one is given, names the file the texts are +//! versions of, which says how they are parsed and which isort configuration sorts their imports. +//! See [`ty_ide::resolve_merge_conflicts`] and [`ty_ide::diff_ignored_ranges`] for what is +//! answered. + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, Uri}; +use ruff_db::system::SystemPathBuf; +use ruff_python_ast::PySourceType; +use ruff_source_file::LineIndex; +use ty_ide::{ConflictLines, MergeTexts, diff_ignored_ranges, resolve_file_merge_conflicts}; + +use crate::document::text_range_to_lsp_range; +use crate::server::api::Error; +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum ResolveMergeConflictsRequest {} + +impl Request for ResolveMergeConflictsRequest { + type Params = ResolveMergeConflictsParams; + type Result = Vec>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/resolveMergeConflicts"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ResolveMergeConflictsParams { + base: String, + ours: String, + theirs: String, + conflicts: Vec, + #[serde(default)] + uri: Option, +} + +/// The lines a conflict spans in each version. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Conflict { + base: LineRange, + ours: LineRange, + theirs: LineRange, +} + +/// lines `start` to `end`, 0-based, `end` excluded, which is not before `start` +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LineRange { + start: usize, + end: usize, +} + +impl TryFrom<&LineRange> for std::ops::Range { + type Error = Error; + + fn try_from(lines: &LineRange) -> Result { + if lines.start > lines.end { + return Err(Error { + code: lsp_server::ErrorCode::InvalidParams, + error: anyhow::anyhow!( + "a conflict's lines end at {} before they start at {}", + lines.end, + lines.start + ), + }); + } + Ok(lines.start..lines.end) + } +} + +pub(crate) struct ResolveMergeConflictsHandler; + +impl RequestHandler for ResolveMergeConflictsHandler { + type RequestType = ResolveMergeConflictsRequest; +} + +impl BackgroundRequestHandler for ResolveMergeConflictsHandler { + /// The text each conflict resolves to, or `null` for one that is not resolved: all `null` + /// without a project to read the configuration of. + fn run( + snapshot: &SessionSnapshot, + _client: &Client, + params: ResolveMergeConflictsParams, + ) -> crate::server::Result>> { + let path = params + .uri + .as_ref() + .and_then(|uri| uri.to_file_path().ok()) + .and_then(|path| SystemPathBuf::from_path_buf(path).ok()); + let conflicts = params + .conflicts + .iter() + .map(|conflict| { + Ok(ConflictLines { + base: (&conflict.base).try_into()?, + ours: (&conflict.ours).try_into()?, + theirs: (&conflict.theirs).try_into()?, + }) + }) + .collect::, Error>>()?; + // the project the file is in, or the first, whose configuration is read for it + let db = path + .as_ref() + .and_then(|path| { + snapshot + .projects() + .iter() + .find(|db| path.starts_with(ty_project::Db::project(*db).root(*db))) + }) + .or_else(|| snapshot.projects().first()); + let Some(db) = db else { + return Ok(vec![None; conflicts.len()]); + }; + Ok(resolve_file_merge_conflicts( + db, + path.as_deref(), + MergeTexts { + base: ¶ms.base, + ours: ¶ms.ours, + theirs: ¶ms.theirs, + }, + &conflicts, + )) + } +} + +impl RetriableRequestHandler for ResolveMergeConflictsHandler {} + +pub(crate) enum DiffIgnoredRangesRequest {} + +impl Request for DiffIgnoredRangesRequest { + type Params = DiffIgnoredRangesParams; + type Result = DiffIgnoredRanges; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/diffIgnoredRanges"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DiffIgnoredRangesParams { + text: String, + #[serde(default)] + uri: Option, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DiffIgnoredRanges { + /// Each import statement. + imports: Vec, + /// Each run of text that is only formatting. + formatting: Vec, +} + +pub(crate) struct DiffIgnoredRangesHandler; + +impl RequestHandler for DiffIgnoredRangesHandler { + type RequestType = DiffIgnoredRangesRequest; +} + +impl BackgroundRequestHandler for DiffIgnoredRangesHandler { + fn run( + snapshot: &SessionSnapshot, + _client: &Client, + params: DiffIgnoredRangesParams, + ) -> crate::server::Result { + let source_type = params + .uri + .as_ref() + .and_then(|uri| uri.to_file_path().ok()) + .and_then(PySourceType::try_from_path) + .unwrap_or_default(); + let ignored = diff_ignored_ranges(¶ms.text, source_type); + let index = LineIndex::from_source_text(¶ms.text); + let encoding = snapshot.position_encoding(); + let convert = |ranges: Vec| { + ranges + .into_iter() + .map(|range| text_range_to_lsp_range(range, ¶ms.text, &index, encoding)) + .collect() + }; + Ok(DiffIgnoredRanges { + imports: convert(ignored.imports), + formatting: convert(ignored.formatting), + }) + } +} + +impl RetriableRequestHandler for DiffIgnoredRangesHandler {} diff --git a/crates/ty_server/src/server/api/requests/type_declaration.rs b/crates/ty_server/src/server/api/requests/type_declaration.rs index 1d01198547..e0bf7a3309 100644 --- a/crates/ty_server/src/server/api/requests/type_declaration.rs +++ b/crates/ty_server/src/server/api/requests/type_declaration.rs @@ -1,11 +1,11 @@ //! `by/typeDeclaration` — the class a dotted type name names, for a debugger's *jump to type -//! source*. +//! source* //! -//! A debugger knows a value's type by its module and qualified name, `shapes.base.Circle`, and +//! a debugger knows a value's type by its module and qualified name, `shapes.base.Circle`, and //! has no position in a document to ask `by/declaration` about; `workspace/symbol` matches a //! query loosely and only among the project's own files, where `builtins.list` or a package's -//! class is not. The name is resolved here as an import of it would be: see -//! [`ty_ide::type_declaration`]. +//! class is not. the name is resolved here as an import of it would be: see +//! [`ty_ide::type_declaration`] use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, Uri}; use ty_ide::type_declaration; @@ -30,24 +30,24 @@ impl Request for TypeDeclarationRequest { #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct TypeDeclarationParams { - /// The type's module and qualified name, dot-separated: `shapes.base.Shape.Meta`. + /// the type's module and qualified name, dot-separated: `shapes.base.Shape.Meta` type_name: String, } -/// The class, as `by/declaration` describes a declaration. +/// the class, as `by/declaration` describes a declaration #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TypeDeclaration { - /// Always `class`. + /// always `class` kind: String, name: String, - /// The class's own module and qualified name, which differs from the name asked about for a - /// class reached through a re-export. + /// the class's own module and qualified name, which differs from the name asked about for a + /// class reached through a re-export qualified_name: Option, uri: Uri, - /// The whole class. + /// the whole class range: Range, - /// Its name. + /// its name selection_range: Range, } @@ -58,8 +58,8 @@ impl RequestHandler for TypeDeclarationHandler { } impl BackgroundRequestHandler for TypeDeclarationHandler { - /// The first project in the session whose environment the name resolves in; `null` for a name - /// that names no class. + /// the first project in the session whose environment the name resolves in; `null` for a name + /// that names no class fn run( snapshot: &SessionSnapshot, _client: &Client, diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index c310687e9c..ab19a94708 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -57,6 +57,7 @@ mod inherited_members; mod initialize; mod injections; mod inlay_hints; +mod merge_conflicts; mod move_symbol; mod named_text; mod notebook; diff --git a/crates/ty_server/tests/e2e/merge_conflicts.rs b/crates/ty_server/tests/e2e/merge_conflicts.rs new file mode 100644 index 0000000000..3a86b8aa7f --- /dev/null +++ b/crates/ty_server/tests/e2e/merge_conflicts.rs @@ -0,0 +1,91 @@ +//! `by/resolveMergeConflicts` and `by/diffIgnoredRanges` over the wire: texts in, no document. + +use anyhow::Result; +use lsp_server::ErrorCode; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use serde_json::{Value, json}; + +use crate::{AwaitResponseError, TestServerBuilder}; + +enum ResolveMergeConflicts {} + +impl Request for ResolveMergeConflicts { + type Params = Value; + type Result = Value; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/resolveMergeConflicts"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +enum DiffIgnoredRanges {} + +impl Request for DiffIgnoredRanges { + type Params = Value; + type Result = Value; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/diffIgnoredRanges"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[test] +fn each_conflict_is_answered_with_its_resolution_or_null() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(SystemPath::new("project/main.by"), "")? + .build() + .wait_until_workspaces_are_initialized(); + let uri = server.file_uri("project/main.by"); + let answer = server.send_request_await::(json!({ + "base": "import os\nLIMIT = 1\n", + "ours": "import os\nimport sys\nLIMIT = 2\n", + "theirs": "import os\nimport json\nLIMIT = 3\n", + "conflicts": [ + {"base": {"start": 1, "end": 1}, "ours": {"start": 1, "end": 2}, "theirs": {"start": 1, "end": 2}}, + {"base": {"start": 1, "end": 2}, "ours": {"start": 2, "end": 3}, "theirs": {"start": 2, "end": 3}}, + ], + "uri": uri, + })); + assert_eq!(answer, json!(["import json\nimport sys\n", null])); + Ok(()) +} + +#[test] +fn lines_that_end_before_they_start_are_invalid_params() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .build() + .wait_until_workspaces_are_initialized(); + let id = server.send_request::(json!({ + "base": "import os\n", + "ours": "import os\n", + "theirs": "import os\n", + "conflicts": [ + {"base": {"start": 1, "end": 0}, "ours": {"start": 0, "end": 1}, "theirs": {"start": 0, "end": 1}}, + ], + })); + match server.try_await_response::(&id, None) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, ErrorCode::InvalidParams as i32); + } + other => panic!("expected invalid params, got {other:?}"), + } + Ok(()) +} + +#[test] +fn imports_and_formatting_are_answered_apart() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .build() + .wait_until_workspaces_are_initialized(); + let answer = server + .send_request_await::(json!({ "text": "import os\nx = [1 ,2]\n" })); + let range = |line: u32, start: u32, end: u32| json!({"start": {"line": line, "character": start}, "end": {"line": line, "character": end}}); + assert_eq!( + answer, + json!({ + "imports": [range(0, 0, 9)], + "formatting": [range(0, 6, 7), range(1, 1, 2), range(1, 3, 4), range(1, 6, 7)], + }) + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 39291d3e15..4f7778c8f2 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1531,6 +1531,68 @@ holding one — leaves its route out rather than listing it at a path it may not have, and so does a FastAPI router built somewhere other than where it is assigned, and a route whose handler, declaration or application cannot be opened +## merges and diffs + +an editor's merge and diff tools compare versions of a file no document holds, so +both requests take texts, and an optional `uri` naming the file they are versions +of: its extension says how they are parsed (python without one), and the isort +configuration buff reads for it sorts imports (the first project's root's +without one). + +`by/resolveMergeConflicts` takes the merge's three versions, `base`, `ours` and +`theirs`, each a whole file, and `conflicts`, the lines each conflict spans in +each version, `{"base": {"start": 3, "end": 3}, "ours": {"start": 3, "end": 5}, "theirs": {"start": 3, "end": 4}}`, 0-based with `end` excluded. it answers, for +each conflict in order, the text that replaces its lines, or `null` where it +cannot be resolved. a conflict is read as what its lines hold in each version's +whole syntax tree, and is resolved only where the lines are in the same body or +display in all three versions, and the two sides' changes provably touch nothing +in common: + +- import statements: every import either side added, without those either side + removed, written as buff's isort configuration sorts and writes them. refused + when both sides changed what one name is bound to differently, when the result + binds one name to two things (`from a import x` and `from b import x`), when a + side imports a name the other side binds elsewhere other than by an import of + the same thing, when either side has a star import among them, and when a + comment is among them +- functions and classes, methods among them, that each side added: each kept + where it was added, at least two blank lines apart at the top level and one + elsewhere. refused when a side changed or removed a statement or added + anything else, or added a name the other side's version binds anywhere in that + scope, in any form, or, in a class, assigns as an attribute anywhere in the + file; a star import in that scope refuses too. what both sides added first or + last in one place is kept once. definitions both sides added differently in + one place are kept, ours first, only when running each binds its name and does + nothing else: no decorator, no base class or class keyword, and every default + and annotation a name or a literal +- elements of a list, set, tuple or dict display, or of a decorator list, that + each side added in a different place: each kept where it was added. refused + when both sides added different elements in one place, since a display + evaluates and keeps its elements in order and decorators apply in order, so + `__all__` entries both sides append are left to you, and when a side changed + or removed one, two elements share a line, an element that would no longer be + last has no comma after it, or both sides add one key of a dict +- sides whose statements or elements are the same syntax tree with the same + comments, which differ only in formatting: ours + +anything else is `null`: lines that are not whole statements or whole elements, +and two sides that changed one statement, as `LIMIT = 20` and `LIMIT = 30` did. + +`by/diffIgnoredRanges` takes `{"text": "…"}` and answers what a diff ignoring +imports and formatting may ignore in it, `{"imports": [...], "formatting": [...]}`, +each a list of ranges. `imports` is each import statement, wherever it is. +`formatting` is the text python's tokenizer discards without its being +significant: whitespace between two tokens of a logical line, the line breaks and +indentation of a line continued inside brackets or after a backslash, and blank +lines. the indentation a logical line starts with is not formatting, since it +says which block the line is in, and nor is anything inside an f-string or a +t-string, where `f"{x = }"` prints the spaces around its `=`. two texts that +differ only in what is ignored tokenize alike. a change that is formatting +across lines, as rewrapping a call is, is ignored only where each side's +whitespace is, which is as much as one text can say: whether two texts differ +only in formatting is a question about both, which `by/resolveMergeConflicts` +answers for a conflict + ## override and implement members an editor's *override methods* and *implement methods* ask twice: once for what From ca3547b8a9efe24c54f41d0e5bd52a216a182f02 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:53:49 +1000 Subject: [PATCH 059/110] place added imports as buff's isort writes them, reading its configuration through the project's files an import a completion, an unresolved name's quick fix, a paste or an inlay hint adds now: - goes under the right section heading, and writes the heading import-heading configures for a section new to the block - is combined into one from import when the file has no import block or no sort order - is wrapped to the line length, and a from import a new member lengthens is rewritten as isort writes it - goes inline between two imports written on one line - in a notebook, joins the block of the cell it is used in the isort configuration, extended files and first-party source roots are read through the db's System and cached against the revision of every file looked at, so an edit to an extended file or a config created later is seen, and completion no longer stats the filesystem for each candidate. ruff_linter's ImportOrder implements ImportOrdering itself --- Cargo.lock | 1 - crates/ruff_linter/src/packaging.rs | 47 +- .../ruff_linter/src/rules/isort/categorize.rs | 116 ++- crates/ruff_linter/src/rules/isort/mod.rs | 8 +- .../ruff_linter/src/rules/isort/placement.rs | 363 +++++--- .../src/rules/isort/rules/organize_imports.rs | 3 +- crates/ruff_linter/src/rules/isort/sorting.rs | 8 +- crates/ruff_python_importer/src/order.rs | 53 +- crates/ruff_workspace/src/configuration.rs | 41 +- crates/ruff_workspace/src/files.rs | 158 ++++ crates/ruff_workspace/src/lib.rs | 1 + crates/ruff_workspace/src/pyproject.rs | 88 +- crates/ruff_workspace/src/resolver.rs | 62 +- crates/ty_ide/Cargo.toml | 1 - crates/ty_ide/src/code_action.rs | 9 +- crates/ty_ide/src/completion.rs | 5 +- crates/ty_ide/src/import_order.rs | 467 +++++++--- crates/ty_ide/src/importer.rs | 655 +++++++++++++- crates/ty_ide/src/inlay_hints.rs | 93 +- crates/ty_ide/src/lib.rs | 48 ++ crates/ty_ide/src/refactor/class_members.rs | 2 + .../ty_ide/src/refactor/return_annotation.rs | 1 + ...nts_are_not_named_outside_basedpython.snap | 11 +- crates/ty_python_semantic/src/importer.rs | 810 +++++++++++------- crates/ty_server/tests/e2e/code_actions.rs | 6 +- crates/ty_server/tests/e2e/completions.rs | 18 +- crates/ty_server/tests/e2e/pasted_imports.rs | 6 +- ...action_attribute_access_on_unimported.snap | 2 +- ...ions__code_action_undefined_decorator.snap | 4 +- ...code_action_undefined_reference_multi.snap | 4 +- ...tion_with_full_diagnostic_output_link.snap | 4 +- docs/basedpython/features/editor.md | 4 +- 32 files changed, 2367 insertions(+), 732 deletions(-) create mode 100644 crates/ruff_workspace/src/files.rs diff --git a/Cargo.lock b/Cargo.lock index 49fd4eb10c..7f044f1ac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4982,7 +4982,6 @@ dependencies = [ "ruff_memory_usage", "ruff_python_ast", "ruff_python_codegen", - "ruff_python_importer", "ruff_python_literal", "ruff_python_parser", "ruff_python_stdlib", diff --git a/crates/ruff_linter/src/packaging.rs b/crates/ruff_linter/src/packaging.rs index 34f6c98924..2487d593bb 100644 --- a/crates/ruff_linter/src/packaging.rs +++ b/crates/ruff_linter/src/packaging.rs @@ -23,23 +23,60 @@ use std::path::{Path, PathBuf}; // Thus, for each file, we iterate up its ancestors, returning the last // directory containing an `__init__.py`. +/// the file system packages and modules are looked up in: the host's, or the view of it another +/// program keeps, as a language server keeps one of its workspace +pub trait PackageFiles { + /// whether `path` is a file + fn is_file(&self, path: &Path) -> bool; + + /// whether `path` is a directory + fn is_directory(&self, path: &Path) -> bool; +} + +/// the host's file system +#[derive(Debug, Clone, Copy, Default)] +pub struct HostPackageFiles; + +impl PackageFiles for HostPackageFiles { + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn is_directory(&self, path: &Path) -> bool { + path.is_dir() + } +} + /// Return `true` if the directory at the given `Path` appears to be a Python /// package. pub fn is_package(path: &Path, namespace_packages: &[PathBuf]) -> bool { + is_package_in(&HostPackageFiles, path, namespace_packages) +} + +/// whether the directory at `path` of `files` is a python package +fn is_package_in(files: &dyn PackageFiles, path: &Path, namespace_packages: &[PathBuf]) -> bool { namespace_packages .iter() .any(|namespace_package| path.starts_with(namespace_package)) - || path.join("__init__.py").is_file() - || path.join("__init__.pyi").is_file() - || path.join("__init__.by").is_file() - || path.join("__init__.byi").is_file() + || ["__init__.py", "__init__.pyi", "__init__.by", "__init__.byi"] + .into_iter() + .any(|init| files.is_file(&path.join(init))) } /// Return the package root for the given path to a directory with Python file. pub fn detect_package_root<'a>(path: &'a Path, namespace_packages: &[PathBuf]) -> Option<&'a Path> { + detect_package_root_in(&HostPackageFiles, path, namespace_packages) +} + +/// the package root of the directory at `path` of `files`, as [`detect_package_root`] finds it +pub fn detect_package_root_in<'a>( + files: &dyn PackageFiles, + path: &'a Path, + namespace_packages: &[PathBuf], +) -> Option<&'a Path> { let mut current = None; for parent in path.ancestors() { - if !is_package(parent, namespace_packages) { + if !is_package_in(files, parent, namespace_packages) { return current; } current = Some(parent); diff --git a/crates/ruff_linter/src/rules/isort/categorize.rs b/crates/ruff_linter/src/rules/isort/categorize.rs index 203a375695..e254c4c52b 100644 --- a/crates/ruff_linter/src/rules/isort/categorize.rs +++ b/crates/ruff_linter/src/rules/isort/categorize.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; use crate::package::PackageRoot; +use crate::packaging::{HostPackageFiles, PackageFiles}; use crate::settings::types::IdentifierPattern; use crate::warn_user_once; use ruff_macros::CacheKey; @@ -89,6 +90,25 @@ enum Reason<'a> { DisabledSection(&'a ImportSection), } +/// the `src` roots a module is looked up in to tell whether it is first-party, and the file +/// system they are looked up in +#[derive(Clone, Copy)] +pub(crate) struct SourceRoots<'a> { + roots: &'a [PathBuf], + files: &'a dyn PackageFiles, +} + +impl<'a> SourceRoots<'a> { + pub(crate) fn new(roots: &'a [PathBuf], files: &'a dyn PackageFiles) -> Self { + Self { roots, files } + } + + /// `roots` on the host's file system + pub(crate) fn host(roots: &'a [PathBuf]) -> Self { + Self::new(roots, &HostPackageFiles) + } +} + #[expect(clippy::too_many_arguments)] pub(crate) fn categorize<'a>( module_name: &str, @@ -101,6 +121,34 @@ pub(crate) fn categorize<'a>( no_sections: bool, section_order: &'a [ImportSection], default_section: &'a ImportSection, +) -> &'a ImportSection { + categorize_in( + module_name, + is_relative, + SourceRoots::host(src), + package, + detect_same_package, + known_modules, + target_version, + no_sections, + section_order, + default_section, + ) +} + +/// the section of `module_name`, looking it up in the `src` roots of `src` +#[expect(clippy::too_many_arguments)] +pub(crate) fn categorize_in<'a>( + module_name: &str, + is_relative: bool, + src: SourceRoots<'_>, + package: Option>, + detect_same_package: bool, + known_modules: &'a KnownModules, + target_version: PythonVersion, + no_sections: bool, + section_order: &'a [ImportSection], + default_section: &'a ImportSection, ) -> &'a ImportSection { let module_base = module_name.split('.').next().unwrap(); let (mut import_type, mut reason) = { @@ -167,17 +215,17 @@ fn same_package(package: Option>, module_base: &str) -> bool { /// - The module named `foo.baz` will match `[SRC]` only if `[SRC]/foo/baz` /// is a directory, or `[SRC]/foo/baz.py` exists, /// or `[SRC]/foo/baz.pyi` exists. -fn match_sources<'a>(paths: &'a [PathBuf], name: &str) -> Option<&'a Path> { +fn match_sources<'a>(src: SourceRoots<'a>, name: &str) -> Option<&'a Path> { let relative_path: PathBuf = name.split('.').collect(); relative_path.components().next()?; - for root in paths { + for root in src.roots { let candidate = root.join(&relative_path); - if candidate.is_dir() { + if src.files.is_directory(&candidate) { return Some(root); } if ["py", "pyi", "by", "byi"] .into_iter() - .any(|extension| candidate.with_extension(extension).is_file()) + .any(|extension| src.files.is_file(&candidate.with_extension(extension))) { return Some(root); } @@ -188,7 +236,7 @@ fn match_sources<'a>(paths: &'a [PathBuf], name: &str) -> Option<&'a Path> { #[expect(clippy::too_many_arguments)] pub(crate) fn categorize_imports<'a>( block: ImportBlock<'a>, - src: &[PathBuf], + src: SourceRoots<'_>, package: Option>, detect_same_package: bool, known_modules: &'a KnownModules, @@ -204,7 +252,7 @@ pub(crate) fn categorize_imports<'a>( reason = "categorization only moves entries between unordered import blocks" )] for (alias, comments) in block.import { - let import_type = categorize( + let import_type = categorize_in( &alias.module_name(), false, src, @@ -228,7 +276,7 @@ pub(crate) fn categorize_imports<'a>( reason = "categorization only moves entries between unordered import blocks" )] for (import_from, aliases) in block.import_from { - let classification = categorize( + let classification = categorize_in( &import_from.module_name(), import_from.level > 0, src, @@ -252,7 +300,7 @@ pub(crate) fn categorize_imports<'a>( reason = "categorization only moves entries between unordered import blocks" )] for ((import_from, alias), aliases) in block.import_from_as { - let classification = categorize( + let classification = categorize_in( &import_from.module_name(), import_from.level > 0, src, @@ -276,7 +324,7 @@ pub(crate) fn categorize_imports<'a>( reason = "categorization only moves entries between unordered import blocks" )] for (import_from, comments) in block.import_from_star { - let classification = categorize( + let classification = categorize_in( &import_from.module_name(), import_from.level > 0, src, @@ -429,7 +477,7 @@ impl fmt::Display for KnownModules { #[cfg(test)] mod tests { - use crate::rules::isort::categorize::match_sources; + use crate::rules::isort::categorize::{SourceRoots, match_sources}; use std::fs; use std::path::{Path, PathBuf}; @@ -471,16 +519,19 @@ mod tests { let paths = vec![project_dir.clone()]; assert_eq!( - match_sources(&paths, "mypackage"), + match_sources(SourceRoots::host(&paths), "mypackage"), Some(project_dir.as_path()) ); assert_eq!( - match_sources(&paths, "mypackage.module1"), + match_sources(SourceRoots::host(&paths), "mypackage.module1"), Some(project_dir.as_path()) ); - assert_eq!(match_sources(&paths, "mypackage.nonexistent",), None); + assert_eq!( + match_sources(SourceRoots::host(&paths), "mypackage.nonexistent",), + None + ); } /// Tests a src-based Python package layout: @@ -505,11 +556,14 @@ mod tests { let paths = vec![src_dir.clone()]; assert_eq!( - match_sources(&paths, "mypackage.module1"), + match_sources(SourceRoots::host(&paths), "mypackage.module1"), Some(src_dir.as_path()) ); - assert_eq!(match_sources(&paths, "mypackage.nonexistent"), None); + assert_eq!( + match_sources(SourceRoots::host(&paths), "mypackage.nonexistent"), + None + ); } /// Tests a nested package layout: @@ -537,12 +591,15 @@ mod tests { let paths = vec![project_dir.clone()]; assert_eq!( - match_sources(&paths, "mypackage.subpackage.module2"), + match_sources(SourceRoots::host(&paths), "mypackage.subpackage.module2"), Some(project_dir.as_path()) ); assert_eq!( - match_sources(&paths, "mypackage.subpackage.nonexistent"), + match_sources( + SourceRoots::host(&paths), + "mypackage.subpackage.nonexistent" + ), None ); } @@ -567,16 +624,19 @@ mod tests { let paths = vec![project_dir.clone()]; assert_eq!( - match_sources(&paths, "namespace.package1"), + match_sources(SourceRoots::host(&paths), "namespace.package1"), Some(project_dir.as_path()) ); assert_eq!( - match_sources(&paths, "namespace.package1.module1"), + match_sources(SourceRoots::host(&paths), "namespace.package1.module1"), Some(project_dir.as_path()) ); - assert_eq!(match_sources(&paths, "namespace.package2.module1"), None); + assert_eq!( + match_sources(SourceRoots::host(&paths), "namespace.package2.module1"), + None + ); } /// Tests a package with type stubs (.pyi files): @@ -600,7 +660,7 @@ mod tests { // Module "mypackage.module1" should match project_dir using .pyi file assert_eq!( - match_sources(&paths, "mypackage.module1"), + match_sources(SourceRoots::host(&paths), "mypackage.module1"), Some(project_dir.as_path()) ); } @@ -631,13 +691,13 @@ mod tests { // Module "mypackage.feature" should match project_dir assert_eq!( - match_sources(&paths, "mypackage.feature"), + match_sources(SourceRoots::host(&paths), "mypackage.feature"), Some(project_dir.as_path()) ); // Module "mypackage.feature.submodule" should match project_dir assert_eq!( - match_sources(&paths, "mypackage.feature.submodule"), + match_sources(SourceRoots::host(&paths), "mypackage.feature.submodule"), Some(project_dir.as_path()) ); } @@ -675,13 +735,13 @@ mod tests { // Module "package1" should match project1_dir assert_eq!( - match_sources(&paths, "package1"), + match_sources(SourceRoots::host(&paths), "package1"), Some(project1_dir.as_path()) ); // Module "package2" should match project2_dir assert_eq!( - match_sources(&paths, "package2"), + match_sources(SourceRoots::host(&paths), "package2"), Some(project2_dir.as_path()) ); @@ -690,7 +750,7 @@ mod tests { // Module "package1" should still match project1_dir assert_eq!( - match_sources(&paths_reversed, "package1"), + match_sources(SourceRoots::host(&paths_reversed), "package1"), Some(project1_dir.as_path()) ); } @@ -713,7 +773,7 @@ mod tests { let paths = vec![project_dir]; - assert_eq!(match_sources(&paths, ""), None); + assert_eq!(match_sources(SourceRoots::host(&paths), ""), None); } /// Tests behavior with an empty list of source paths @@ -721,6 +781,6 @@ mod tests { fn test_empty_paths() { let paths: Vec = vec![]; - assert_eq!(match_sources(&paths, "mypackage"), None); + assert_eq!(match_sources(SourceRoots::host(&paths), "mypackage"), None); } } diff --git a/crates/ruff_linter/src/rules/isort/mod.rs b/crates/ruff_linter/src/rules/isort/mod.rs index cf56ec8a20..d262ed113f 100644 --- a/crates/ruff_linter/src/rules/isort/mod.rs +++ b/crates/ruff_linter/src/rules/isort/mod.rs @@ -1,12 +1,10 @@ //! Rules from [isort](https://pypi.org/project/isort/). -use std::path::PathBuf; - use annotate::annotate_imports; use block::{Block, Trailer}; pub(crate) use categorize::categorize; -use categorize::categorize_imports; pub use categorize::{ImportSection, ImportType}; +use categorize::{SourceRoots, categorize_imports}; use comments::Comment; use normalize::normalize_imports; use order::order_imports; @@ -74,7 +72,7 @@ pub(crate) fn format_imports( line_length: LineLength, indentation_width: LineWidthBuilder, stylist: &Stylist, - src: &[PathBuf], + src: SourceRoots<'_>, package: Option>, source_type: PySourceType, target_version: PythonVersion, @@ -158,7 +156,7 @@ fn format_import_block( line_length: LineLength, indentation_width: LineWidthBuilder, stylist: &Stylist, - src: &[PathBuf], + src: SourceRoots<'_>, package: Option>, target_version: PythonVersion, settings: &Settings, diff --git a/crates/ruff_linter/src/rules/isort/placement.rs b/crates/ruff_linter/src/rules/isort/placement.rs index a06e151ae9..6e65628ae5 100644 --- a/crates/ruff_linter/src/rules/isort/placement.rs +++ b/crates/ruff_linter/src/rules/isort/placement.rs @@ -1,25 +1,32 @@ -//! Where isort puts one import among the imports of a block it has already sorted. +//! where isort puts one import among the imports of a block it has already sorted //! -//! An editor that adds an import (an auto-import completion, a quick fix for an unresolved +//! an editor that adds an import (an auto-import completion, a quick fix for an unresolved //! name) inserts a single statement, or a single member into an existing `from` import, and -//! leaves every other line alone. To leave the block as `I001` would write it, the statement +//! leaves every other line alone. to leave the block as `I001` would write it, the statement //! has to land in its section, in the order isort sorts that section, separated from its //! neighbours by the blank lines isort puts there. [`ImportOrder`] answers those questions //! with the same keys, sections and settings [`super::format_imports`] sorts with, so the two -//! cannot disagree. +//! cannot disagree +use std::cell::RefCell; use std::cmp::Ordering; use std::path::PathBuf; +use std::sync::Arc; -use ruff_python_ast::PythonVersion; +use ruff_python_ast::{PySourceType, PythonVersion}; +use ruff_python_codegen::Stylist; +use ruff_python_importer::ImportOrdering; pub use ruff_python_importer::SortedImport; +use rustc_hash::FxHashMap; -use super::categorize::{ImportSection, ImportType, categorize}; +use super::categorize::{ImportSection, ImportType, SourceRoots, categorize_in}; use super::settings::Settings; use super::sorting::{ImportStyle, MemberKey, ModuleKey}; use crate::package::PackageRoot; +use crate::packaging::{HostPackageFiles, PackageFiles}; +use crate::settings::LinterSettings; -/// The import style isort groups `import` by within a section. +/// the import style isort groups `import` by within a section fn style(import: &SortedImport) -> ImportStyle { match import { SortedImport::Straight { .. } => ImportStyle::Straight, @@ -27,9 +34,9 @@ fn style(import: &SortedImport) -> ImportStyle { } } -/// Where an import statement sorts in its block: its section, its group within the section -/// (plain or `from`, eager or lazy) and its key within the group. Positions compare as isort -/// orders the statements. +/// where an import statement sorts in its block: its section, its group within the section +/// (plain or `from`, eager or lazy) and its key within the group. positions compare as isort +/// orders the statements #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct ImportPosition<'a> { section: usize, @@ -39,12 +46,12 @@ pub struct ImportPosition<'a> { } impl ImportPosition<'_> { - /// The index of the statement's section in the configured section order. + /// the index of the statement's section in the configured section order pub fn section(&self) -> usize { self.section } - /// The position, owning the names it compares. + /// the position, owning the names it compares pub fn into_owned(self) -> ImportPosition<'static> { ImportPosition { section: self.section, @@ -55,34 +62,70 @@ impl ImportPosition<'_> { } } -/// The ordering isort applies to the imports of one file. -#[derive(Debug, Clone, Copy)] -pub struct ImportOrder<'a> { - settings: &'a Settings, - src: &'a [PathBuf], - package: Option>, +/// The ordering isort applies to the imports of one file, looking modules up in `files` to tell +/// whether they are first-party. +pub struct ImportOrder { + settings: Arc, + package: Option, target_version: PythonVersion, + files: F, + /// the index of the section of each module categorized so far, by the name `categorize` + /// reads, which has the leading dots of a relative import + sections: RefCell>, } -impl<'a> ImportOrder<'a> { - /// The ordering for a file in `package` (as `detect_package_root` finds it), under the - /// project's isort `settings`, its `src` roots and the Python version it targets. +impl ImportOrder { + /// the ordering for a file in `package` (as `detect_package_root` finds it), under the + /// project's linter `settings` and the Python version it targets. pub fn new( - settings: &'a Settings, - src: &'a [PathBuf], - package: Option>, + settings: Arc, + package: Option, target_version: PythonVersion, + files: F, ) -> Self { Self { settings, - src, package, target_version, + files, + sections: RefCell::default(), } } - /// Where `import` sorts. + fn isort(&self) -> &Settings { + &self.settings.isort + } + + /// The index of the section `module_name` sorts in, in the configured section order. + fn section_index(&self, module_name: String, is_relative: bool) -> usize { + if let Some(section) = self.sections.borrow().get(&module_name) { + return *section; + } + let settings = self.isort(); + let section = categorize_in( + &module_name, + is_relative, + SourceRoots::new(&self.settings.src, &self.files), + self.package.as_deref().map(PackageRoot::root), + settings.detect_same_package, + &settings.known_modules, + self.target_version, + settings.no_sections, + &settings.section_order, + &settings.default_section, + ); + let index = settings + .section_order + .iter() + .position(|candidate| candidate == section) + .unwrap_or(settings.section_order.len()); + self.sections.borrow_mut().insert(module_name, index); + index + } + + /// where `import` sorts pub fn position<'b>(&self, import: &SortedImport<'b>) -> ImportPosition<'b> { + let settings = self.isort(); let (module_name, is_relative) = match import { SortedImport::Straight { name, .. } => ((*name).to_string(), false), SortedImport::From { module, level, .. } => ( @@ -90,45 +133,29 @@ impl<'a> ImportOrder<'a> { *level > 0, ), }; - let section = categorize( - &module_name, - is_relative, - self.src, - self.package, - self.settings.detect_same_package, - &self.settings.known_modules, - self.target_version, - self.settings.no_sections, - &self.settings.section_order, - &self.settings.default_section, - ); - let is_future = matches!(section, ImportSection::Known(ImportType::Future)); - let section = self - .settings - .section_order - .iter() - .position(|candidate| candidate == section) - .unwrap_or(self.settings.section_order.len()); + let section = self.section_index(module_name, is_relative); + let is_future = + settings.section_order.get(section) == Some(&ImportSection::Known(ImportType::Future)); let style = style(import); let key = match import { SortedImport::Straight { name, asname, .. } => { - ModuleKey::from_module(Some(name), *asname, 0, None, style, self.settings) + ModuleKey::from_module(Some(name), *asname, 0, None, style, settings) } SortedImport::From { module, level, first_member, .. - } => ModuleKey::from_module(*module, None, *level, *first_member, style, self.settings), + } => ModuleKey::from_module(*module, None, *level, *first_member, style, settings), }; - // Mirrors `order_imports`: the `__future__` section always puts its `from` imports + // mirrors `order_imports`: the `__future__` section always puts its `from` imports // first, `force-sort-within-sections` mixes the two kinds, and otherwise each kind is - // its own run, plain imports first unless `from-first`. Lazy imports follow the eager - // ones in every case. - let from_first = is_future || self.settings.from_first; - let kind = if !is_future && self.settings.force_sort_within_sections { + // its own run, plain imports first unless `from-first`. lazy imports follow the eager + // ones in every case + let from_first = is_future || settings.from_first; + let kind = if !is_future && settings.force_sort_within_sections { 0 } else { match (style, from_first) { @@ -146,78 +173,195 @@ impl<'a> ImportOrder<'a> { } } - /// How many blank lines isort writes between a statement at `before` and the statement at - /// `after` that follows it. + /// how many blank lines isort writes between a statement at `before` and the statement at + /// `after` that follows it pub fn blank_lines_between(&self, before: &ImportPosition, after: &ImportPosition) -> usize { + let settings = self.isort(); if before.section != after.section { // `format_import_block` separates two sections with a blank line unless every // section from the one after `before` up to `after`'s is listed in - // `no-lines-before`. - let separated = self - .settings + // `no-lines-before` + let separated = settings .section_order .iter() .take(after.section + 1) .skip(before.section + 1) - .any(|section| !self.settings.no_lines_before.contains(section)); + .any(|section| !settings.no_lines_before.contains(section)); return usize::from(separated); } - if self.settings.force_sort_within_sections || self.settings.lines_between_types == 0 { + if settings.force_sort_within_sections || settings.lines_between_types == 0 { return 0; } - let first_kind = if self.settings.from_first { + let first_kind = if settings.from_first { ImportStyle::From } else { ImportStyle::Straight }; if before.style == first_kind && after.style != first_kind { - self.settings.lines_between_types + settings.lines_between_types } else { 0 } } - /// The comment isort writes above the first import of the section at `position`, if one is - /// configured. - pub fn section_heading(&self, position: &ImportPosition) -> Option<&'a str> { - let section = self.settings.section_order.get(position.section)?; - self.settings + /// the comment isort writes above the first import of the section at `position`, if one is + /// configured + pub fn section_heading(&self, position: &ImportPosition) -> Option<&str> { + let settings = self.isort(); + let section = settings.section_order.get(position.section)?; + settings.import_headings.get(section).map(String::as_str) + } + + /// Whether `comment` is the heading isort writes above a section, which it takes into the + /// block when it is directly above the block's first import. + pub fn is_heading(&self, comment: &str) -> bool { + self.isort() .import_headings - .get(section) - .map(String::as_str) + .values() + .any(|heading| heading == comment) } - /// How isort orders two members of one `from` import. + /// how isort orders two members of one `from` import pub fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { - MemberKey::from_member(a.0, a.1, self.settings).cmp(&MemberKey::from_member( - b.0, - b.1, - self.settings, - )) + let settings = self.isort(); + MemberKey::from_member(a.0, a.1, settings).cmp(&MemberKey::from_member(b.0, b.1, settings)) } - /// Whether isort writes `member` (imported `as asname`, if given) from `module` in a `from` + /// whether isort writes `member` (imported `as asname`, if given) from `module` in a `from` /// import of its own rather than among the members of an existing one, under - /// `force-single-line` or, for an aliased member, without `combine-as-imports`. + /// `force-single-line` or, for an aliased member, without `combine-as-imports` pub fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { - (self.settings.force_single_line - && module.is_none_or(|module| !self.settings.single_line_exclusions.contains(module))) - || (asname.is_some() && !self.settings.combine_as_imports) + let settings = self.isort(); + (settings.force_single_line + && module.is_none_or(|module| !settings.single_line_exclusions.contains(module))) + || (asname.is_some() && !settings.combine_as_imports) + } + + /// How many blank lines isort writes between the last import of a block and the statement + /// after it, as `format_imports` writes them after a block in a source file. + pub fn blank_lines_after_block(&self, before_definition: bool) -> usize { + match usize::try_from(self.isort().lines_after_imports) { + Ok(lines) => lines, + Err(_) if before_definition => 2, + Err(_) => 1, + } + } + + /// `source`, a run of import statements, as isort writes them in `stylist`'s style, without + /// the headings of their sections; see [`sort_import_statements`]. + pub fn write_statements(&self, source: &str, stylist: &Stylist) -> Option { + let written = write_import_statements( + source, + &self.settings, + SourceRoots::new(&self.settings.src, &self.files), + self.package.as_deref().map(PackageRoot::root), + PySourceType::Python, + self.target_version, + Some(stylist), + )?; + // `source` has no comments, so the only ones written are headings + Some( + written + .split_inclusive('\n') + .filter(|line| !self.is_heading(line.trim())) + .collect(), + ) + } +} + +impl ImportOrdering for ImportOrder { + type Position = ImportPosition<'static>; + + fn position(&self, import: &SortedImport<'_>) -> Self::Position { + ImportOrder::position(self, import).into_owned() + } + + fn section(&self, position: &Self::Position) -> usize { + position.section() + } + + fn blank_lines_between(&self, before: &Self::Position, after: &Self::Position) -> usize { + ImportOrder::blank_lines_between(self, before, after) + } + + fn section_heading(&self, position: &Self::Position) -> Option<&str> { + ImportOrder::section_heading(self, position) + } + + fn is_heading(&self, comment: &str) -> bool { + ImportOrder::is_heading(self, comment) + } + + fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { + ImportOrder::compare_members(self, a, b) + } + + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + ImportOrder::imports_member_alone(self, module, asname) + } + + fn blank_lines_after_block(&self, before_definition: bool) -> usize { + ImportOrder::blank_lines_after_block(self, before_definition) + } + + fn write_statements(&self, source: &str, stylist: &Stylist) -> Option { + ImportOrder::write_statements(self, source, stylist) } } /// `source`, a run of import statements and nothing else, sorted and written as `I001`'s fix /// writes an import block with the same settings: one block, deduplicated, `from` imports of one -/// module combined, in sections. `None` for source with anything other than import statements in -/// it, or a comment, which isort keeps beside the import it is on and this cannot place. +/// module combined, in sections, each wrapped to the line length. `None` for source with +/// anything other than import statements in it, or a comment, which isort keeps beside the import +/// it is on and this cannot place. pub fn sort_import_statements( source: &str, - settings: &crate::settings::LinterSettings, + settings: &LinterSettings, + package: Option>, + source_type: PySourceType, + target_version: PythonVersion, +) -> Option { + sort_import_statements_in( + source, + settings, + &HostPackageFiles, + package, + source_type, + target_version, + ) +} + +/// [`sort_import_statements`], looking modules up in `files` to tell whether they are +/// first-party +pub fn sort_import_statements_in( + source: &str, + settings: &LinterSettings, + files: &dyn PackageFiles, + package: Option>, + source_type: PySourceType, + target_version: PythonVersion, +) -> Option { + write_import_statements( + source, + settings, + SourceRoots::new(&settings.src, files), + package, + source_type, + target_version, + None, + ) +} + +/// [`sort_import_statements`], in `stylist`'s style rather than the style of `source` itself +fn write_import_statements( + source: &str, + settings: &LinterSettings, + src: SourceRoots<'_>, package: Option>, - source_type: ruff_python_ast::PySourceType, + source_type: PySourceType, target_version: PythonVersion, + stylist: Option<&Stylist>, ) -> Option { - use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; let parsed = ruff_python_parser::parse_unchecked_source(source, source_type); @@ -241,15 +385,22 @@ pub fn sort_import_statements( trailer: None, }; let locator = crate::Locator::new(source); - let stylist = Stylist::from_tokens(parsed.tokens(), source); + let own_style; + let stylist = match stylist { + Some(stylist) => stylist, + None => { + own_style = Stylist::from_tokens(parsed.tokens(), source); + &own_style + } + }; Some(super::format_imports( &block, Vec::new(), &locator, settings.line_length, crate::line_width::LineWidthBuilder::new(settings.tab_size), - &stylist, - &settings.src, + stylist, + src, package, source_type, target_version, @@ -260,14 +411,32 @@ pub fn sort_import_statements( #[cfg(test)] mod tests { + use std::sync::Arc; + use ruff_python_ast::PythonVersion; use rustc_hash::{FxHashMap, FxHashSet}; use super::{ImportOrder, SortedImport}; + use crate::packaging::HostPackageFiles; use crate::rules::isort::categorize::{ImportSection, ImportType, KnownModules}; use crate::rules::isort::settings::Settings; + use crate::settings::LinterSettings; use crate::settings::types::IdentifierPattern; + /// the order `settings` sort with, with no `src` roots for a module to be found in + fn order(settings: Settings) -> ImportOrder { + ImportOrder::new( + Arc::new(LinterSettings { + isort: settings, + src: Vec::new(), + ..LinterSettings::default() + }), + None, + PythonVersion::PY313, + HostPackageFiles, + ) + } + fn straight(name: &str) -> SortedImport<'_> { SortedImport::Straight { name, @@ -285,8 +454,8 @@ mod tests { } } - /// Sorts `imports` by position and returns them as statements. - fn sorted(order: &ImportOrder, imports: &[SortedImport]) -> Vec { + /// sorts `imports` by position and returns them as statements + fn sorted(order: &ImportOrder, imports: &[SortedImport]) -> Vec { let mut imports: Vec<_> = imports .iter() .map(|import| (order.position(import), import)) @@ -311,8 +480,7 @@ mod tests { #[test] fn sections_then_plain_before_from() { - let settings = Settings::default(); - let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + let order = order(Settings::default()); assert_eq!( sorted( &order, @@ -338,11 +506,10 @@ mod tests { #[test] fn force_sort_within_sections_mixes_kinds() { - let settings = Settings { + let order = order(Settings { force_sort_within_sections: true, ..Settings::default() - }; - let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + }); assert_eq!( sorted( &order, @@ -362,7 +529,7 @@ mod tests { #[test] fn blank_lines() { - let settings = Settings { + let order = order(Settings { lines_between_types: 1, known_modules: KnownModules::new( vec![IdentifierPattern::new("pkg").unwrap()], @@ -373,8 +540,7 @@ mod tests { ), no_lines_before: FxHashSet::from_iter([ImportSection::Known(ImportType::FirstParty)]), ..Settings::default() - }; - let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + }); let json = order.position(&straight("json")); let collections = order.position(&from("collections", "OrderedDict")); let attr = order.position(&straight("attr")); @@ -391,8 +557,7 @@ mod tests { #[test] fn members_by_type() { - let settings = Settings::default(); - let order = ImportOrder::new(&settings, &[], None, PythonVersion::PY313); + let order = order(Settings::default()); assert!( order .compare_members(("Counter", None), ("OrderedDict", None)) diff --git a/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs b/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs index 8215f8d5a3..6cda97d799 100644 --- a/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs +++ b/crates/ruff_linter/src/rules/isort/rules/organize_imports.rs @@ -16,6 +16,7 @@ use crate::codes::Category; use crate::line_width::LineWidthBuilder; use crate::package::PackageRoot; use crate::rules::isort::block::Block; +use crate::rules::isort::categorize::SourceRoots; use crate::rules::isort::{comments, format_imports}; use crate::settings::LinterSettings; use crate::{Edit, Fix, FixAvailability, Violation}; @@ -165,7 +166,7 @@ pub(crate) fn organize_imports( settings.line_length, LineWidthBuilder::new(settings.tab_size).add_str(indentation), stylist, - &settings.src, + SourceRoots::host(&settings.src), package, source_type, target_version, diff --git a/crates/ruff_linter/src/rules/isort/sorting.rs b/crates/ruff_linter/src/rules/isort/sorting.rs index df5a4bedea..97d72f8fc7 100644 --- a/crates/ruff_linter/src/rules/isort/sorting.rs +++ b/crates/ruff_linter/src/rules/isort/sorting.rs @@ -54,7 +54,7 @@ impl PartialOrd for NatOrdStr<'_> { } impl NatOrdStr<'_> { - pub(crate) fn into_owned(self) -> NatOrdStr<'static> { + fn into_owned(self) -> NatOrdStr<'static> { NatOrdStr(Cow::Owned(self.0.into_owned())) } } @@ -150,7 +150,7 @@ impl<'a> ModuleKey<'a> { } impl ModuleKey<'_> { - /// The key, owning the names it compares. + /// the key, owning the names it compares pub(crate) fn into_owned(self) -> ModuleKey<'static> { ModuleKey { force_to_top: self.force_to_top, @@ -202,8 +202,8 @@ impl<'a> MemberKey<'a> { } impl MemberKey<'_> { - /// The key, owning the names it compares. - pub(crate) fn into_owned(self) -> MemberKey<'static> { + /// the key, owning the names it compares + fn into_owned(self) -> MemberKey<'static> { MemberKey { not_star_import: self.not_star_import, member_type: self.member_type, diff --git a/crates/ruff_python_importer/src/order.rs b/crates/ruff_python_importer/src/order.rs index b94aa2f129..f284d409e9 100644 --- a/crates/ruff_python_importer/src/order.rs +++ b/crates/ruff_python_importer/src/order.rs @@ -1,20 +1,22 @@ -//! How an import sorter orders the imports of a file, for an importer to add an import where the -//! sorter would put it. +//! how an import sorter orders the imports of a file, for an importer to add an import where the +//! sorter would put it use std::cmp::Ordering; -/// One import statement, as an import sorter reads it to sort it. +use ruff_python_codegen::Stylist; + +/// one import statement, as an import sorter reads it to sort it #[derive(Debug, Clone, Copy)] pub enum SortedImport<'a> { /// `import name` or `import name as asname`. isort sorts a statement importing several - /// modules by its first. + /// modules by its first Straight { name: &'a str, asname: Option<&'a str>, is_lazy: bool, }, /// `from module import member`, with `level` leading dots. isort sorts it by its module and - /// then by its first member. + /// then by its first member From { module: Option<&'a str>, level: u32, @@ -31,32 +33,45 @@ impl SortedImport<'_> { } } -/// The order a sorter puts the imports of one file in: which section each import belongs to, -/// where it sorts within it, and the blank lines and headings written between them. +/// the order a sorter puts the imports of one file in: which section each import belongs to, +/// where it sorts within it, and the blank lines and headings written between them pub trait ImportOrdering { - /// Where an import sorts in its block. Positions compare as the sorter orders the - /// statements. + /// where an import sorts in its block. positions compare as the sorter orders the + /// statements type Position: Ord; - /// Where `import` sorts. + /// where `import` sorts fn position(&self, import: &SortedImport<'_>) -> Self::Position; - /// The index of the section an import at `position` belongs to, in the sorter's section - /// order. + /// the index of the section an import at `position` belongs to, in the sorter's section + /// order fn section(&self, position: &Self::Position) -> usize; - /// How many blank lines the sorter writes between a statement at `before` and the statement - /// at `after` that follows it. + /// how many blank lines the sorter writes between a statement at `before` and the statement + /// at `after` that follows it fn blank_lines_between(&self, before: &Self::Position, after: &Self::Position) -> usize; - /// The comment the sorter writes above the first import of the section at `position`, if one - /// is configured. + /// the comment the sorter writes above the first import of the section at `position`, if one + /// is configured fn section_heading(&self, position: &Self::Position) -> Option<&str>; - /// How the sorter orders two members of one `from` import, each a name and its alias. + /// Whether `comment`, a whole comment line, is a heading the sorter writes above a section. + /// The sorter rewrites the headings directly above the first import of a block. + fn is_heading(&self, comment: &str) -> bool; + + /// how the sorter orders two members of one `from` import, each a name and its alias fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering; - /// Whether the sorter writes `member` (imported `as asname`, if given) from `module` in a - /// `from` import of its own rather than among the members of an existing one. + /// whether the sorter writes `member` (imported `as asname`, if given) from `module` in a + /// `from` import of its own rather than among the members of an existing one fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; + + /// How many blank lines the sorter writes between the last import of a block and the + /// statement after it, a function or class definition or another statement. + fn blank_lines_after_block(&self, before_definition: bool) -> usize; + + /// `source`, a run of import statements, as the sorter writes them in `stylist`'s style, + /// wrapped to its line length, ending in a line ending, without the headings of their + /// sections. `None` when the sorter can't say, as for source with a comment in it + fn write_statements(&self, source: &str, stylist: &Stylist) -> Option; } diff --git a/crates/ruff_workspace/src/configuration.rs b/crates/ruff_workspace/src/configuration.rs index bf4dc5dd7e..544b1ce2e9 100644 --- a/crates/ruff_workspace/src/configuration.rs +++ b/crates/ruff_workspace/src/configuration.rs @@ -9,7 +9,6 @@ use std::num::{NonZeroU8, NonZeroU16}; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow}; -use glob::{GlobError, Paths, PatternError, glob}; use itertools::Itertools; use log::debug; use regex::Regex; @@ -46,6 +45,7 @@ use ruff_python_formatter::{ AssignmentAlignment, DocstringCode, DocstringCodeLineWidth, MagicTrailingComma, QuoteStyle, }; +use crate::files::{ConfigurationFiles, HostFiles}; use crate::options::{ AnalyzeOptions, Flake8AnnotationsOptions, Flake8BanditOptions, Flake8BooleanTrapOptions, Flake8BugbearOptions, Flake8BuiltinsOptions, Flake8ComprehensionsOptions, @@ -530,6 +530,17 @@ impl Configuration { options: Options, path: Option<&Path>, project_root: &Path, + ) -> Result { + Self::from_options_in(&HostFiles, options, path, project_root) + } + + /// the configuration `options` read from `path`, as [`Self::from_options`] converts them, + /// with the `src` and `namespace-packages` globs matched against `files` + pub(crate) fn from_options_in( + files: &dyn ConfigurationFiles, + options: Options, + path: Option<&Path>, + project_root: &Path, ) -> Result { warn_about_deprecated_top_level_lint_options(&options.lint_top_level.0, path); @@ -614,7 +625,7 @@ impl Configuration { indent_width: options.indent_width, namespace_packages: options .namespace_packages - .map(|namespace_package| resolve_src(&namespace_package, project_root)) + .map(|namespace_package| resolve_src(files, &namespace_package, project_root)) .transpose()?, preview: options.preview.map(PreviewMode::from), required_version: options.required_version, @@ -622,7 +633,7 @@ impl Configuration { show_fixes: options.show_fixes, src: options .src - .map(|src| resolve_src(&src, project_root)) + .map(|src| resolve_src(files, &src, project_root)) .transpose()?, target_version: options.target_version.map(ast::PythonVersion::from), per_file_target_version: options.per_file_target_version.map(|versions| { @@ -699,13 +710,14 @@ impl Configuration { #[must_use] pub(crate) fn apply_fallbacks( mut self, + files: &dyn ConfigurationFiles, origin: ConfigurationOrigin, initial_config_path: &Path, ) -> Self { if matches!(origin, ConfigurationOrigin::Ancestor) { self.target_version = self.target_version.or_else(|| { let dir = initial_config_path.parent()?; - let fallback = pyproject::find_fallback_target_version(dir)?; + let fallback = pyproject::find_fallback_target_version_in(files, dir)?; debug!("Derived `target-version` from `requires-python`: {fallback:?}"); Some(fallback.into()) }); @@ -1479,21 +1491,20 @@ impl CombinePluginOptions for Option { /// Given a list of source paths, which could include glob patterns, resolve the /// matching paths. -fn resolve_src(src: &[String], project_root: &Path) -> Result> { +fn resolve_src( + files: &dyn ConfigurationFiles, + src: &[String], + project_root: &Path, +) -> Result> { let expansions = src .iter() .map(shellexpand::full) .collect::>, LookupError>>()?; - let globs = expansions - .iter() - .map(|path| Path::new(path.as_ref())) - .map(|path| fs::normalize_path_to(path, project_root)) - .map(|path| glob(&path.to_string_lossy())) - .collect::, PatternError>>()?; - let paths: Vec = globs - .into_iter() - .flatten() - .collect::, GlobError>>()?; + let mut paths = Vec::new(); + for path in &expansions { + let path = fs::normalize_path_to(Path::new(path.as_ref()), project_root); + paths.extend(files.glob(&path.to_string_lossy())?); + } Ok(paths) } diff --git a/crates/ruff_workspace/src/files.rs b/crates/ruff_workspace/src/files.rs new file mode 100644 index 0000000000..e3f1c98a8c --- /dev/null +++ b/crates/ruff_workspace/src/files.rs @@ -0,0 +1,158 @@ +//! where configuration files are read from: the host's file system, or the view of it another +//! program has, as a language server reads a workspace through its own file system + +use std::io; +use std::path::{Path, PathBuf}; + +use anyhow::Result; + +/// the paths a glob matches, or why it can't be matched +pub type GlobResult = Result>; + +/// the files configuration is found in and read from +pub trait ConfigurationFiles { + /// whether `path` is a file + fn is_file(&self, path: &Path) -> bool; + + /// the contents of the file at `path` + fn read_to_string(&self, path: &Path) -> io::Result; + + /// the paths matching the glob `pattern` + fn glob(&self, pattern: &str) -> GlobResult; +} + +/// the host's file system +#[derive(Debug, Clone, Copy, Default)] +pub struct HostFiles; + +impl ConfigurationFiles for HostFiles { + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn read_to_string(&self, path: &Path) -> io::Result { + std::fs::read_to_string(path) + } + + fn glob(&self, pattern: &str) -> GlobResult { + Ok(glob::glob(pattern)?.collect::, _>>()?) + } +} + +/// the paths matching the glob `pattern`, as [`glob::glob`] matches them, found by listing +/// directories with `list`, which answers the paths of a directory's entries and whether each is +/// a directory, and checking each match exists with `exists`. for a file system other than the +/// host's, which `glob::glob` can't read +pub fn glob_by_listing( + pattern: &str, + list: &dyn Fn(&Path) -> Vec<(PathBuf, bool)>, + exists: &dyn Fn(&Path) -> bool, +) -> GlobResult { + let options = glob::MatchOptions::new(); + let mut matches = vec![PathBuf::new()]; + let mut components = Path::new(pattern).components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + for path in &mut matches { + path.push(component); + } + continue; + }; + let name_pattern = name + .to_str() + .filter(|name| name.contains(['*', '?', '['])) + .map(glob::Pattern::new) + .transpose()?; + matches = match name_pattern { + None => matches.into_iter().map(|path| path.join(name)).collect(), + // `**` is every directory below the directory, and the directory itself unless the + // pattern ends with it + Some(_) if name == "**" => { + let last = components.peek().is_none(); + let mut found = Vec::new(); + let mut pending: Vec<(PathBuf, bool)> = + matches.into_iter().rev().map(|path| (path, true)).collect(); + while let Some((directory, is_start)) = pending.pop() { + let mut entries: Vec = list(&directory) + .into_iter() + .filter_map(|(entry, is_directory)| is_directory.then_some(entry)) + .collect(); + entries.sort(); + if !(last && is_start) { + found.push(directory); + } + pending.extend(entries.into_iter().rev().map(|entry| (entry, false))); + } + found + } + Some(name_pattern) => { + let mut found = Vec::new(); + for directory in matches { + let mut entries: Vec = list(&directory) + .into_iter() + .map(|(entry, _)| entry) + .filter(|entry| { + entry + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name_pattern.matches_with(name, options)) + }) + .collect(); + entries.sort(); + found.extend(entries); + } + found + } + }; + } + matches.retain(|path| exists(path)); + Ok(matches) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use anyhow::Result; + + use super::{ConfigurationFiles, HostFiles, glob_by_listing}; + + /// matching by listing directories finds what `glob::glob` finds + #[test] + fn listing_matches_as_glob_does() -> Result<()> { + let root = tempfile::tempdir()?; + for directory in ["libs/one/inner", "libs/two", "src", ".hidden"] { + std::fs::create_dir_all(root.path().join(directory))?; + } + std::fs::write(root.path().join("libs/file.py"), "")?; + let list = |directory: &Path| { + std::fs::read_dir(directory) + .into_iter() + .flatten() + .flatten() + .map(|entry| (entry.path(), entry.path().is_dir())) + .collect::>() + }; + let exists = |path: &Path| path.exists(); + for pattern in [ + "src", + "missing", + "libs/*", + "libs/t*", + "libs/**", + "*", + "libs/[to]*", + "libs/**/inner", + "**/two", + ] { + let pattern = root.path().join(pattern); + let pattern = pattern.to_string_lossy(); + assert_eq!( + glob_by_listing(&pattern, &list, &exists)?, + HostFiles.glob(&pattern)?, + "{pattern}" + ); + } + Ok(()) + } +} diff --git a/crates/ruff_workspace/src/lib.rs b/crates/ruff_workspace/src/lib.rs index 5ac23d8deb..1db9f0da8a 100644 --- a/crates/ruff_workspace/src/lib.rs +++ b/crates/ruff_workspace/src/lib.rs @@ -1,4 +1,5 @@ pub mod configuration; +pub mod files; pub mod options; pub mod pyproject; pub mod resolver; diff --git a/crates/ruff_workspace/src/pyproject.rs b/crates/ruff_workspace/src/pyproject.rs index 7e6c7c6766..d52139873f 100644 --- a/crates/ruff_workspace/src/pyproject.rs +++ b/crates/ruff_workspace/src/pyproject.rs @@ -14,6 +14,7 @@ use strum::IntoEnumIterator; use ruff_linter::settings::types::{PythonVersion, RequiredVersion}; +use crate::files::{ConfigurationFiles, HostFiles}; use crate::options::{Options, validate_required_version}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -33,7 +34,11 @@ pub struct Pyproject { project: Option, } -fn parse_toml(path: &Path, table_path: &[&str]) -> Result { +fn parse_toml( + files: &dyn ConfigurationFiles, + path: &Path, + table_path: &[&str], +) -> Result { let _guard = ValueSourceGuard::new( ValueSource::File(Arc::new(SystemPathBuf::from_path_buf_lossy( path.to_path_buf(), @@ -41,7 +46,8 @@ fn parse_toml(path: &Path, table_path: &[&str]) -> Result(path: &Path, table_path: &[&str]) -> Result Result { - parse_toml(path, &[]) +fn parse_ruff_toml(files: &dyn ConfigurationFiles, path: &Path) -> Result { + parse_toml(files, path, &[]) } /// Parse a `pyproject.toml` file. -fn parse_pyproject_toml(path: &Path) -> Result { - parse_toml(path, &["tool", "ruff"]) +fn parse_pyproject_toml(files: &dyn ConfigurationFiles, path: &Path) -> Result { + parse_toml(files, path, &["tool", "ruff"]) } /// Return `true` if a `pyproject.toml` contains a `[tool.ruff]` section. -fn ruff_enabled>(path: P) -> Result { - let pyproject = parse_pyproject_toml(path.as_ref())?; +fn ruff_enabled(files: &dyn ConfigurationFiles, path: &Path) -> Result { + let pyproject = parse_pyproject_toml(files, path)?; Ok(pyproject.tool.and_then(|tool| tool.ruff).is_some()) } /// Return the path to the `pyproject.toml` or `ruff.toml` file in a given /// directory. pub fn settings_toml>(path: P) -> Result> { - let path = path.as_ref(); + settings_toml_in(&HostFiles, path.as_ref()) +} + +/// the `pyproject.toml` or `ruff.toml` file in `path`, a directory of `files` +fn settings_toml_in(files: &dyn ConfigurationFiles, path: &Path) -> Result> { // Check for `.ruff.toml`. let ruff_toml = path.join(".ruff.toml"); - if ruff_toml.is_file() { + if files.is_file(&ruff_toml) { return Ok(Some(ruff_toml)); } // Check for `ruff.toml`. let ruff_toml = path.join("ruff.toml"); - if ruff_toml.is_file() { + if files.is_file(&ruff_toml) { return Ok(Some(ruff_toml)); } // Check for `pyproject.toml`. let pyproject_toml = path.join("pyproject.toml"); - if pyproject_toml.is_file() && ruff_enabled(&pyproject_toml)? { + if files.is_file(&pyproject_toml) && ruff_enabled(files, &pyproject_toml)? { return Ok(Some(pyproject_toml)); } @@ -107,8 +117,16 @@ pub fn settings_toml>(path: P) -> Result> { /// Find the path to the `pyproject.toml` or `ruff.toml` file, if such a file /// exists. pub fn find_settings_toml>(path: P) -> Result> { - for directory in path.as_ref().ancestors() { - if let Some(pyproject) = settings_toml(directory)? { + find_settings_toml_in(&HostFiles, path.as_ref()) +} + +/// the closest `pyproject.toml` or `ruff.toml` file of `files` in `path` or a directory above it +pub fn find_settings_toml_in( + files: &dyn ConfigurationFiles, + path: &Path, +) -> Result> { + for directory in path.ancestors() { + if let Some(pyproject) = settings_toml_in(files, directory)? { return Ok(Some(pyproject)); } } @@ -145,8 +163,17 @@ fn check_required_version(value: &toml::de::DeTable, table_path: &[&str]) -> Res /// Derive target version from `required-version` in `pyproject.toml`, if /// such a file exists in an ancestor directory. pub fn find_fallback_target_version>(path: P) -> Option { - for directory in path.as_ref().ancestors() { - if let Some(fallback) = get_fallback_target_version(directory) { + find_fallback_target_version_in(&HostFiles, path.as_ref()) +} + +/// the target version the `requires-python` of the closest `pyproject.toml` of `files` in `path` +/// or a directory above it implies +pub fn find_fallback_target_version_in( + files: &dyn ConfigurationFiles, + path: &Path, +) -> Option { + for directory in path.ancestors() { + if let Some(fallback) = get_fallback_target_version(files, directory) { return Some(fallback); } } @@ -160,12 +187,18 @@ pub fn find_user_settings_toml() -> Option { use etcetera::BaseStrategy; let strategy = etcetera::base_strategy::choose_base_strategy().ok()?; - let config_dir = strategy.config_dir().join("ruff"); + user_settings_toml_in(&HostFiles, &strategy.config_dir()) +} + +/// the user-specific `pyproject.toml` or `ruff.toml` of `files`, in the `ruff` directory of +/// `config_dir`, the user's configuration directory +pub fn user_settings_toml_in(files: &dyn ConfigurationFiles, config_dir: &Path) -> Option { + let config_dir = config_dir.join("ruff"); // Search for a user-specific `.ruff.toml`, then a `ruff.toml`, then a `pyproject.toml`. for filename in [".ruff.toml", "ruff.toml", "pyproject.toml"] { let path = config_dir.join(filename); - if path.is_file() { + if files.is_file(&path) { return Some(path); } } @@ -179,10 +212,9 @@ pub fn find_user_settings_toml() -> Option { } /// Load `Options` from a `pyproject.toml` or `ruff.toml` file. -pub(super) fn load_options>(path: P) -> Result { - let path = path.as_ref(); +pub(super) fn load_options(files: &dyn ConfigurationFiles, path: &Path) -> Result { if path.ends_with("pyproject.toml") { - let pyproject = parse_pyproject_toml(path)?; + let pyproject = parse_pyproject_toml(files, path)?; let mut ruff = pyproject .tool .and_then(|tool| tool.ruff) @@ -196,7 +228,7 @@ pub(super) fn load_options>(path: P) -> Result { } Ok(ruff) } else { - let ruff = parse_ruff_toml(path); + let ruff = parse_ruff_toml(files, path); if let Ok(ruff) = ruff { if ruff.target_version.is_none() { debug!("No `target-version` found in `{}`", path.display()); @@ -210,12 +242,15 @@ pub(super) fn load_options>(path: P) -> Result { /// Extract `target-version` from `pyproject.toml` in the given directory /// if the file exists and has `requires-python`. -fn get_fallback_target_version(dir: &Path) -> Option { +fn get_fallback_target_version( + files: &dyn ConfigurationFiles, + dir: &Path, +) -> Option { let pyproject_path = dir.join("pyproject.toml"); - if !pyproject_path.exists() { + if !files.is_file(&pyproject_path) { return None; } - let parsed_pyproject = parse_pyproject_toml(&pyproject_path); + let parsed_pyproject = parse_pyproject_toml(files, &pyproject_path); let pyproject = match parsed_pyproject { Ok(pyproject) => pyproject, @@ -281,6 +316,7 @@ mod tests { use ruff_linter::settings::types::PatternPrefixPair; use ruff_ranged_value::{ValueSource, ValueSourceGuard}; + use crate::files::HostFiles; use crate::options::{Flake8BuiltinsOptions, LintCommonOptions, LintOptions, Options}; use crate::pyproject::{Pyproject, Tools, find_settings_toml, parse_pyproject_toml}; @@ -545,7 +581,7 @@ per-file-ignores = { "__init__.py" = ["F401"] } let pyproject = find_settings_toml(tempdir.path())?.context("Failed to find pyproject.toml")?; - let pyproject = parse_pyproject_toml(&pyproject)?; + let pyproject = parse_pyproject_toml(&HostFiles, &pyproject)?; let config = pyproject .tool .context("Expected to find [tool] field")? diff --git a/crates/ruff_workspace/src/resolver.rs b/crates/ruff_workspace/src/resolver.rs index ffebbe5f06..40065babc7 100644 --- a/crates/ruff_workspace/src/resolver.rs +++ b/crates/ruff_workspace/src/resolver.rs @@ -23,6 +23,7 @@ use ruff_linter::package::PackageRoot; use ruff_linter::packaging::is_package; use crate::configuration::Configuration; +use crate::files::{ConfigurationFiles, HostFiles}; use crate::pyproject::settings_toml; use crate::settings::Settings; use crate::{FileResolverSettings, pyproject}; @@ -84,12 +85,12 @@ pub enum Relativity { } impl Relativity { - fn resolve(self, path: &Path) -> &Path { + fn resolve<'a>(self, path: &'a Path, cwd: &'a Path) -> &'a Path { match self { Relativity::Parent => path .parent() .expect("Expected pyproject.toml file to be in parent directory"), - Relativity::Cwd => &path_dedot::CWD, + Relativity::Cwd => cwd, } } } @@ -346,10 +347,19 @@ pub fn resolve_configuration( transformer: &dyn ConfigurationTransformer, origin: ConfigurationOrigin, ) -> Result { - resolve_configuration_with_cache(initial_config_path, transformer, origin, None) + resolve_configuration_with_cache( + &HostFiles, + &path_dedot::CWD, + initial_config_path, + transformer, + origin, + None, + ) } fn resolve_configuration_with_cache( + files: &dyn ConfigurationFiles, + cwd: &Path, initial_config_path: &Path, transformer: &dyn ConfigurationTransformer, origin: ConfigurationOrigin, @@ -357,7 +367,7 @@ fn resolve_configuration_with_cache( ) -> Result { let relativity = Relativity::from(origin); let mut configurations = indexmap::IndexMap::new(); - let mut next = Some(fs::normalize_path(initial_config_path)); + let mut next = Some(fs::normalize_path_to(initial_config_path, cwd)); while let Some(path) = next { if configurations.contains_key(&path) { bail!(format!( @@ -370,9 +380,9 @@ fn resolve_configuration_with_cache( )); } - let project_root = relativity.resolve(&path); + let project_root = relativity.resolve(&path, cwd); let load = || { - let options = pyproject::load_options(&path).with_context(|| { + let options = pyproject::load_options(files, &path).with_context(|| { if configurations.is_empty() { format!( "Failed to load configuration `{path}`", @@ -390,7 +400,7 @@ fn resolve_configuration_with_cache( ) } })?; - Configuration::from_options(options, Some(&path), project_root) + Configuration::from_options_in(files, options, Some(&path), project_root) }; let configuration = if let Some(cache) = configuration_cache { cache.get_or_try_insert_with(&path, project_root, load)? @@ -419,7 +429,7 @@ fn resolve_configuration_with_cache( configuration = configuration.combine(extend); } - let configuration = configuration.apply_fallbacks(origin, initial_config_path); + let configuration = configuration.apply_fallbacks(files, origin, initial_config_path); Ok(transformer.transform(configuration)) } @@ -427,6 +437,8 @@ fn resolve_configuration_with_cache( /// Extract the project root (scope) and [`Settings`] from a given /// `pyproject.toml`. fn resolve_scoped_settings( + files: &dyn ConfigurationFiles, + cwd: &Path, pyproject: &Path, transformer: &dyn ConfigurationTransformer, origin: ConfigurationOrigin, @@ -434,9 +446,15 @@ fn resolve_scoped_settings( ) -> Result<(PathBuf, Settings)> { let relativity = Relativity::from(origin); - let configuration = - resolve_configuration_with_cache(pyproject, transformer, origin, configuration_cache)?; - let project_root = relativity.resolve(pyproject); + let configuration = resolve_configuration_with_cache( + files, + cwd, + pyproject, + transformer, + origin, + configuration_cache, + )?; + let project_root = relativity.resolve(pyproject, cwd); let settings = configuration.into_settings(project_root)?; Ok((project_root.to_path_buf(), settings)) } @@ -448,7 +466,21 @@ pub fn resolve_root_settings( transformer: &dyn ConfigurationTransformer, origin: ConfigurationOrigin, ) -> Result { - let (_project_root, settings) = resolve_scoped_settings(pyproject, transformer, origin, None)?; + resolve_root_settings_in(&HostFiles, &path_dedot::CWD, pyproject, transformer, origin) +} + +/// the [`Settings`] the configuration file `pyproject` of `files` resolves to, as +/// [`resolve_root_settings`] resolves them, with `cwd` standing in for the current directory a +/// configuration from the user's own settings resolves its paths against +pub fn resolve_root_settings_in( + files: &dyn ConfigurationFiles, + cwd: &Path, + pyproject: &Path, + transformer: &dyn ConfigurationTransformer, + origin: ConfigurationOrigin, +) -> Result { + let (_project_root, settings) = + resolve_scoped_settings(files, cwd, pyproject, transformer, origin, None)?; Ok(settings) } @@ -501,6 +533,8 @@ pub fn project_files_in_path<'a>( if seen.insert(ancestor) { if let Some(pyproject) = settings_toml(ancestor)? { let (root, settings) = resolve_scoped_settings( + &HostFiles, + &path_dedot::CWD, &pyproject, transformer, ConfigurationOrigin::Ancestor, @@ -695,6 +729,8 @@ impl ParallelVisitor for PythonFilesVisitor<'_, '_> { { match settings_toml(entry.path()) { Ok(Some(pyproject)) => match resolve_scoped_settings( + &HostFiles, + &path_dedot::CWD, &pyproject, self.transformer, ConfigurationOrigin::Ancestor, @@ -824,6 +860,8 @@ pub fn project_file_at_path( for ancestor in path.ancestors() { if let Some(pyproject) = settings_toml(ancestor)? { let (root, settings) = resolve_scoped_settings( + &HostFiles, + &path_dedot::CWD, &pyproject, transformer, ConfigurationOrigin::Unknown, diff --git a/crates/ty_ide/Cargo.toml b/crates/ty_ide/Cargo.toml index 723e45ebd4..30f13732f5 100644 --- a/crates/ty_ide/Cargo.toml +++ b/crates/ty_ide/Cargo.toml @@ -22,7 +22,6 @@ ruff_linter = { workspace = true } ruff_memory_usage = { workspace = true } ruff_python_ast = { workspace = true } ruff_python_codegen = { workspace = true } -ruff_python_importer = { workspace = true } ruff_python_literal = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } diff --git a/crates/ty_ide/src/code_action.rs b/crates/ty_ide/src/code_action.rs index 8a23e207df..450854aedb 100644 --- a/crates/ty_ide/src/code_action.rs +++ b/crates/ty_ide/src/code_action.rs @@ -1059,7 +1059,8 @@ mod tests { help: This is a preferred code action | 1 + from warnings import deprecated - 2 | + 2 + + 3 | | info[code-action]: import typing_extensions.deprecated @@ -1070,7 +1071,8 @@ mod tests { help: This is a preferred code action | 1 + from typing_extensions import deprecated - 2 | + 2 + + 3 | | info[code-action]: Ignore 'unresolved-reference' for this line @@ -1386,7 +1388,8 @@ mod tests { help: This is the only import that resolves the name | 1 + from acme.reports import NamespaceReport - 2 | NamespaceReport() + 2 + + 3 | NamespaceReport() | info[code-action]: Ignore 'unresolved-reference' for this line diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 1c38874ef5..b597db3760 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -13568,11 +13568,13 @@ if foo: .build() .snapshot(); - assert_snapshot!(snapshot, @r" + assert_snapshot!(snapshot, @" Mapping :: typing :: from typing import Mapping + Mapping :: collections.abc :: from collections.abc import Mapping + Mapping :: typing_extensions :: from typing_extensions import Mapping "); } @@ -16452,6 +16454,7 @@ np.ara assert_snapshot!(builder.build().snapshot(), @" name :: a :: from a import Asdf + name :: b :: from b import Asdf "); } diff --git a/crates/ty_ide/src/import_order.rs b/crates/ty_ide/src/import_order.rs index 736d03a842..f119f862f2 100644 --- a/crates/ty_ide/src/import_order.rs +++ b/crates/ty_ide/src/import_order.rs @@ -1,81 +1,98 @@ -//! The isort configuration buff sorts a file's imports with. +//! the isort configuration buff sorts a file's imports with //! -//! An import added to a file is placed where `buff check --select I001 --fix` would put it, so +//! an import added to a file is placed where `buff check --select I001 --fix` would put it, so //! adding one never leaves a block buff then reports as unsorted. buff reads its configuration //! the way ruff does: from the closest `ruff.toml`, `.ruff.toml` or `pyproject.toml` with a //! `[tool.ruff]` table above the file, then from the user's own configuration, and otherwise its -//! defaults with the `target-version` the closest `requires-python` implies. This resolves it -//! with the same `ruff_workspace` code, so the two agree on sections, known modules and order. +//! defaults with the `target-version` the closest `requires-python` implies. this resolves it +//! with the same `ruff_workspace` code, so the two agree on sections, known modules and order //! -//! The files are read from disk, not through the database: they are not Python sources and the -//! server does not watch them. A resolved configuration is cached against the revision (the -//! modification time) of the file it was read from, so an edit to it is seen by the next import -//! added. +//! The configuration files are read through the database's [`System`], as everything else the +//! project reads is, but not tracked by it: they need not be in a directory the server watches, +//! as the user's own configuration is not. A resolved configuration is cached instead with the +//! revision of every file resolving it looked at, including the ones it found missing, and used +//! again only while each is unchanged, so an edit to a configuration file, to a file it +//! `extend`s, or a configuration file created closer to the file is seen by the next import +//! added +//! +//! Whether a module is the project's own is decided, as buff decides it, by looking for it in +//! the configured `src` directories. That lookup goes through the database's files, which keep +//! what they found until the server reports a change, so a completion offering hundreds of +//! imports doesn't look at the file system for each. -use std::cmp::Ordering; +use std::cell::RefCell; +use std::io; use std::path::{Path, PathBuf}; use std::sync::{Arc, LazyLock, Mutex}; use ruff_db::file_revision::FileRevision; +use ruff_db::files::{FileError, system_path_to_file}; use ruff_db::parsed::ParsedModuleRef; -use ruff_db::source::source_text; -use ruff_db::system::{System, SystemPath}; +use ruff_db::system::{System, SystemPath, SystemPathBuf}; use ruff_linter::package::PackageRoot; -use ruff_linter::packaging::detect_package_root; -use ruff_linter::rules::isort::placement::{ImportOrder, ImportPosition, sort_import_statements}; +use ruff_linter::packaging::{PackageFiles, detect_package_root_in}; +use ruff_linter::rules::isort::placement::{ImportOrder, sort_import_statements_in}; +use ruff_linter::settings::LinterSettings; use ruff_python_ast::{PySourceType, PythonVersion}; -use ruff_python_importer::{ImportOrdering, SortedImport}; use ruff_workspace::configuration::Configuration; +use ruff_workspace::files::{ConfigurationFiles, GlobResult, glob_by_listing}; use ruff_workspace::pyproject::{ - find_fallback_target_version, find_settings_toml, find_user_settings_toml, + find_fallback_target_version_in, find_settings_toml_in, user_settings_toml_in, }; use ruff_workspace::resolver::{ - ConfigurationOrigin, ConfigurationTransformer, resolve_root_settings, + ConfigurationOrigin, ConfigurationTransformer, resolve_root_settings_in, }; use rustc_hash::FxHashMap; use ty_project::Db; use ty_python_core::ProgramFile; use ty_python_semantic::importer::Importer; -/// An importer for `file` that adds each import where buff's isort configuration sorts it, read -/// the first time an import is placed. +/// an importer for `file` that adds each import where buff's isort configuration sorts it, read +/// the first time an import is placed pub(crate) fn importer<'a>( db: &'a dyn Db, file: ProgramFile<'a>, parsed: &'a ParsedModuleRef, ) -> Importer<'a> { - Importer::new(db, file, parsed).sorted_by(move || IsortConfig::for_program_file(db, file)) + Importer::new(db, file, parsed).sorted_by(move || order(db, file)) +} + +/// The order buff's isort configuration sorts the imports of `file` in, or `None` for a file +/// that is not on disk. A notebook's cells are each sorted in this order on their own +fn order<'a>(db: &'a dyn Db, file: ProgramFile<'a>) -> Option>> { + let path = file.file(db).path(db).as_system_path()?.as_std_path(); + let files = ProjectFiles(db); + let config = IsortConfig::new( + db.system(), + &files, + path.parent().unwrap_or(path), + Some(path), + db.project().root(db).as_std_path(), + ); + Some(ImportOrder::new( + config.settings, + config.package_root, + config.target_version, + files, + )) } -/// The isort settings that apply to one file, and what isort needs to know about the file to -/// categorize its imports. -#[derive(Debug)] -pub(crate) struct IsortConfig { - settings: Arc, +/// the isort settings that apply to one file, and what isort needs to know about the file to +/// categorize its imports +pub(crate) struct IsortConfig<'a> { + settings: Arc, package_root: Option, target_version: PythonVersion, + system: &'a dyn System, } -impl IsortConfig { - /// The configuration that applies to `file`, or `None` for a file that is not on disk or is - /// a notebook. - fn for_program_file(db: &dyn Db, file: ProgramFile<'_>) -> Option { - if source_text(db, file.file(db)).is_notebook() { - return None; - } - let path = file.file(db).path(db).as_system_path()?; - Some(Self::for_file( - db.system(), - path.as_std_path(), - db.project().root(db).as_std_path(), - )) - } - - /// The configuration that applies to the file at `path`. `project_root` stands in for the - /// directory buff resolves `src` against when no configuration file is found. - pub(crate) fn for_file(system: &dyn System, path: &Path, project_root: &Path) -> Self { +impl<'a> IsortConfig<'a> { + /// the configuration that applies to the file at `path`. `project_root` stands in for the + /// directory buff resolves `src` against when no configuration file is found + pub(crate) fn for_file(system: &'a dyn System, path: &Path, project_root: &Path) -> Self { Self::new( system, + &SystemFiles(system), path.parent().unwrap_or(path), Some(path), project_root, @@ -85,81 +102,157 @@ impl IsortConfig { /// the configuration that applies to a file in `directory`, without a per-file target /// version, which only a file's path can match pub(crate) fn for_directory( - system: &dyn System, + system: &'a dyn System, directory: &Path, project_root: &Path, ) -> Self { - Self::new(system, directory, None, project_root) + Self::new(system, &SystemFiles(system), directory, None, project_root) } + /// The configuration of a file in `directory`, its package found in `files`. fn new( - system: &dyn System, + system: &'a dyn System, + files: &dyn PackageFiles, directory: &Path, file: Option<&Path>, project_root: &Path, ) -> Self { let settings = settings_for(system, directory, project_root); - let package_root = detect_package_root(directory, &settings.linter.namespace_packages) + let package_root = detect_package_root_in(files, directory, &settings.namespace_packages) .map(Path::to_path_buf); let target_version = file - .map_or(settings.linter.unresolved_target_version, |file| { - settings.linter.resolve_target_version(file) + .map_or(settings.unresolved_target_version, |file| { + settings.resolve_target_version(file) }) .linter_version(); Self { settings, package_root, target_version, + system, } } + /// the linter settings buff reads, and the package root of the file they are for + #[cfg(test)] + pub(crate) fn settings(&self) -> (&LinterSettings, Option<&Path>) { + (&self.settings, self.package_root.as_deref()) + } + /// `source`, a run of import statements, as buff's `I001` fix sorts and writes them, or /// `None` for source with anything else in it, a comment among them. pub(crate) fn sort(&self, source: &str, source_type: PySourceType) -> Option { - sort_import_statements( + sort_import_statements_in( source, - &self.settings.linter, + &self.settings, + &SystemFiles(self.system), self.package_root.as_deref().map(PackageRoot::root), source_type, self.target_version, ) } +} - pub(crate) fn order(&self) -> ImportOrder<'_> { - ImportOrder::new( - &self.settings.linter.isort, - &self.settings.linter.src, - self.package_root.as_deref().map(PackageRoot::root), - self.target_version, - ) +/// The files of the database, which remember what they found at a path until the server reports +/// a change to it. +pub(crate) struct ProjectFiles<'db>(&'db dyn Db); + +impl PackageFiles for ProjectFiles<'_> { + fn is_file(&self, path: &Path) -> bool { + SystemPath::from_std_path(path) + .is_some_and(|path| system_path_to_file(self.0, path).is_ok()) + } + + fn is_directory(&self, path: &Path) -> bool { + SystemPath::from_std_path(path).is_some_and(|path| { + matches!( + system_path_to_file(self.0, path), + Err(FileError::IsADirectory) + ) + }) } } -impl ImportOrdering for IsortConfig { - type Position = ImportPosition<'static>; +/// The files of a [`System`], looked at each time they are asked about. +struct SystemFiles<'a>(&'a dyn System); - fn position(&self, import: &SortedImport<'_>) -> Self::Position { - self.order().position(import).into_owned() +impl PackageFiles for SystemFiles<'_> { + fn is_file(&self, path: &Path) -> bool { + SystemPath::from_std_path(path).is_some_and(|path| self.0.is_file(path)) } - fn section(&self, position: &Self::Position) -> usize { - position.section() + fn is_directory(&self, path: &Path) -> bool { + SystemPath::from_std_path(path).is_some_and(|path| self.0.is_directory(path)) } +} + +/// The configuration files of a [`System`], recording the revision of every file looked at: +/// `None` for one that is missing. +struct RecordedFiles<'a> { + system: &'a dyn System, + revisions: RefCell)>>, +} - fn blank_lines_between(&self, before: &Self::Position, after: &Self::Position) -> usize { - self.order().blank_lines_between(before, after) +impl RecordedFiles<'_> { + fn revision(&self, path: &Path) -> Option { + let revision = file_revision(self.system, path); + self.revisions + .borrow_mut() + .push((path.to_path_buf(), revision)); + revision } +} - fn section_heading(&self, position: &Self::Position) -> Option<&str> { - self.order().section_heading(position) +impl ConfigurationFiles for RecordedFiles<'_> { + fn is_file(&self, path: &Path) -> bool { + self.revision(path).is_some() } - fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { - self.order().compare_members(a, b) + fn read_to_string(&self, path: &Path) -> io::Result { + self.revision(path); + let path = SystemPath::from_std_path(path) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "not a UTF-8 path"))?; + self.system.read_to_string(path) } - fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { - self.order().imports_member_alone(module, asname) + fn glob(&self, pattern: &str) -> GlobResult { + let list = |directory: &Path| { + SystemPath::from_std_path(directory) + .and_then(|directory| self.system.read_directory(directory).ok()) + .into_iter() + .flatten() + .flatten() + .map(|entry| { + let is_directory = entry.file_type().is_directory(); + (entry.into_path().into_std_path_buf(), is_directory) + }) + .collect() + }; + let exists = |path: &Path| { + SystemPath::from_std_path(path).is_some_and(|path| self.system.path_exists(path)) + }; + glob_by_listing(pattern, &list, &exists) + } +} + +/// The revision of the file at `path`, or `None` when there is none. +fn file_revision(system: &dyn System, path: &Path) -> Option { + let metadata = system + .path_metadata(SystemPath::from_std_path(path)?) + .ok()?; + metadata.file_type().is_file().then(|| metadata.revision()) +} + +/// Sets the target version the closest `requires-python` implies, which takes precedence over +/// the user's own configuration, as buff gives it. +struct FallbackTargetVersion(Option); + +impl ConfigurationTransformer for FallbackTargetVersion { + fn transform(&self, mut configuration: Configuration) -> Configuration { + if let Some(fallback) = self.0 { + configuration.target_version = Some(fallback.into()); + } + configuration } } @@ -171,69 +264,197 @@ impl ConfigurationTransformer for Unchanged { } } -/// A resolved configuration file and the revision it was resolved at. +/// The settings resolved for a directory, and the revision of every file resolving them looked +/// at. struct Resolved { - revision: Option, - settings: Arc, + revisions: Vec<(PathBuf, Option)>, + settings: Arc, } -static RESOLVED: LazyLock>> = LazyLock::new(Mutex::default); +/// The resolved settings, by the directory, project root and user configuration directory they +/// were resolved for. +static RESOLVED: LazyLock>> = LazyLock::new(Mutex::default); + +type ResolvedKey = (PathBuf, PathBuf, Option); + +/// The linter settings buff reads for a file in `directory`, from the cache while no file +/// resolving them looked at has changed. +fn settings_for(system: &dyn System, directory: &Path, project_root: &Path) -> Arc { + let user_config_directory = system.user_config_directory(); + let key = ( + directory.to_path_buf(), + project_root.to_path_buf(), + user_config_directory.clone(), + ); + { + let cache = RESOLVED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(resolved) = cache.get(&key) + && resolved + .revisions + .iter() + .all(|(path, revision)| file_revision(system, path) == *revision) + { + return resolved.settings.clone(); + } + } + let files = RecordedFiles { + system, + revisions: RefCell::default(), + }; + let settings = Arc::new(resolve( + &files, + directory, + project_root, + user_config_directory.as_deref(), + )); + RESOLVED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + key, + Resolved { + revisions: files.revisions.into_inner(), + settings: settings.clone(), + }, + ); + settings +} -fn settings_for( - system: &dyn System, +/// The linter settings buff reads for a file in `directory`, from the configuration files of +/// `files`. A configuration file that can't be read or is invalid is passed over for the next, +/// as buff falls back on the next configuration too. +fn resolve( + files: &RecordedFiles, directory: &Path, project_root: &Path, -) -> Arc { - let project = find_settings_toml(directory).ok().flatten(); - if let Some(path) = &project - && let Some(settings) = resolved(system, path, ConfigurationOrigin::Ancestor) + user_config_directory: Option<&SystemPath>, +) -> LinterSettings { + if let Some(path) = find_settings_toml_in(files, directory).ok().flatten() + && let Ok(settings) = resolve_root_settings_in( + files, + project_root, + &path, + &Unchanged, + ConfigurationOrigin::Ancestor, + ) { - return settings; + return settings.linter; } - // the user's own configuration is left out of tests, which would otherwise depend on the - // machine they run on - if !cfg!(test) - && let Some(path) = find_user_settings_toml() - && let Some(settings) = resolved(system, &path, ConfigurationOrigin::UserSettings) + // the user's own configuration resolves its paths against the current directory buff runs + // in, which the project root stands in for + if let Some(config_directory) = user_config_directory + && let Some(path) = user_settings_toml_in(files, config_directory.as_std_path()) + && let Ok(settings) = resolve_root_settings_in( + files, + project_root, + &path, + &FallbackTargetVersion(find_fallback_target_version_in(files, project_root)), + ConfigurationOrigin::UserSettings, + ) { - return settings; + return settings.linter; } let configuration = Configuration { - target_version: find_fallback_target_version(directory).map(Into::into), + target_version: find_fallback_target_version_in(files, directory).map(Into::into), ..Configuration::default() }; - Arc::new( - configuration - .into_settings(project_root) - .unwrap_or_default(), - ) -} - -/// The settings resolved from the configuration file at `path`, or `None` when it cannot be -/// read or is invalid, as buff then falls back on the next configuration too. -fn resolved( - system: &dyn System, - path: &Path, - origin: ConfigurationOrigin, -) -> Option> { - let revision = SystemPath::from_std_path(path) - .and_then(|path| system.path_metadata(path).ok()) - .map(|metadata| metadata.revision()); - let mut cache = RESOLVED - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(resolved) = cache.get(path) - && resolved.revision == revision - { - return Some(resolved.settings.clone()); - } - let settings = Arc::new(resolve_root_settings(path, &Unchanged, origin).ok()?); - cache.insert( - path.to_path_buf(), - Resolved { - revision, - settings: settings.clone(), - }, - ); - Some(settings) + configuration + .into_settings(project_root) + .map(|settings| settings.linter) + .unwrap_or_else(|_| LinterSettings::new(project_root)) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use ruff_db::system::{SystemPathBuf, TestSystem}; + + use super::IsortConfig; + + /// The line length buff reads for `/project/main.py` of `system`. + fn line_length(system: &TestSystem) -> usize { + IsortConfig::for_file(system, Path::new("/project/main.py"), Path::new("/project")) + .settings + .line_length + .value() + .into() + } + + fn system(files: &[(&str, &str)]) -> TestSystem { + let system = TestSystem::default(); + system + .memory_file_system() + .write_files_all(files.iter().copied()) + .expect("files to be written"); + system + } + + /// A configuration file `extend`s another: an edit to the other is seen. + #[test] + fn edit_to_an_extended_file_is_seen() { + let system = system(&[ + ("/project/main.py", ""), + ("/project/ruff.toml", "extend = \"../base.toml\"\n"), + ("/base.toml", "line-length = 40\n"), + ]); + assert_eq!(line_length(&system), 40); + system + .memory_file_system() + .write_file("/base.toml", "line-length = 50\n") + .expect("file to be written"); + assert_eq!(line_length(&system), 50); + } + + /// A configuration file created where none was found is read. + #[test] + fn configuration_created_later_is_read() { + let system = system(&[("/project/main.py", "")]); + assert_eq!(line_length(&system), 88); + system + .memory_file_system() + .write_file("/project/ruff.toml", "line-length = 40\n") + .expect("file to be written"); + assert_eq!(line_length(&system), 40); + } + + /// The user's own configuration is read from the system's configuration directory. + #[test] + fn user_configuration_from_the_system() { + let system = system(&[ + ("/project/main.py", ""), + ("/config/ruff/ruff.toml", "line-length = 60\n"), + ]); + assert_eq!(line_length(&system), 88); + system + .in_memory() + .set_user_configuration_directory(Some(SystemPathBuf::from("/config"))); + assert_eq!(line_length(&system), 60); + } + + /// `src` globs are matched against the system's files: a module in a matched directory is + /// the project's own. + #[test] + fn src_globs_match_the_systems_files() { + let system = system(&[ + ("/project/main.py", ""), + ("/project/ruff.toml", "src = [\"libs/*\"]\n"), + ("/project/libs/one/shapes.py", ""), + ("/project/libs/two/colours.py", ""), + ]); + let config = IsortConfig::for_file( + &system, + Path::new("/project/main.py"), + Path::new("/project"), + ); + assert_eq!( + config.settings.src, + [ + Path::new("/project/libs/one").to_path_buf(), + Path::new("/project/libs/two").to_path_buf(), + ] + ); + } } diff --git a/crates/ty_ide/src/importer.rs b/crates/ty_ide/src/importer.rs index b6522e71f7..b4df8dc5ee 100644 --- a/crates/ty_ide/src/importer.rs +++ b/crates/ty_ide/src/importer.rs @@ -3,13 +3,22 @@ #[cfg(test)] mod tests { use insta::assert_snapshot; - - use crate::import_order; + use ruff_db::Db as _; + use ruff_linter::linter::{ParseSource, lint_only}; + use ruff_linter::package::PackageRoot; + use ruff_linter::registry::Rule; + use ruff_linter::settings::flags::Noqa; + use ruff_linter::settings::rule_table::RuleTable; + use ruff_linter::source_kind::SourceKind; + use ruff_python_ast::PySourceType; + use ty_project::Db as _; + + use crate::import_order::{self, IsortConfig}; use crate::tests::{CursorTest, cursor_test}; use ruff_python_ast::find_node::covering_node; use ruff_text_size::{Ranged, TextRange}; use ty_python_core::ProgramFile; - use ty_python_semantic::importer::{ImportRequest, Importer}; + use ty_python_semantic::importer::{ImportAction, ImportRequest, Importer}; impl CursorTest { fn import(&self, module: &str, member: &str) -> String { @@ -54,9 +63,115 @@ mod tests { ); source.replace_range(edit.range().to_std_range(), edit.content().unwrap_or("")); } + // when buff accepts the file's imports with the symbol written in, it accepts them + // with the import added too. a new import set apart from the code after it can't + // be checked when the symbol is the first code after it, as the file the importer + // sees has none there + let mut with_symbol = self.cursor.source.to_string(); + with_symbol.insert_str(self.cursor.offset.to_usize(), resp.symbol_text()); + let follows_code = + resp.import().is_none_or(|edit| { + self.cursor.parsed.suite().iter().any(|stmt| { + stmt.start() >= edit.start() && stmt.start() <= self.cursor.offset + }) + }); + if follows_code && self.unsorted_imports(&with_symbol) == 0 { + assert_eq!( + self.unsorted_imports(&source), + 0, + "buff reports the imports unsorted once one is added:\n{source}" + ); + } source } + /// The file with everything `requests` import added at once, as + /// [`Importer::combined_edits`] adds them, with each request's symbol written on a line + /// of its own at the cursor. A file buff accepts the imports of with the symbols written + /// in still has imports it accepts once they are added. + fn add_all(&self, requests: &[ImportRequest<'_>]) -> String { + let (with_symbols, source) = self.add_all_with(&self.importer(), requests); + if self.unsorted_imports(&with_symbols) == 0 { + assert_eq!( + self.unsorted_imports(&source), + 0, + "buff reports the imports unsorted once they are added:\n{source}" + ); + } + source + } + + /// The file with the symbols `requests` are referred to by written at the cursor, and + /// the file with `importer`'s imports of them added too. + fn add_all_with( + &self, + importer: &Importer<'_>, + requests: &[ImportRequest<'_>], + ) -> (String, String) { + let node = covering_node( + self.cursor.parsed.syntax().into(), + TextRange::empty(self.cursor.offset), + ) + .node(); + let members = importer.members_in_scope_at(node, self.cursor.offset); + let actions: Vec<_> = requests + .iter() + .map(|request| importer.import(*request, &members)) + .collect(); + let symbols: Vec<&str> = actions.iter().map(ImportAction::symbol_text).collect(); + let mut with_symbols = self.cursor.source.to_string(); + with_symbols.insert_str(self.cursor.offset.to_usize(), &symbols.join("\n")); + let mut source = with_symbols.clone(); + let mut edits = importer.combined_edits(&actions); + edits.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + for edit in edits { + assert!(edit.range().end() <= self.cursor.offset); + source.replace_range(edit.range().to_std_range(), edit.content().unwrap_or("")); + } + (with_symbols, source) + } + + /// How many `I001` diagnostics buff's isort check reports for `source` as the contents + /// of the cursor's file, with the configuration buff reads for it. buff sorts each cell + /// of a notebook on its own, so of a notebook only the cursor's cell is checked, which + /// is where an import is added + fn unsorted_imports(&self, source: &str) -> usize { + let Some(path) = self.cursor.file.path(&self.db).as_system_path() else { + return 0; + }; + let source = + match self.cursor.source.as_notebook().and_then(|notebook| { + notebook.cell_offsets().containing_range(self.cursor.offset) + }) { + Some(cell) => { + let grown = source.len() - self.cursor.source.as_str().len(); + &source[cell.start().to_usize()..cell.end().to_usize() + grown] + } + None => source, + }; + let root = self.db.project().root(&self.db); + let config = + IsortConfig::for_file(self.db.system(), path.as_std_path(), root.as_std_path()); + let (settings, package) = config.settings(); + let mut settings = settings.clone(); + settings.rules = RuleTable::from_iter([Rule::UnsortedImports]); + lint_only( + path.as_std_path(), + package.map(PackageRoot::root), + &settings, + Noqa::Enabled, + &SourceKind::Python { + code: source.to_string(), + is_stub: false, + is_basedpython: false, + }, + PySourceType::Python, + ParseSource::None, + ) + .diagnostics + .len() + } + fn importer(&self) -> Importer<'_> { import_order::importer( &self.db, @@ -70,8 +185,8 @@ mod tests { } } - /// A new import goes where buff's isort would sort it: into its section of the block, in - /// order among the section's imports, rather than at the top of the block. + /// a new import goes where buff's isort would sort it: into its section of the block, in + /// order among the section's imports, rather than at the top of the block #[test] fn sorted_into_its_section() { let source = r#" @@ -132,8 +247,107 @@ import attr "#); } - /// A member added to a `from` import is listed where isort orders it: constants, then - /// classes, then everything else, each alphabetically. + /// a notebook's cells are sorted each on its own, so an import goes into the import block of + /// the cell it is used in, sorted there, and the cells before it are left alone + #[test] + fn a_notebook_import_is_sorted_in_its_cell() { + let test = CursorTest::builder() + .notebook( + "main.ipynb", + &[ + "import sys", + "import json\nimport os\n\nimport attr\n\n", + ], + ) + .build(); + assert_snapshot!(test.module("math"), @" + import sys + import json + import math + import os + + import attr + + math + "); + assert_snapshot!(test.import_from("pathlib", "Path"), @" + import sys + import json + import os + from pathlib import Path + + import attr + + Path + "); + } + + /// a new block in a cell without imports goes at the top of the cell, set apart from the + /// cell's code as the sorter sets a block apart, and a statement in the next cell is not + /// code after it + #[test] + fn a_notebook_import_starts_a_block_in_its_cell() { + let test = CursorTest::builder() + .notebook( + "main.ipynb", + &["import sys", "def f():\n return ", "x = 1"], + ) + .build(); + assert_snapshot!(test.module("math"), @" + import sys + import math + + + def f(): + return math + x = 1 + "); + let test = CursorTest::builder() + .notebook( + "main.ipynb", + &["import sys", "", "def g():\n pass"], + ) + .build(); + assert_snapshot!(test.module("math"), @" + import sys + import math + math + def g(): + pass + "); + } + + /// a heading above a cell's first import is that cell's, and a heading in the cell before + /// is not taken into its block + #[test] + fn a_notebook_heading_search_stays_in_its_cell() { + let test = CursorTest::builder() + .source("ruff.toml", HEADINGS) + .notebook( + "main.ipynb", + &[ + "# Standard Library\nimport sys\n\n# Third Party", + "import attr\n\n", + ], + ) + .build(); + assert_snapshot!(test.module("os"), @" + # Standard Library + import sys + + # Third Party + # Standard Library + import os + + # Third Party + import attr + + os + "); + } + + /// a member added to a `from` import is listed where isort orders it: constants, then + /// classes, then everything else, each alphabetically #[test] fn member_sorted_among_members() { let test = cursor_test( @@ -156,7 +370,7 @@ from collections import OrderedDict, deque "); } - /// One per line in an import written one per line. + /// one per line in an import written one per line #[test] fn member_sorted_among_members_one_per_line() { let test = cursor_test( @@ -178,8 +392,8 @@ from collections import ( "); } - /// An import of a section the block has none of yet is set apart by a blank line on each - /// side, as isort separates sections. + /// an import of a section the block has none of yet is set apart by a blank line on each + /// side, as isort separates sections #[test] fn new_section_is_set_apart() { let test = cursor_test( @@ -213,7 +427,7 @@ import attr "); } - /// Comments above an import stay with it, as isort keeps them. + /// comments above an import stay with it, as isort keeps them #[test] fn placed_above_the_comments_of_the_import_after() { let test = cursor_test( @@ -235,6 +449,377 @@ import sys "); } + /// A test whose project configures buff with `ruff_toml`, a `ruff.toml`, and whose file is + /// `main`. + fn configured_test(ruff_toml: &str, main: &str) -> CursorTest { + CursorTest::builder() + .source("ruff.toml", ruff_toml) + .source("main.py", main) + .build() + } + + const HEADINGS: &str = r#" +[lint.isort.import-heading] +standard-library = "Standard Library" +third-party = "Third Party" +"#; + + /// A statement of a section before the block's first goes above the heading of that first + /// section, under a heading of its own: isort takes the headings directly above a block's + /// first import into the block and writes them again. + #[test] + fn heading_of_a_new_first_section() { + let test = configured_test( + HEADINGS, + "\ +# Third Party +import attr + + +", + ); + assert_snapshot!(test.module("os"), @" + # Standard Library + import os + + # Third Party + import attr + + os + "); + } + + /// A statement that joins the block's first section goes under its heading. + #[test] + fn heading_of_the_first_section_stays_first() { + let test = configured_test( + HEADINGS, + "\ +# Standard Library +import sys + +# Third Party +import attr + + +", + ); + assert_snapshot!(test.module("os"), @" + # Standard Library + import os + import sys + + # Third Party + import attr + + os + "); + } + + /// A new section after the others is written with its heading. + #[test] + fn heading_of_a_new_last_section() { + let test = configured_test( + HEADINGS, + "\ +# Standard Library +import os + + +", + ); + assert_snapshot!(test.module("attr"), @" + # Standard Library + import os + + # Third Party + import attr + + attr + "); + } + + /// Two members added from a module the file doesn't import yet are one `from` import, with + /// no import block to join. + #[test] + fn members_combined_without_an_import_block() { + let test = cursor_test( + "\ +x = 1 + +", + ); + assert_snapshot!(test.add_all(&[ + ImportRequest::import_from("pathlib", "PurePath"), + ImportRequest::import_from("pathlib", "Path"), + ]), @" + from pathlib import Path, PurePath + + x = 1 + PurePath + Path + "); + } + + /// And without an order to follow, as for a file buff has no configuration for. + #[test] + fn members_combined_without_an_order() { + let test = cursor_test( + "\ +x = 1 + +", + ); + let importer = Importer::new( + &test.db, + ProgramFile::new( + &test.db, + test.cursor.file, + test.db.program_environment().program(&test.db), + ), + &test.cursor.parsed, + ); + let (_, source) = test.add_all_with( + &importer, + &[ + ImportRequest::import_from("pathlib", "PurePath"), + ImportRequest::import_from("pathlib", "Path"), + ], + ); + assert_snapshot!(source, @" + from pathlib import PurePath, Path + x = 1 + PurePath + Path + "); + } + + /// A member that makes an import too long for its line wraps it, as isort writes it. + #[test] + fn member_wraps_an_import_past_the_line_length() { + let test = configured_test( + "line-length = 50\n", + "\ +from collections import OrderedDict, deque + +", + ); + assert_snapshot!(test.import("collections", "defaultdict"), @" + from collections import ( + OrderedDict, + defaultdict, + deque, + ) + defaultdict + "); + } + + /// A new statement too long for its line is written wrapped. + #[test] + fn new_statement_wrapped_to_the_line_length() { + let test = configured_test( + "line-length = 40\n", + "\ +import os + + +", + ); + assert_snapshot!(test.add_all(&[ + ImportRequest::import_from("collections", "OrderedDict"), + ImportRequest::import_from("collections", "defaultdict"), + ]), @" + import os + from collections import ( + OrderedDict, + defaultdict, + ) + + OrderedDict + defaultdict + "); + } + + /// Between two imports of one line, a statement is written on that line, in order. + #[test] + fn between_imports_of_one_line() { + let test = cursor_test( + "\ +import os; import sys + + +", + ); + assert_snapshot!(test.module("re"), @" + import os; import re; import sys + + re + "); + } + + /// Under `force-single-line`, a member is imported by a `from` import of its own. + #[test] + fn force_single_line() { + let test = configured_test( + "[lint.isort]\nforce-single-line = true\n", + "\ +from collections import OrderedDict + +", + ); + assert_snapshot!(test.import_from("collections", "deque"), @" + from collections import OrderedDict + from collections import deque + deque + "); + } + + /// Without `combine-as-imports`, an aliased member is imported by a `from` import of its + /// own. + #[test] + fn aliased_member_alone_without_combine_as_imports() { + let test = cursor_test( + "\ +from pathlib import Path + +", + ); + assert_snapshot!(test.add(ImportRequest::import_from_as("pathlib", "PurePath", "PP")), @" + from pathlib import Path + from pathlib import PurePath as PP + PP + "); + } + + /// With it, an aliased member joins the others. + #[test] + fn aliased_member_joins_with_combine_as_imports() { + let test = configured_test( + "[lint.isort]\ncombine-as-imports = true\n", + "\ +from pathlib import Path + +", + ); + assert_snapshot!(test.add(ImportRequest::import_from_as("pathlib", "PurePath", "PP")), @" + from pathlib import Path, PurePath as PP + PP + "); + } + + /// A section listed in `no-lines-before` follows the section before it without a blank line. + #[test] + fn no_lines_before() { + let test = configured_test( + "[lint.isort]\nno-lines-before = [\"third-party\"]\n", + "\ +import os + + +", + ); + assert_snapshot!(test.module("attr"), @" + import os + import attr + + attr + "); + } + + /// `lines-between-types` sets the `from` imports of a section apart from its plain ones. + #[test] + fn lines_between_types() { + let test = configured_test( + "[lint.isort]\nlines-between-types = 1\n", + "\ +import os + + +", + ); + assert_snapshot!(test.import_from("collections", "deque"), @" + import os + + from collections import deque + + deque + "); + } + + /// `from-first` puts a section's `from` imports before its plain ones. + #[test] + fn from_first() { + let test = configured_test( + "[lint.isort]\nfrom-first = true\n", + "\ +import os + + +", + ); + assert_snapshot!(test.import_from("collections", "deque"), @" + from collections import deque + import os + + deque + "); + } + + /// An eager import goes before a section's lazy ones, and a lazy one after its eager ones. + #[test] + fn eager_before_lazy() { + let test = cursor_test( + "\ +import sys +lazy import json + + +", + ); + assert_snapshot!(test.module("os"), @" + import os + import sys + lazy import json + + os + "); + } + + /// Statements added at once that go in different places of the block are each placed. + #[test] + fn several_places_at_once() { + let test = cursor_test( + "\ +import json +import sys + +import attr + + +", + ); + assert_snapshot!(test.add_all(&[ + ImportRequest::module("zlib"), + ImportRequest::module("abc"), + ImportRequest::module("attrs"), + ImportRequest::module("os"), + ]), @" + import abc + import json + import os + import sys + import zlib + + import attr + import attrs + + zlib + abc + attrs + os + "); + } + #[test] fn empty_source_qualified() { let test = cursor_test(""); @@ -355,7 +940,8 @@ from collections import Counter as C ); assert_snapshot!( test.import("collections", "defaultdict"), @" - from collections import Counter as C, defaultdict + from collections import Counter as C + from collections import defaultdict defaultdict "); } @@ -467,7 +1053,8 @@ from collections import ( test.import("collections", "defaultdict"), @" from collections import ( Counter, - OrderedDict, defaultdict, + OrderedDict, + defaultdict, ) defaultdict "); @@ -677,6 +1264,8 @@ from collections import defaultdict assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + + def foo(): collections.defaultdict from collections import defaultdict @@ -775,12 +1364,14 @@ defaultdict = 1 assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + defaultdict = 1 (collections.defaultdict) "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + defaultdict = 1 (collections.defaultdict) "); @@ -798,12 +1389,14 @@ collections = 1 assert_snapshot!( test.import("collections", "defaultdict"), @" from collections import defaultdict + collections = 1 (defaultdict) "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + collections = 1 (defaultdict) "); @@ -840,6 +1433,7 @@ defaultdict = 2 assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + collections = 1 defaultdict = 2 (collections.defaultdict) @@ -847,6 +1441,7 @@ defaultdict = 2 assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + collections = 1 defaultdict = 2 (collections.defaultdict) @@ -865,12 +1460,16 @@ def foo(): assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + + def foo(): (collections.defaultdict) "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + + def foo(): (defaultdict) "); @@ -888,12 +1487,16 @@ def defaultdict(): assert_snapshot!( test.import("collections", "defaultdict"), @" import collections + + def defaultdict(): (collections.defaultdict) "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" import collections + + def defaultdict(): (collections.defaultdict) "); @@ -911,12 +1514,16 @@ def collections(): assert_snapshot!( test.import("collections", "defaultdict"), @" from collections import defaultdict + + def collections(): (defaultdict) "); assert_snapshot!( test.import_from("collections", "defaultdict"), @" from collections import defaultdict + + def collections(): (defaultdict) "); @@ -1179,10 +1786,11 @@ import numpy as np import json import re - import foo from whenever import ZonedDateTime import numpy as np + import foo + (foo.Bar) "); assert_snapshot!( @@ -1190,10 +1798,11 @@ import numpy as np import json import re - from foo import Bar from whenever import ZonedDateTime import numpy as np + from foo import Bar + (Bar) "); } @@ -1220,6 +1829,7 @@ else: assert_snapshot!( test.import("quux", "MAGIC"), @r#" import quux + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1230,6 +1840,7 @@ else: assert_snapshot!( test.import_from("quux", "MAGIC"), @r#" import quux + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1265,6 +1876,7 @@ else: assert_snapshot!( test.import("foo", "MAGIC"), @r#" import foo + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1275,6 +1887,7 @@ else: assert_snapshot!( test.import_from("foo", "MAGIC"), @r#" from foo import MAGIC + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1306,6 +1919,7 @@ else: assert_snapshot!( test.import("bar", "MAGIC"), @r#" import bar + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1316,6 +1930,7 @@ else: assert_snapshot!( test.import_from("bar", "MAGIC"), @r#" import bar + if os.getenv("WHATEVER"): from foo import MAGIC else: @@ -1355,6 +1970,7 @@ else: assert_snapshot!( test.import("foo", "MAGIC"), @r#" import foo + if os.getenv("WHATEVER"): import foo as fubar else: @@ -1365,6 +1981,7 @@ else: assert_snapshot!( test.import_from("foo", "MAGIC"), @r#" from foo import MAGIC + if os.getenv("WHATEVER"): import foo as fubar else: @@ -1396,6 +2013,7 @@ else: assert_snapshot!( test.import("bar", "MAGIC"), @r#" import bar + if os.getenv("WHATEVER"): import foo as fubar else: @@ -1406,6 +2024,7 @@ else: assert_snapshot!( test.import_from("bar", "MAGIC"), @r#" from bar import MAGIC + if os.getenv("WHATEVER"): import foo as fubar else: @@ -1437,6 +2056,7 @@ except ImportError: assert_snapshot!( test.import("quux", "MAGIC"), @" import quux + try: from foo import MAGIC except ImportError: @@ -1447,6 +2067,7 @@ except ImportError: assert_snapshot!( test.import_from("quux", "MAGIC"), @" import quux + try: from foo import MAGIC except ImportError: @@ -1482,6 +2103,7 @@ except ImportError: assert_snapshot!( test.import("foo", "MAGIC"), @" import foo + try: from foo import MAGIC except ImportError: @@ -1492,6 +2114,7 @@ except ImportError: assert_snapshot!( test.import_from("foo", "MAGIC"), @" from foo import MAGIC + try: from foo import MAGIC except ImportError: @@ -1523,6 +2146,7 @@ except ImportError: assert_snapshot!( test.import("bar", "MAGIC"), @" import bar + try: from foo import MAGIC except ImportError: @@ -1533,6 +2157,7 @@ except ImportError: assert_snapshot!( test.import_from("bar", "MAGIC"), @" import bar + try: from foo import MAGIC except ImportError: diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 0a9ed4a903..617331743c 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -3556,19 +3556,20 @@ mod tests { --> main.py:1:1 | 1 + from typing import Literal - 2 | - 3 | def stringify(function): + 2 + + 3 | + 4 | def stringify(function): - def wrapper(): - return str(function()) - 4 + def wrapper() -> str: - 5 + return str(object=function()) - 6 | - 7 | return wrapper - 8 | - 9 | @stringify + 5 + def wrapper() -> str: + 6 + return str(object=function()) + 7 | + 8 | return wrapper + 9 | + 10 | @stringify - def number(): - 10 + def number() -> Literal[42]: - 11 | return 42 + 11 + def number() -> Literal[42]: + 12 | return 42 | "#); @@ -4564,18 +4565,19 @@ Source with applied edits: --> main.py:1:1 | 1 + from typing import Literal - 2 | + 2 + + 3 | -------------------------------------------------------------------------------- - 6 | x = 1 + 7 | x = 1 - y = x - z = i(1) - w = z - 7 + y: Literal[1] = x - 8 + z: int = i(1) - 9 + w: int = z - 10 | aa = b'foo' + 8 + y: Literal[1] = x + 9 + z: int = i(1) + 10 + w: int = z + 11 | aa = b'foo' - bb = aa - 11 + bb: Literal[b"foo"] = aa + 12 + bb: Literal[b"foo"] = aa | "#); } @@ -4815,10 +4817,11 @@ Source with applied edits: --> main.py:1:1 | 1 + from typing import Literal - 2 | - 3 | def f(n: int): + 2 + + 3 | + 4 | def f(n: int): - pairs = (1, 'a') * n - 4 + pairs: tuple[Literal[1, "a"], ...] = (1, 'a') * n + 5 + pairs: tuple[Literal[1, "a"], ...] = (1, 'a') * n | "#); } @@ -5281,15 +5284,16 @@ Source with applied edits: --> main.py:1:1 | 1 + from typing import Literal - 2 | + 2 + + 3 | -------------------------------------------------------------------------------- - 8 | x = (1, 'abc') + 9 | x = (1, 'abc') - y = x - z = (i(1), s('abc')) - w = z - 9 + y: tuple[Literal[1], Literal["abc"]] = x - 10 + z: tuple[int, str] = (i(1), s('abc')) - 11 + w: tuple[int, str] = z + 10 + y: tuple[Literal[1], Literal["abc"]] = x + 11 + z: tuple[int, str] = (i(1), s('abc')) + 12 + w: tuple[int, str] = z | "#); } @@ -5514,17 +5518,18 @@ Source with applied edits: --------------------------------------------- info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 - | - 1 + from typing import Literal - 2 | + | + 1 + from typing import Literal + 2 + + 3 | -------------------------------------------------------------------------------- - 6 | x: int = 1 - - y = x - 7 + y: Literal[1] = x - 8 | z: int = i(1) - - w = z - 9 + w: int = z - | + 7 | x: int = 1 + - y = x + 8 + y: Literal[1] = x + 9 | z: int = i(1) + - w = z + 10 + w: int = z + | "); } @@ -5766,12 +5771,13 @@ Source with applied edits: --> main.py:1:1 | 1 + from typing import Literal - 2 | - 3 | def my_func(command: str): - 4 | match command.split(): - 5 | case ["get", ("a" | "b") as ab]: + 2 + + 3 | + 4 | def my_func(command: str): + 5 | match command.split(): + 6 | case ["get", ("a" | "b") as ab]: - x = ab - 6 + x: Literal["a", "b"] = ab + 7 + x: Literal["a", "b"] = ab | "#); } @@ -9233,11 +9239,12 @@ Source with applied edits: --> main.py:1:1 | 1 + from typing import Literal - 2 | + 2 + + 3 | -------------------------------------------------------------------------------- - 13 | x = None + 14 | x = None - y = x - 14 + y: Literal[1, 2, 3, "hello"] | None = x + 15 + y: Literal[1, 2, 3, "hello"] | None = x | "#); } diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 9e18a0cdc5..d8f5a9522d 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -887,6 +887,54 @@ mod tests { } } + impl CursorTestBuilder { + /// add a notebook at `path` whose code cells hold `cells`, one of which may hold the + /// ``, an offset of the text the cells make together + pub(super) fn notebook( + &mut self, + path: impl Into, + cells: &[&str], + ) -> &mut CursorTestBuilder { + const MARKER: &str = ""; + + let mut cursor_offset = None; + let mut before = TextSize::default(); + let mut written = Vec::new(); + for cell in cells { + let cell = match cell.find(MARKER) { + Some(at) => { + assert!( + cursor_offset.is_none(), + "more than one cell holds ``" + ); + cursor_offset = Some( + before + TextSize::try_from(at).expect("cell to be smaller than 4GB"), + ); + cell.replacen(MARKER, "", 1) + } + None => (*cell).to_string(), + }; + // each cell is followed by the line break the notebook's text joins cells with + before += TextSize::of(cell.as_str()) + TextSize::of("\n"); + written.push(format!( + r#"{{"cell_type": "code", "execution_count": null, "metadata": {{}}, "outputs": [], "source": "{}"}}"#, + cell.replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + )); + } + self.sources.push(Source { + path: path.into(), + contents: format!( + r#"{{"cells": [{}], "metadata": {{}}, "nbformat": 4, "nbformat_minor": 5}}"#, + written.join(", ") + ), + cursor_offset, + }); + self + } + } + fn add_source( sources: &mut Vec, path: impl Into, diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index 1dd84f4ae1..39e2401348 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -1884,6 +1884,8 @@ class Other(Base): assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @r#" --- /main.py from typing import override + + class Base: def describe(self) -> str: return "child" diff --git a/crates/ty_ide/src/refactor/return_annotation.rs b/crates/ty_ide/src/refactor/return_annotation.rs index 91efc9684c..b2ce6671b0 100644 --- a/crates/ty_ide/src/refactor/return_annotation.rs +++ b/crates/ty_ide/src/refactor/return_annotation.rs @@ -194,6 +194,7 @@ mod tests { --- from typing import Literal + async def f() -> Literal[1]: return 1 "); diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__type_arguments_are_not_named_outside_basedpython.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__type_arguments_are_not_named_outside_basedpython.snap index 33fe1db81d..a5c522e450 100644 --- a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__type_arguments_are_not_named_outside_basedpython.snap +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__type_arguments_are_not_named_outside_basedpython.snap @@ -91,10 +91,11 @@ info[inlay-hint-edit]: Inlay hint edits --> main.py:1:1 | 1 + from typing import Literal -2 | -3 | class Pair[Key, Value]: -4 | def __init__(self, key: Key, value: Value) -> None: ... -5 | +2 + +3 | +4 | class Pair[Key, Value]: +5 | def __init__(self, key: Key, value: Value) -> None: ... +6 | - a = Pair(1, 'x') -6 + a: Pair[Literal[1], Literal["x"]] = Pair(1, 'x') +7 + a: Pair[Literal[1], Literal["x"]] = Pair(1, 'x') | diff --git a/crates/ty_python_semantic/src/importer.rs b/crates/ty_python_semantic/src/importer.rs index 01cc458537..829dcbd200 100644 --- a/crates/ty_python_semantic/src/importer.rs +++ b/crates/ty_python_semantic/src/importer.rs @@ -24,6 +24,7 @@ use crate::{Db, SemanticModel}; use ruff_db::source::{SourceText, source_text}; use ruff_diagnostics::Edit; use ruff_python_ast as ast; +use ruff_python_ast::helpers::format_import_from; use ruff_python_ast::name::Name; use ruff_python_ast::token::Tokens; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal, walk_stmt}; @@ -53,9 +54,9 @@ pub struct Importer<'a> { stylist: &'a Stylist<'static>, /// The list of visited, top-level runtime imports in the Python AST. imports: &'a [AstNodeRef], - /// The order an import sorter puts this file's imports in, for imports to be added where it + /// the order an import sorter puts this file's imports in, for imports to be added where it /// would put them. `None` adds a statement at the top of the file and a member after an - /// import's last. + /// import's last sorting: Option>, } @@ -82,11 +83,11 @@ impl<'a> Importer<'a> { } } - /// This importer, adding each import where `order` sorts it: a statement in its section of - /// the file's import block and a member among the members of a `from` import. + /// this importer, adding each import where `order` sorts it: a statement in its section of + /// the file's import block and a member among the members of a `from` import /// /// `resolve` is called the first time an import is placed, as finding the order can be - /// costly; when it returns `None` the imports are added as without an order. + /// costly; when it returns `None` the imports are added as without an order #[must_use] pub fn sorted_by(mut self, resolve: F) -> Self where @@ -96,7 +97,7 @@ impl<'a> Importer<'a> { self.sorting = Some(Box::new(LazyOrder { resolve, order: OnceCell::new(), - block: OnceCell::new(), + blocks: OnceCell::new(), })); self } @@ -171,7 +172,7 @@ impl<'a> Importer<'a> { let placement = Placement::statement(&request, members.at); let import = self .placed_edits(std::slice::from_ref(&placement)) - .and_then(|mut edits| edits.pop()) + .pop() .unwrap_or_else(|| { self.statement_insertion(members.at) .into_edit(&request.to_string()) @@ -246,7 +247,7 @@ impl<'a> Importer<'a> { }; let import = self .placed_edits(std::slice::from_ref(&placement)) - .and_then(|mut edits| edits.pop()) + .pop() .unwrap_or_else(|| { let imported = request.imported_member(); if let Some(insertion) = @@ -463,14 +464,15 @@ impl<'a> Importer<'a> { None } - /// The edits that add everything `actions` import at once. + /// the edits that add everything `actions` import at once /// - /// Each action's own edit is computed as if it were the only import added, so two of them + /// each action's own edit is computed as if it were the only import added, so two of them /// can land at one place in an order the file's import sorter wouldn't write, or both - /// separate themselves from the imports around them. These are placed together instead: + /// separate themselves from the imports around them. these are placed together instead: /// statements of one section are sorted among each other and separated as the sorter /// separates them, two members added to one `from` import are sorted, and two members added - /// from a module the file doesn't import yet are listed by one new `from` import. + /// from a module the file doesn't import yet are listed by one new `from` import, with or + /// without a sorter's order to follow. pub fn combined_edits<'b>( &self, actions: impl IntoIterator, @@ -484,72 +486,131 @@ impl<'a> Importer<'a> { (_, None) => {} } } - if let Some(placed) = self.placed_edits(&placements) { - edits.extend(placed); - } else { - edits.extend( - placements - .iter() - .filter_map(|placement| self.unsorted_edit(placement)), - ); - } + edits.extend(self.placed_edits(&placements)); edits } - /// Where a new statement goes without a sorter's order: after the last `from __future__` - /// import before `at`, or otherwise at the top of the file (or of `at`'s notebook cell). + /// where a new statement goes without a sorter's order: after the last `from __future__` + /// import before `at`, or otherwise at the top of the file (or of `at`'s notebook cell) fn statement_insertion(&self, at: TextSize) -> Insertion<'static> { if let Some(future) = self.find_last_future_import(at) { Insertion::end_of_statement(future.stmt, &self.source, self.stylist) } else { - let range = self - .source - .as_notebook() - .and_then(|notebook| notebook.cell_offsets().containing_range(at)); + let range = self.cell_at(at).map(|(_, range)| range); Insertion::start_of_file(self.parsed.suite(), &self.source, self.stylist, range) } } - /// The edit that adds `placement` without a sorter's order: a statement at the top of the - /// file, a member after an import's last. - fn unsorted_edit(&self, placement: &Placement) -> Option { - match placement { - Placement::Statement { at, .. } => Some( - self.statement_insertion(*at) - .into_edit(&placement.statement_text()), - ), - Placement::Member { import, .. } => { - let stmt = self - .imports() - .find(|candidate| candidate.stmt.start() == *import)?; - Insertion::existing_import(stmt.stmt, self.tokens) - .map(|insertion| insertion.into_edit(&placement.imported_member())) - } + /// the notebook cell an import for a use at `at` is added to, the one `at` is in, as its + /// index among the notebook's cells and its range. `None` for a file that is not a notebook, + /// whose imports go in the file as a whole + fn cell_at(&self, at: TextSize) -> Option<(usize, TextRange)> { + let offsets = self.source.as_notebook()?.cell_offsets(); + // the end of the last cell is the end of the notebook, which a use at the end of it is + // still in + offsets + .array_windows::<2>() + .enumerate() + .find(|(_, [start, end])| { + *start <= at && (at < *end || *end == TextSize::of(self.source.as_str())) + }) + .map(|(index, [start, end])| (index, TextRange::new(*start, *end))) + } + + /// the top-level statements of the cell an import for a use at `at` is added to, or of the + /// whole file when it is not a notebook, as a range of the module's body. `None` when `at` + /// is in no cell of a notebook + fn part_at(&self, at: TextSize) -> Option> { + let body = self.parsed.suite(); + if self.source.as_notebook().is_none() { + return Some(0..body.len()); } + let (_, cell) = self.cell_at(at)?; + let start = body.partition_point(|stmt| stmt.start() < cell.start()); + let end = body.partition_point(|stmt| stmt.start() < cell.end()); + Some(start..end) } - /// The edits that make the additions in `placements` where the file's import sorter would - /// make them, or `None` when the importer has no order to follow, or a statement has no - /// import block to join, for the caller to add them without one. - fn placed_edits(&self, placements: &[Placement]) -> Option> { - self.sorting.as_ref()?.placed_edits(self, placements) + /// The edits that make the additions in `placements`: where the file's import sorter would + /// make them, or without an order to follow, statements at the top of the file and members + /// after an import's last. + fn placed_edits(&self, placements: &[Placement]) -> Vec { + if let Some(sorting) = &self.sorting + && let Some(edits) = sorting.placed_edits(self, placements) + { + return edits; + } + let (statements, members) = combined(placements, &Unordered); + let mut edits = Vec::new(); + if let Some(at) = statements.iter().map(|statement| statement.at).min() { + let insertion = self.statement_insertion(at); + let separator = if insertion.is_inline() { + "; " + } else { + self.stylist.line_ending().as_str() + }; + let text = statements + .iter() + .map(NewStatement::text) + .collect::>() + .join(separator); + edits.push(insertion.into_edit(&text)); + } + for (import, added) in members { + let Some(stmt) = self + .imports() + .find(|candidate| candidate.stmt.start() == import) + else { + continue; + }; + let imported = added + .iter() + .map(|member| imported_member(*member)) + .collect::>() + .join(", "); + edits.push( + if let Some(insertion) = Insertion::existing_import(stmt.stmt, self.tokens) { + insertion.into_edit(&imported) + } else { + let from = match stmt.kind { + AstImportKind::ImportFrom(from) => { + format_import_from(from.level, from.module.as_deref()) + } + AstImportKind::Import(_) => continue, + }; + Insertion::end_of_statement(stmt.stmt, &self.source, self.stylist) + .into_edit(&format!("from {from} import {imported}")) + }, + ); + } + edits } - /// The imports the sorter sorts as one block that new imports join, the run of consecutive - /// import statements the module's body starts its imports with, and where `order` sorts - /// each of them, worked out once into `block`. `None` when it starts at or after `at`, - /// where the imports have to be available. + /// the imports the sorter sorts as one block that new imports join, the run of consecutive + /// import statements the module's body starts its imports with, or in a notebook the body + /// of `at`'s cell, which the sorter sorts on its own, and where `order` sorts each of them, + /// worked out once into `blocks`. `None` when it starts at or after `at`, where the imports + /// have to be available. fn import_block<'b, O: ImportOrdering>( &self, order: &O, - block: &'b ImportBlock, + blocks: &'b ImportBlocks, at: TextSize, ) -> Option<(&'a [ast::Stmt], &'b [O::Position])> { let body: &'a [ast::Stmt] = self.parsed.suite(); - let (range, positions) = block + let cell = self.cell_at(at).map_or(0, |(index, _)| index); + let part = self.part_at(at)?; + let (range, positions) = blocks + .get_or_init(|| { + let parts = self.source.as_notebook().map_or(1, |notebook| { + notebook.cell_offsets().len().saturating_sub(1) + }); + (0..parts).map(|_| OnceCell::new()).collect() + }) + .get(cell)? .get_or_init(|| { - let start = body.iter().position(is_import)?; - let len = body[start..] + let start = part.start + body[part.clone()].iter().position(is_import)?; + let len = body[start..part.end] .iter() .take_while(|stmt| is_import(stmt)) .count(); @@ -564,93 +625,19 @@ impl<'a> Importer<'a> { (statements.first()?.start() < at).then_some((statements, positions.as_slice())) } - /// The edits that make the additions in `placements` where `order` puts them. + /// the edits that make the additions in `placements` where `order` puts them fn placed_edits_in( &self, order: &O, - block: &ImportBlock, + blocks: &ImportBlocks, placements: &[Placement], ) -> Option> { let mut edits = Vec::new(); - - // Statements: two `from` imports of one module, as a member the file imports nothing - // from yet makes, are one statement listing both - let mut statements: Vec = Vec::new(); - for placement in placements { - let Placement::Statement { - module, - level, - member, - asname, - at, - } = placement - else { - continue; - }; - let joins = member.is_some() - && !order.imports_member_alone(module.as_deref(), asname.as_deref()); - if joins - && let Some(existing) = statements.iter_mut().find(|existing| { - existing.module == module.as_deref() - && existing.level == *level - && !existing.members.is_empty() - && !order.imports_member_alone(existing.module, existing.members[0].1) - }) - { - if let Some(member) = member { - existing.members.push((member.as_str(), asname.as_deref())); - } - existing.at = existing.at.min(*at); - continue; - } - statements.push(NewStatement { - module: module.as_deref(), - level: *level, - members: member - .iter() - .map(|member| (member.as_str(), asname.as_deref())) - .collect(), - asname: if member.is_some() { - None - } else { - asname.as_deref() - }, - at: *at, - }); - } - for statement in &mut statements { - statement - .members - .sort_by(|a, b| order.compare_members(*a, *b)); - statement.members.dedup(); - } - if !statements.is_empty() { - let at = statements.iter().map(|statement| statement.at).min()?; - edits.extend(self.placed_statements(order, block, &statements, at)?); + let (statements, members) = combined(placements, order); + if let Some(at) = statements.iter().map(|statement| statement.at).min() { + edits.extend(self.placed_statements(order, blocks, &statements, at)); } - - // Members added to `from` imports the file has - let mut by_import: Vec<(TextSize, Vec)> = Vec::new(); - for placement in placements { - if let Placement::Member { - import, - member, - asname, - } = placement - { - let entry = match by_import.iter_mut().find(|(start, _)| start == import) { - Some(entry) => entry, - None => { - by_import.push((*import, Vec::new())); - by_import.last_mut()? - } - }; - entry.1.push((member.as_str(), asname.as_deref())); - } - } - for (import, mut added) in by_import { - added.sort_by(|a, b| order.compare_members(*a, *b)); - added.dedup(); + for (import, added) in members { let AstImportKind::ImportFrom(from) = self .imports() .find(|candidate| candidate.stmt.start() == import)? @@ -663,23 +650,90 @@ impl<'a> Importer<'a> { Some(edits) } - /// The edits adding `statements` to the file's import block, each in its section, sorted + /// `statement` as `order`'s sorter writes it, wrapped to its line length. + fn written(&self, order: &O, statement: &NewStatement) -> String { + let line_ending = self.stylist.line_ending().as_str(); + let text = statement.text(); + order + .write_statements(&format!("{text}{line_ending}"), self.stylist) + .map(|written| { + written + .strip_suffix(line_ending) + .unwrap_or(&written) + .to_string() + }) + .unwrap_or(text) + } + + /// the edits adding `statements` to the file's import block, each in its section, sorted /// among the section's imports and each other, with the blank lines and section headings - /// the sorter writes around them. + /// the sorter writes around them. Without an import block before `at`, the statements are + /// one of their own, where a statement goes without an order. fn placed_statements( &self, order: &O, - block: &ImportBlock, + blocks: &ImportBlocks, statements: &[NewStatement], at: TextSize, - ) -> Option> { - let (block, positions) = self.import_block(order, block, at)?; + ) -> Vec { + let source: &str = &self.source; + let line_ending = self.stylist.line_ending().as_str(); + // in a notebook, the imports are sorted, and added, within `at`'s cell + let cell = self.cell_at(at).map(|(_, cell)| cell); + + let Some((block, positions)) = self.import_block(order, blocks, at) else { + let insertion = self.statement_insertion(at); + let insertion_is_inline = insertion.is_inline(); + let text = if insertion_is_inline { + statements + .iter() + .map(NewStatement::text) + .collect::>() + .join("; ") + } else { + let mut added: Vec<_> = statements + .iter() + .map(|statement| (order.position(&statement.sorted_import()), statement)) + .collect(); + added.sort_by(|a, b| a.0.cmp(&b.0)); + let text = self.separated(order, None, &added, None); + text.strip_suffix(line_ending).unwrap_or(&text).to_string() + }; + let mut edit = insertion.into_edit(&text); + // a new block is set apart from the statement after it as the sorter sets it apart, + // unless that is an import the block runs on into + if !insertion_is_inline + && let Some(next) = self + .parsed + .suite() + .iter() + .find(|stmt| stmt.start() >= edit.start()) + && cell.is_none_or(|cell| cell.contains(next.start())) + && !is_import(next) + { + let before_definition = + matches!(next, ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_)); + let existing = source[TextRange::new(edit.start(), next.start())] + .split_inclusive('\n') + .take_while(|line| line.trim().is_empty()) + .count(); + let missing = order + .blank_lines_after_block(before_definition) + .saturating_sub(existing); + if missing > 0 { + let mut text = edit.content().unwrap_or_default().to_string(); + text.push_str(&line_ending.repeat(missing)); + edit = Edit::insertion(text, edit.start()); + } + } + return vec![edit]; + }; - // The sorter's order is total, so in a sorted block a new statement goes before the - // first import that sorts after it. In a block that isn't sorted, that is still a place - // among the imports it sorts with. Statements that go before the same import are sorted - // among each other. - let mut slots: Vec> = Vec::new(); + // the sorter's order is total, so in a sorted block a new statement goes before the + // first import that sorts after it. in a block that isn't sorted, that is still a place + // among the imports it sorts with. statements that go before the same import are sorted + // among each other + let mut slots: Vec> = Vec::new(); for statement in statements { let position = order.position(&statement.sorted_import()); let after = positions.iter().position(|existing| *existing > position); @@ -687,126 +741,142 @@ impl<'a> Importer<'a> { Some(slot) => slot, None => { slots.push((after, Vec::new())); - slots.last_mut()? + let Some(slot) = slots.last_mut() else { + continue; + }; + slot } }; - slot.1.push((position, statement.text())); + slot.1.push((position, statement)); } - let source: &str = &self.source; - let line_ending = self.stylist.line_ending().as_str(); let mut edits = Vec::new(); for (after, mut added) in slots { added.sort_by(|a, b| a.0.cmp(&b.0)); let before = match after { Some(0) => None, Some(after) => Some(after - 1), - None => Some(positions.len() - 1), + None => positions.len().checked_sub(1), }; - // The comments directly above an import are the sorter's to move with it, except - // the heading of its section, so statements placed before an import go above them. - let floor = before.map_or_else( - || source.line_start(block[0].start()), + // Between two imports of one line, as in `import os; import sys`, the statements + // can only be written on that line + if let (Some(before), Some(after)) = (before, after) + && !source + .contains_line_break(TextRange::new(block[before].end(), block[after].start())) + { + let mut text = String::new(); + for (_, statement) in &added { + text.push_str(&statement.text()); + text.push_str("; "); + } + edits.push(Edit::insertion(text, block[after].start())); + continue; + } + + // The sorter takes the headings directly above the block's first import into the + // block and writes them again, and moves the other comments directly above an import + // with it. So the lines between the imports the statements go between, which are + // regenerated with them, run from the line after the import before them to the + // headings and blank lines above the import after them. + let start = before.map_or_else( + || { + let mut start = source.line_start(block[0].start()); + let top = cell.map_or(TextSize::default(), TextRange::start); + while start > top { + let previous = source.line_start(start - TextSize::from(1)); + if !order.is_heading(source[TextRange::new(previous, start)].trim()) { + break; + } + start = previous; + } + start + }, |before| source.full_line_end(block[before].end()), ); - let anchor = after.map(|after| { - let section_heading = order.section_heading(&positions[after]); - let mut anchor = source.line_start(block[after].start()); - while anchor > floor { - let previous = source.line_start(anchor - TextSize::from(1)); - let line = source[TextRange::new(previous, anchor)].trim(); - if !line.starts_with('#') || Some(line) == section_heading { - break; + let end = after.map_or(start, |after| { + let mut end = source.line_start(block[after].start()); + if after > 0 { + while end > start { + let previous = source.line_start(end - TextSize::from(1)); + let line = source[TextRange::new(previous, end)].trim(); + if !line.starts_with('#') || order.is_heading(line) { + break; + } + end = previous; } - anchor = previous; } - anchor - }); - let existing_blank_lines = anchor.map_or(0, |anchor| { - source[TextRange::new(floor, anchor)] - .lines() - .filter(|line| line.trim().is_empty()) - .count() + end }); - - // The added statements and the blank lines and headings between them and the - // imports they go between - let mut text = String::new(); - let mut previous = before.map(|before| &positions[before]); - for (position, statement) in &added { - let blank_lines = - previous.map_or(0, |previous| order.blank_lines_between(previous, position)); - for _ in 0..blank_lines { + let between = &source[TextRange::new(start, end)]; + + let previous = before.map(|before| &positions[before]); + let next = after.map(|after| &positions[after]); + let regenerated = between + .lines() + .all(|line| line.trim().is_empty() || order.is_heading(line.trim())); + let edit = if regenerated { + let text = self.separated(order, previous, &added, next); + minimal_edit(between, &text, start) + } else { + // a comment between the imports is left where it is, and the statements go + // right after the import before them + let mut text = String::new(); + if !source[..start.to_usize()].ends_with(['\n', '\r']) { text.push_str(line_ending); } - if previous - .is_none_or(|previous| order.section(previous) != order.section(position)) - && after.is_none_or(|after| { - order.section(&positions[after]) != order.section(position) - }) - && let Some(heading) = order.section_heading(position) - { - text.push_str(heading); + text.push_str(&self.separated(order, previous, &added, None)); + Some(Edit::insertion(text, start)) + }; + edits.extend(edit); + } + edits + } + + /// `added`, sorted, written as the sorter writes them after the import at `previous` and + /// before the import at `next`: each preceded by the blank lines the sorter separates it + /// from the statement before with, and by its section's heading when it starts the section, + /// and followed by the blank lines and heading the sorter writes above `next`. + fn separated<'p, O: ImportOrdering>( + &self, + order: &O, + mut previous: Option<&'p O::Position>, + added: &'p [(O::Position, &NewStatement)], + next: Option<&O::Position>, + ) -> String { + let line_ending = self.stylist.line_ending().as_str(); + let mut text = String::new(); + let write_before = |text: &mut String, + previous: Option<&O::Position>, + position: &O::Position| { + if let Some(previous) = previous { + for _ in 0..order.blank_lines_between(previous, position) { text.push_str(line_ending); } - text.push_str(statement); + } + if previous.is_none_or(|previous| order.section(previous) != order.section(position)) + && let Some(heading) = order.section_heading(position) + { + text.push_str(heading); text.push_str(line_ending); - previous = Some(position); } - let first = &added.first()?.0; - let last = &added.last()?.0; - let blank_before = before.map_or(0, |before| { - order.blank_lines_between(&positions[before], first) - }); - let blank_after = after.map_or(0, |after| { - order.blank_lines_between(last, &positions[after]) - }); - - let (offset, text) = match (before, anchor) { - // Joins the imports after them: placed above those, as far below the imports - // before them as the sorter separates the two, which the lines already there - // are. - (_, Some(anchor)) if blank_after == 0 && (before.is_none() || blank_before > 0) => { - let text = text - .strip_prefix(&line_ending.repeat(blank_before)) - .unwrap_or(&text); - let mut padded = - line_ending.repeat(blank_before.saturating_sub(existing_blank_lines)); - padded.push_str(text); - (anchor, padded) - } - // Placed below the imports before them, as far above the imports after them as - // the sorter separates the two. - (Some(before), _) => { - let offset = source.full_line_end(block[before].end()); - let mut padded = String::new(); - if !source[..offset.to_usize()].ends_with(['\n', '\r']) { - padded.push_str(line_ending); - } - padded.push_str(&text); - if anchor.is_some() { - padded.push_str( - &line_ending.repeat(blank_after.saturating_sub(existing_blank_lines)), - ); - } - (offset, padded) - } - // First in the block, in a section of their own. - (None, Some(anchor)) => { - let mut padded = text; - padded.push_str(&line_ending.repeat(blank_after)); - (anchor, padded) - } - (None, None) => return None, - }; - edits.push(Edit::insertion(text, offset)); + }; + for (position, statement) in added { + write_before(&mut text, previous, position); + text.push_str(&self.written(order, statement)); + text.push_str(line_ending); + previous = Some(position); } - Some(edits) + if let Some(next) = next { + write_before(&mut text, previous, next); + } + text } - /// The edits listing `added`, sorted, among the members of the existing `from` import - /// `from`, each where `order` sorts it. + /// the edits listing `added`, sorted, among the members of the existing `from` import + /// `from`, each where `order` sorts it. When the import was written as the sorter writes it + /// and the sorter would write it differently with the members added, as it wraps an import + /// that no longer fits on its line, the import is written again as the sorter writes it. fn placed_members( &self, order: &O, @@ -819,15 +889,12 @@ impl<'a> Importer<'a> { let mut edits = Vec::new(); let mut appended = Vec::new(); let mut slots: Vec<(usize, Vec)> = Vec::new(); - for (member, asname) in added { - let imported = match asname { - Some(asname) => format!("{member} as {asname}"), - None => (*member).to_string(), - }; + for member in added { + let imported = imported_member(*member); let next = names.iter().position(|alias| { order .compare_members( - (member, *asname), + *member, ( alias.name.as_str(), alias.asname.as_ref().map(ast::Identifier::as_str), @@ -844,8 +911,8 @@ impl<'a> Importer<'a> { } } for (next, members) in slots { - // Written with the separator the import already puts between its members, so a - // member lands on a line of its own in an import that writes one per line. + // written with the separator the import already puts between its members, so a + // member lands on a line of its own in an import that writes one per line let separator = if next + 1 < names.len() { TextRange::new(names[next].end(), names[next + 1].start()) } else if next > 0 { @@ -880,6 +947,31 @@ impl<'a> Importer<'a> { Insertion::existing_import(stmt.stmt, self.tokens)?.into_edit(&appended.join(", ")), ); } + + let written = |statement: &str| { + order + .write_statements(&format!("{statement}{line_ending}"), self.stylist) + .map(|written| { + written + .strip_suffix(line_ending) + .unwrap_or(&written) + .to_string() + }) + }; + let original = &source[from.range()]; + let mut edited = original.to_string(); + let mut sorted_edits: Vec<&Edit> = edits.iter().collect(); + sorted_edits.sort_by_key(|edit| std::cmp::Reverse(edit.start())); + for edit in sorted_edits { + let range = edit.range() - from.start(); + edited.replace_range(range.to_std_range(), edit.content().unwrap_or_default()); + } + if written(original).as_deref() == Some(original) + && let Some(rewritten) = written(&edited) + && rewritten != edited + { + return Some(vec![Edit::range_replacement(rewritten, from.range())]); + } Some(edits) } @@ -986,27 +1078,31 @@ impl<'a> Importer<'a> { } } -/// The import block new imports join, as the range of the module's body it spans, and where an +/// the import block new imports join, as the range of the module's body it spans, and where an /// order sorts each of its statements. `None` when the module has no import block, or one with -/// a statement the order can't read. +/// a statement the order can't read type ImportBlock = OnceCell, Vec<::Position>)>>; -/// An importer's sorting, with the type of its order erased. +/// the import block of each part of the file sorted on its own: each cell of a notebook, or the +/// file as a whole +type ImportBlocks = OnceCell]>>; + +/// an importer's sorting, with the type of its order erased trait Sorting { - /// The edits that make the additions in `placements` where the order puts them, or `None` - /// when there is no order or the additions can't follow it. + /// the edits that make the additions in `placements` where the order puts them, or `None` + /// when there is no order or the additions can't follow it fn placed_edits(&self, importer: &Importer<'_>, placements: &[Placement]) -> Option>; - /// Whether the order writes `member` (imported `as asname`, if given) from `module` in a - /// `from` import of its own. + /// whether the order writes `member` (imported `as asname`, if given) from `module` in a + /// `from` import of its own fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; } -/// An order found the first time an import is placed, and the import block as it sorts it. +/// an order found the first time an import is placed, and the import block as it sorts it struct LazyOrder { resolve: F, order: OnceCell>, - block: ImportBlock, + blocks: ImportBlocks, } impl Option> LazyOrder { @@ -1017,7 +1113,7 @@ impl Option> LazyOrder { impl Option> Sorting for LazyOrder { fn placed_edits(&self, importer: &Importer<'_>, placements: &[Placement]) -> Option> { - importer.placed_edits_in(self.order()?, &self.block, placements) + importer.placed_edits_in(self.order()?, &self.blocks, placements) } fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { @@ -1030,8 +1126,8 @@ fn is_import(stmt: &ast::Stmt) -> bool { matches!(stmt, ast::Stmt::Import(_) | ast::Stmt::ImportFrom(_)) } -/// An import statement as a sorter reads it to sort it: an `import` by its first module, a -/// `from` import by its module and the member `order` lists first. +/// an import statement as a sorter reads it to sort it: an `import` by its first module, a +/// `from` import by its module and the member `order` lists first fn sorted_import<'a>(stmt: &'a ast::Stmt, order: &impl ImportOrdering) -> Option> { match stmt { ast::Stmt::Import(import) => { @@ -1211,25 +1307,25 @@ enum MemberImportKind<'ast> { pub struct ImportAction { import: Option, symbol_text: Box, - /// What `import` adds, for [`Importer::combined_edits`] to place it together with what - /// other actions add. + /// what `import` adds, for [`Importer::combined_edits`] to place it together with what + /// other actions add placement: Option, } -/// What an import action adds to the file. +/// what an import action adds to the file #[derive(Debug, Clone)] enum Placement { - /// A new import statement: `import module as asname`, or `from module import member as - /// asname`, `level` dots up. + /// a new import statement: `import module as asname`, or `from module import member as + /// asname`, `level` dots up Statement { module: Option, level: u32, member: Option, asname: Option, - /// Where what is imported has to be available. + /// where what is imported has to be available at: TextSize, }, - /// A member listed by the existing `from` import starting at `import`. + /// a member listed by the existing `from` import starting at `import` Member { import: TextSize, member: String, @@ -1238,7 +1334,7 @@ enum Placement { } impl Placement { - /// The statement `request` adds when the file has no import that satisfies it. + /// the statement `request` adds when the file has no import that satisfies it fn statement(request: &ImportRequest<'_>, at: TextSize) -> Self { let member = match (&request.style, request.member) { (ImportStyle::ImportFrom, Some(member)) => Some(member.to_string()), @@ -1252,62 +1348,168 @@ impl Placement { at, } } +} - /// The statement a [`Placement::Statement`] adds. - fn statement_text(&self) -> String { - match self { +/// A member of a `from` import, and the name it is imported as. +type Member<'a> = (&'a str, Option<&'a str>); + +/// The statements added before the import of the block at the index, or after the block's last +/// import for `None`, each with where it sorts. +type StatementSlot<'s, P> = (Option, Vec<(P, &'s NewStatement<'s>)>); + +/// `member`, or `member as asname`, as a `from` import lists it +fn imported_member((member, asname): Member) -> String { + match asname { + Some(asname) => format!("{member} as {asname}"), + None => member.to_string(), + } +} + +/// The edit that changes `old`, the text at `start`, to `new`, touching only what differs. +/// `None` when nothing does. +fn minimal_edit(old: &str, new: &str, start: TextSize) -> Option { + let prefix = old + .char_indices() + .zip(new.chars()) + .find(|((_, a), b)| a != b) + .map_or(old.len().min(new.len()), |((index, _), _)| index); + let (old_rest, new_rest) = (&old[prefix..], &new[prefix..]); + let suffix = old_rest + .chars() + .rev() + .zip(new_rest.chars().rev()) + .take_while(|(a, b)| a == b) + .map(|(a, _)| a.len_utf8()) + .sum::(); + let old_changed = &old_rest[..old_rest.len() - suffix]; + let new_changed = &new_rest[..new_rest.len() - suffix]; + if old_changed.is_empty() && new_changed.is_empty() { + return None; + } + let range = TextRange::at( + start + TextSize::of(&old[..prefix]), + TextSize::of(old_changed), + ); + Some(if new_changed.is_empty() { + Edit::range_deletion(range) + } else { + Edit::range_replacement(new_changed.to_string(), range) + }) +} + +/// The statements and members `placements` add, combined: two members a sorter lists together +/// that a module the file doesn't import yet adds are one statement listing both, and the +/// members added to one existing `from` import, by its start, are listed together. Each +/// statement's and import's members are sorted by `compare_members`, and `imports_member_alone` +/// says which member the sorter writes in a `from` import of its own. +fn combined<'p>( + placements: &'p [Placement], + order: &impl MemberOrder, +) -> (Vec>, Vec<(TextSize, Vec>)>) { + let mut statements: Vec = Vec::new(); + let mut members: Vec<(TextSize, Vec)> = Vec::new(); + for placement in placements { + match placement { Placement::Statement { module, level, member, asname, - .. - } => NewStatement { - module: module.as_deref(), - level: *level, - members: member - .iter() - .map(|member| (member.as_str(), asname.as_deref())) - .collect(), - asname: if member.is_some() { - None - } else { - asname.as_deref() - }, - at: TextSize::default(), + at, + } => { + let joins = member.is_some() + && !order.imports_member_alone(module.as_deref(), asname.as_deref()); + if joins + && let Some(member) = member + && let Some(existing) = statements.iter_mut().find(|existing| { + existing.module == module.as_deref() + && existing.level == *level + && existing.members.first().is_some_and(|first| { + !order.imports_member_alone(existing.module, first.1) + }) + }) + { + existing.members.push((member.as_str(), asname.as_deref())); + existing.at = existing.at.min(*at); + continue; + } + statements.push(NewStatement { + module: module.as_deref(), + level: *level, + members: member + .iter() + .map(|member| (member.as_str(), asname.as_deref())) + .collect(), + asname: if member.is_some() { + None + } else { + asname.as_deref() + }, + at: *at, + }); } - .text(), - Placement::Member { .. } => String::new(), - } - } - - /// The member a [`Placement::Member`] lists: `member`, or `member as asname`. - fn imported_member(&self) -> String { - match self { Placement::Member { + import, member, - asname: Some(asname), - .. - } => format!("{member} as {asname}"), - Placement::Member { member, .. } => member.clone(), - Placement::Statement { .. } => String::new(), + asname, + } => { + let added = (member.as_str(), asname.as_deref()); + match members.iter_mut().find(|(start, _)| start == import) { + Some((_, listed)) => listed.push(added), + None => members.push((*import, vec![added])), + } + } } } + for statement in &mut statements { + statement + .members + .sort_by(|a, b| order.compare_members(*a, *b)); + statement.members.dedup(); + } + for (_, added) in &mut members { + added.sort_by(|a, b| order.compare_members(*a, *b)); + added.dedup(); + } + (statements, members) } -/// A member of a `from` import, and the name it is imported as. -type Member<'a> = (&'a str, Option<&'a str>); +/// How a sorter lists the members of a `from` import, as [`combined`] needs to know it. +trait MemberOrder { + fn compare_members(&self, a: Member, b: Member) -> std::cmp::Ordering; -/// The statements added before the import of the block at the index, or after the block's last -/// import for `None`, each with where it sorts. -type StatementSlot

= (Option, Vec<(P, String)>); + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; +} + +impl MemberOrder for O { + fn compare_members(&self, a: Member, b: Member) -> std::cmp::Ordering { + ImportOrdering::compare_members(self, a, b) + } + + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + ImportOrdering::imports_member_alone(self, module, asname) + } +} + +/// No sorter: members are listed in the order they are added, all in one `from` import. +struct Unordered; + +impl MemberOrder for Unordered { + fn compare_members(&self, _: Member, _: Member) -> std::cmp::Ordering { + std::cmp::Ordering::Equal + } + + fn imports_member_alone(&self, _: Option<&str>, _: Option<&str>) -> bool { + false + } +} -/// An import statement to add: `import module as asname`, or `from module import members`. +/// an import statement to add: `import module as asname`, or `from module import members` #[derive(Debug)] struct NewStatement<'a> { module: Option<&'a str>, level: u32, - /// The members a `from` import lists, sorted; empty for an `import`. + /// the members a `from` import lists, sorted; empty for an `import` members: Vec>, asname: Option<&'a str>, at: TextSize, diff --git a/crates/ty_server/tests/e2e/code_actions.rs b/crates/ty_server/tests/e2e/code_actions.rs index 4acefc1b9f..26774dabbb 100644 --- a/crates/ty_server/tests/e2e/code_actions.rs +++ b/crates/ty_server/tests/e2e/code_actions.rs @@ -513,7 +513,7 @@ def f(user: User | None): Ok(()) } -/// The text of `content` with the edits of the code action titled `title` applied. +/// the text of `content` with the edits of the code action titled `title` applied fn applied(actions: &[lsp_types::CodeActionResponse], title: &str, content: &str) -> String { let action = actions .iter() @@ -547,10 +547,10 @@ fn applied(actions: &[lsp_types::CodeActionResponse], title: &str, content: &str lines.join("\n") } -/// An import a quick fix adds goes where buff's isort configuration sorts it: here, with +/// an import a quick fix adds goes where buff's isort configuration sorts it: here, with /// `force-sort-within-sections`, among the standard library's imports by module whatever their /// kind, and after the project's other import of its own package, which buff finds is -/// first-party because the package is in the project. +/// first-party because the package is in the project #[test] fn an_import_is_added_where_the_project_s_isort_configuration_sorts_it() -> Result<()> { let main = SystemPath::new("main.py"); diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 3e0de7cde4..18d2302afc 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -57,7 +57,7 @@ walktr "character": 0 } }, - "newText": "from inspect import walktree\n" + "newText": "from inspect import walktree\n\n" } ] } @@ -873,7 +873,7 @@ TypedDi "character": 0 } }, - "newText": "from typing import TypedDict\n" + "newText": "from typing import TypedDict\n\n" } ] }, @@ -907,7 +907,7 @@ TypedDi "character": 0 } }, - "newText": "from typing import is_typeddict\n" + "newText": "from typing import is_typeddict\n\n" } ] }, @@ -941,7 +941,7 @@ TypedDi "character": 0 } }, - "newText": "from typing_extensions import TypedDict\n" + "newText": "from typing_extensions import TypedDict\n\n" } ] }, @@ -975,7 +975,7 @@ TypedDi "character": 0 } }, - "newText": "from _typeshed._type_checker_internals import TypedDictFallback\n" + "newText": "from _typeshed._type_checker_internals import TypedDictFallback\n\n" } ] }, @@ -1009,7 +1009,7 @@ TypedDi "character": 0 } }, - "newText": "from typing_extensions import is_typeddict\n" + "newText": "from typing_extensions import is_typeddict\n\n" } ] }, @@ -1043,7 +1043,7 @@ TypedDi "character": 0 } }, - "newText": "from logging.config import _FilterConfigurationTypedDict\n" + "newText": "from logging.config import _FilterConfigurationTypedDict\n\n" } ] }, @@ -1077,7 +1077,7 @@ TypedDi "character": 0 } }, - "newText": "from logging.config import _FormatterConfigurationTypedDict\n" + "newText": "from logging.config import _FormatterConfigurationTypedDict\n\n" } ] }, @@ -1111,7 +1111,7 @@ TypedDi "character": 0 } }, - "newText": "import _typeshed.dbapi\n" + "newText": "import _typeshed.dbapi\n\n" } ] } diff --git a/crates/ty_server/tests/e2e/pasted_imports.rs b/crates/ty_server/tests/e2e/pasted_imports.rs index 8ee756e194..88b50361ea 100644 --- a/crates/ty_server/tests/e2e/pasted_imports.rs +++ b/crates/ty_server/tests/e2e/pasted_imports.rs @@ -50,6 +50,8 @@ def area(r: float) -> float: const MAIN: &str = "\ from pathlib import PurePath +import attr + def f() -> float: return math.pi * SCALE "; @@ -94,7 +96,7 @@ fn a_copy_answers_its_imports_and_a_paste_the_ones_it_is_missing() -> Result<()> let pasted = server .send_request_await::(serde_json::json!({ "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.py")) }, - "range": { "start": { "line": 3, "character": 4 }, "end": { "line": 3, "character": 26 } }, + "range": { "start": { "line": 5, "character": 4 }, "end": { "line": 5, "character": 26 } }, "imports": copied, "textHash": text_hash(MAIN), })) @@ -109,7 +111,7 @@ fn a_copy_answers_its_imports_and_a_paste_the_ones_it_is_missing() -> Result<()> assert_eq!( added, // `math` joins the standard library's imports, above `pathlib`'s, and `shapes`, which is - // the project's own, starts a section of its own below them + // the project's own, starts a section of its own below the third-party `attr` ["import math\n", "\nfrom shapes import SCALE\n"], "{pasted:#}" ); diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap index 12bf198438..afe733274c 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap @@ -42,7 +42,7 @@ expression: code_actions "character": 0 } }, - "newText": "import typing\n" + "newText": "import typing\n\n" } ] } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap index f0aa232746..424cb61fff 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_decorator.snap @@ -42,7 +42,7 @@ expression: code_actions "character": 0 } }, - "newText": "from warnings import deprecated\n" + "newText": "from warnings import deprecated\n\n\n" } ] } @@ -87,7 +87,7 @@ expression: code_actions "character": 0 } }, - "newText": "from typing_extensions import deprecated\n" + "newText": "from typing_extensions import deprecated\n\n\n" } ] } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap index 885f683dd6..b802c1d5da 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_undefined_reference_multi.snap @@ -42,7 +42,7 @@ expression: code_actions "character": 0 } }, - "newText": "from typing import Literal\n" + "newText": "from typing import Literal\n\n" } ] } @@ -87,7 +87,7 @@ expression: code_actions "character": 0 } }, - "newText": "from typing_extensions import Literal\n" + "newText": "from typing_extensions import Literal\n\n" } ] } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap index 9ada88e52e..bd1ff0aaee 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_with_full_diagnostic_output_link.snap @@ -42,7 +42,7 @@ expression: code_actions "character": 0 } }, - "newText": "from typing import Literal\n" + "newText": "from typing import Literal\n\n" } ] } @@ -87,7 +87,7 @@ expression: code_actions "character": 0 } }, - "newText": "from typing_extensions import Literal\n" + "newText": "from typing_extensions import Literal\n\n" } ] } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 4f7778c8f2..77b3a38f1f 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -241,7 +241,9 @@ resolves it — the closest `ruff.toml`, `.ruff.toml` or `pyproject.toml` with a `no-lines-before`, `lines-between-types`, `force-single-line` and the rest apply as they do to `buff`. the import has to run before the name it binds is used, so a file whose imports all come after that point, or that has none, still gets it at -the top, below a docstring and any `from __future__` import +the top, below a docstring and any `from __future__` import. `buff` sorts each cell +of a notebook on its own, so in a notebook the block an import joins, and the top it +otherwise goes at, are those of the cell the name is used in ### names inside a string From 4916ebf8cd469707955d710cdcd23301c5310fdb Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:39:44 +1000 Subject: [PATCH 060/110] dedent a raw triple-quoted string to the plain string holding its dedented text, its backslashes and quotes escaped, instead of keeping a backslash and newline at either end where a line continuation is text, and find a literal's quotes from its flags so a ''' string holding """ is dedented too --- .../src/transforms/dedent_string.rs | 217 ++++++++++++------ crates/ruff_python_trivia/src/basedpython.rs | 85 +++++++ docs/basedpython/features/dedent-strings.md | 15 +- 3 files changed, 246 insertions(+), 71 deletions(-) diff --git a/crates/by_transforms/src/transforms/dedent_string.rs b/crates/by_transforms/src/transforms/dedent_string.rs index 4132411cdc..45fb2f74ea 100644 --- a/crates/by_transforms/src/transforms/dedent_string.rs +++ b/crates/by_transforms/src/transforms/dedent_string.rs @@ -1,9 +1,12 @@ //! strips common indentation from triple-quoted multiline strings use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, ModModule, Stmt, StringFlags}; +use ruff_python_ast::{ + AnyStringFlags, Expr, InterpolatedStringElement, InterpolatedStringElements, ModModule, Stmt, + StringFlags, +}; use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_body}; -use ruff_text_size::{Ranged, TextRange}; +use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; @@ -39,13 +42,20 @@ struct State<'src> { } impl State<'_> { - fn dedent(&mut self, string_expr: &dyn Ranged) { - let start = usize::from(string_expr.range().start()); - let end = usize::from(string_expr.range().end()); - let raw = &self.source[start..end]; - match dedent_triple_string(raw) { + /// dedent the lone triple-quoted literal `string`, whose literal text — the + /// text that is not a replacement field — is `literals`, or all of its body + /// when that is `None` + fn dedent(&mut self, string: TextRange, flags: AnyStringFlags, literals: Option<&[TextRange]>) { + let raw = &self.source[string]; + let literals = literals.map(|literals| { + literals + .iter() + .map(|literal| *literal - string.start()) + .collect::>() + }); + match dedent_triple_string(raw, flags, literals.as_deref()) { DedentOutcome::Transformed(out) => { - self.edits.push((string_expr.range(), out)); + self.edits.push((string, out)); } DedentOutcome::Error(msg) => self.errors.push(msg), DedentOutcome::Unchanged => {} @@ -61,25 +71,26 @@ impl<'ast> Visitor<'ast> for State<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { match expr { Expr::StringLiteral(string_expr) => { - if let Some(part) = string_expr.as_single_part_string() { - let flags = part.flags; - if flags.is_triple_quoted() { - self.dedent(string_expr); - } + if let Some(part) = string_expr.as_single_part_string() + && part.flags.is_triple_quoted() + { + self.dedent(part.range(), part.flags.into(), None); } } Expr::FString(string_expr) => { - if let Some(fstring) = string_expr.as_single_part_fstring() { - if fstring.flags.is_triple_quoted() { - self.dedent(string_expr); - } + if let Some(fstring) = string_expr.as_single_part_fstring() + && fstring.flags.is_triple_quoted() + { + let literals = literal_ranges(&fstring.elements); + self.dedent(fstring.range(), fstring.flags.into(), Some(&literals)); } } Expr::TString(string_expr) => { - if let Some(tstring) = string_expr.as_single_part_tstring() { - if tstring.flags.is_triple_quoted() { - self.dedent(string_expr); - } + if let Some(tstring) = string_expr.as_single_part_tstring() + && tstring.flags.is_triple_quoted() + { + let literals = literal_ranges(&tstring.elements); + self.dedent(tstring.range(), tstring.flags.into(), Some(&literals)); } } _ => {} @@ -88,51 +99,94 @@ impl<'ast> Visitor<'ast> for State<'_> { } } -pub(crate) enum DedentOutcome { +/// the ranges of an f- or t-string's literal text, including the literal text of +/// its format specs, which a raw prefix makes raw too +fn literal_ranges(elements: &InterpolatedStringElements) -> Vec { + let mut ranges = Vec::new(); + let mut pending = vec![elements]; + while let Some(elements) = pending.pop() { + for element in elements { + match element { + InterpolatedStringElement::Literal(literal) => ranges.push(literal.range()), + InterpolatedStringElement::Interpolation(interpolation) => { + if let Some(spec) = &interpolation.format_spec { + pending.push(&spec.elements); + } + } + } + } + } + ranges +} + +enum DedentOutcome { Unchanged, Transformed(String), Error(String), } -fn dedent_triple_string(raw: &str) -> DedentOutcome { - let Some(quote_start) = raw.find("\"\"\"").or_else(|| raw.find("'''")) else { +/// dedent the triple-quoted literal `raw`, prefix and quotes included +/// +/// the dedented text is written between a quote ending in a line continuation +/// and a line continuation before the closing quote, so the literal keeps the +/// lines it was written on. a raw literal has no line continuation — its +/// backslash and newline are both text — so it is written as the plain literal +/// that holds the same text instead: every backslash and quote of its literal +/// text, `literals` (relative to `raw`, or all of the body when `None`), escaped +fn dedent_triple_string( + raw: &str, + flags: AnyStringFlags, + literals: Option<&[TextRange]>, +) -> DedentOutcome { + if flags.is_unclosed() { + return DedentOutcome::Unchanged; + } + let opener = flags.opener_len().to_usize(); + let Some(body) = raw.get(opener..raw.len() - flags.closer_len().to_usize()) else { return DedentOutcome::Unchanged; }; - let prefix = &raw[..quote_start]; - let quote = &raw[quote_start..quote_start + 3]; - let after_open = &raw[quote_start + 3..]; + let quote = flags.quote_str(); - if !after_open.ends_with(quote) { - return DedentOutcome::Unchanged; + let dedent = dedent_triple_quoted_body(body); + if dedent == TripleQuotedDedent::ClosingOverIndented { + return DedentOutcome::Error(format!( + "closing `{quote}` is indented more than the content of the triple-quoted string" + )); } - let body = &after_open[..after_open.len() - 3]; + let Some(runs) = dedent.runs() else { + return DedentOutcome::Unchanged; + }; - match dedent_triple_quoted_body(body) { - TripleQuotedDedent::Unchanged => DedentOutcome::Unchanged, - TripleQuotedDedent::ClosingOverIndented => DedentOutcome::Error(format!( - "closing `{quote}` is indented more than the content of the triple-quoted string" - )), - TripleQuotedDedent::Dedents { content, indent } => { - let deindented: Vec<&str> = content - .split('\n') - .map(|line| { - if line.trim().is_empty() { - "" - } else { - &line[indent.len()..] - } - }) - .collect(); + let raw_prefix = flags.prefix().is_raw(); + let mut prefix = raw[..opener - quote.len()].to_string(); + if raw_prefix { + prefix.retain(|c| !matches!(c, 'r' | 'R')); + } + let body_start = TextSize::try_from(opener).unwrap_or_default(); + let is_literal = |offset: TextSize| { + literals.is_none_or(|literals| literals.iter().any(|literal| literal.contains(offset))) + }; + let quote_char = flags.quote_style().as_char(); - let content = deindented.join("\n"); - DedentOutcome::Transformed(format!("{prefix}{quote}\\\n{content}\\\n{quote}")) + let mut content = String::with_capacity(body.len()); + for run in runs { + if !raw_prefix { + content.push_str(&body[run]); + continue; + } + for (index, c) in body[run].char_indices() { + let offset = body_start + run.start() + TextSize::try_from(index).unwrap_or_default(); + if (c == '\\' || c == quote_char) && is_literal(offset) { + content.push('\\'); + } + content.push(c); } } + DedentOutcome::Transformed(format!("{prefix}{quote}\\\n{content}\\\n{quote}")) } #[cfg(test)] mod tests { - use super::{DedentOutcome, dedent_triple_string}; use crate::python_passthrough::unchanged; use crate::{Config, transpile}; use indoc::indoc; @@ -212,24 +266,63 @@ mod tests { ); } + /// a raw literal has no line continuation, its backslash and newline both + /// being text, so it is written as the plain literal holding the same text #[test] fn rstring_dedent() { check( indoc! {r#" text = r""" - start-of-line - | + \d+ "quoted" + |\ """ "#}, indoc! {r#" - text = r"""\ - start-of-line - |\ + text = """\ + \\d+ \"quoted\" + |\\\ """ "#}, ); } + /// only the literal text of a raw f-string is escaped: a replacement field + /// is code, and the literal text of its format spec is raw too + #[test] + fn raw_fstring_dedent() { + check( + indoc! {r#" + a = "asdf" + text = Rf""" + \w{a!r:\>10} + """ + "#}, + indoc! {r#" + a = "asdf" + text = f"""\ + \\w{a!r:\\>10}\ + """ + "#}, + ); + } + + /// the quotes that close a literal are its own, whatever other quotes it holds + #[test] + fn single_quoted_triple_holding_double_quotes() { + check( + indoc! {r#" + text = ''' + say """hi""" + ''' + "#}, + indoc! {r#" + text = '''\ + say """hi"""\ + ''' + "#}, + ); + } + #[test] fn multiline_content() { check( @@ -284,22 +377,6 @@ mod tests { ); } - #[test] - fn unit_dedent_basic() { - match dedent_triple_string("\"\"\"\n hello\n \"\"\"") { - DedentOutcome::Transformed(s) => assert_eq!(s, "\"\"\"\\\nhello\\\n\"\"\""), - _ => panic!("expected Transformed"), - } - } - - #[test] - fn unit_dedent_no_indent_returns_unchanged() { - assert!(matches!( - dedent_triple_string("\"\"\"\nhello\n\"\"\""), - DedentOutcome::Unchanged - )); - } - #[test] fn closing_more_indented_errors() { let err = transpile( diff --git a/crates/ruff_python_trivia/src/basedpython.rs b/crates/ruff_python_trivia/src/basedpython.rs index 28d59a756d..d16c5644a0 100644 --- a/crates/ruff_python_trivia/src/basedpython.rs +++ b/crates/ruff_python_trivia/src/basedpython.rs @@ -1,5 +1,7 @@ //! basedpython-specific trivia analysis. +use ruff_text_size::{TextRange, TextSize}; + /// How a triple-quoted basedpython string's body is auto-dedented. /// /// See [`dedent_triple_quoted_body`]. @@ -57,6 +59,51 @@ pub fn dedent_triple_quoted_body(body: &str) -> TripleQuotedDedent<'_> { TripleQuotedDedent::Dedents { content, indent } } +impl TripleQuotedDedent<'_> { + /// The runs of the body this was read from that the dedent keeps, as ranges + /// of the body in order, or `None` when the body is not dedented. + /// + /// The text the string holds is these runs read one after another: each line + /// after the indent the lines share, with the newline that ends it, except + /// that a blank line keeps only its newline, the opening newline is dropped, + /// and so is the line the closing quotes sit on together with the newline + /// before it. A run is never empty. What is dropped is whitespace and nothing + /// else, so an escape sequence never straddles two runs. + pub fn runs(&self) -> Option> { + let Self::Dedents { content, indent } = self else { + return None; + }; + + // `content` is the body without its opening newline and without the line + // the closing quotes sit on, so it starts one character in + let mut offset = TextSize::new(1); + let mut runs = Vec::new(); + let mut lines = content.split('\n').peekable(); + while let Some(line) = lines.next() { + // a blank line is erased entirely; what is left of it is the newline + // that ends it + let stripped = if line.trim().is_empty() { + TextSize::of(line) + } else { + TextSize::of(*indent) + }; + let line_end = offset + TextSize::of(line); + // the newline between two lines is part of the text; the last line + // has none, its newline being the one before the closing line + let end = if lines.peek().is_some() { + line_end + TextSize::new(1) + } else { + line_end + }; + if offset + stripped < end { + runs.push(TextRange::new(offset + stripped, end)); + } + offset = line_end + TextSize::new(1); + } + Some(runs) + } +} + fn leading_whitespace(line: &str) -> &str { let end = line .find(|c: char| c != ' ' && c != '\t') @@ -77,6 +124,44 @@ fn common_prefix<'a>(a: &'a str, b: &'a str) -> &'a str { mod tests { use super::{TripleQuotedDedent, dedent_triple_quoted_body}; + /// the text `body` holds once it is dedented, read off its runs + fn dedented(body: &str) -> Option { + let runs = dedent_triple_quoted_body(body).runs()?; + Some(runs.iter().map(|run| &body[*run]).collect()) + } + + #[test] + fn runs_hold_each_line_after_the_indent() { + assert_eq!( + dedented("\n a\n (b\n ").as_deref(), + Some("a\n (b") + ); + } + + #[test] + fn runs_keep_only_the_newline_of_a_blank_line() { + assert_eq!( + dedented("\n a\n \n\n b\n ").as_deref(), + Some("a\n\n\nb") + ); + } + + #[test] + fn runs_are_never_empty() { + let body = "\n a\n\n b\n"; + let runs = dedent_triple_quoted_body(body) + .runs() + .expect("a dedented body"); + assert!(runs.iter().all(|run| !run.is_empty()), "{runs:?}"); + } + + #[test] + fn an_undedented_body_has_no_runs() { + assert_eq!(dedented("\nhello\n"), None); + assert_eq!(dedented("one line"), None); + assert_eq!(dedented("\n asdf\n "), None); + } + #[test] fn dedents_indented_body() { assert_eq!( diff --git a/docs/basedpython/features/dedent-strings.md b/docs/basedpython/features/dedent-strings.md index d8b8ee00cb..26c1c4e4f6 100644 --- a/docs/basedpython/features/dedent-strings.md +++ b/docs/basedpython/features/dedent-strings.md @@ -57,6 +57,19 @@ greet = f""" hello {name} """ # → f"""\ -# hello {name} +# hello {name}\ +# """ +``` + +a raw string holds its backslashes, so a line continuation can't drop its +opening and closing newlines: it is written as the plain string that holds the +same text instead, with the backslashes and quotes of its literal text escaped + +```by +pattern = r""" + \d+ "units" + """ +# → """\ +# \\d+ \"units\"\ # """ ``` From 3e23cabc3b9897b72eb5dab853fd5d9a7ad18d89 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:52:06 +1000 Subject: [PATCH 061/110] read a .by file's lone triple-quoted string as the value its dedent leaves, "a\n(b" rather than the indented text written, so its literal type, the pattern an invalid-regex check reads and the character it points at, the names __all__ exports and an f-string's literal text are what the program holds, from one rule the transpiler, the parser and injected fragments share --- .../src/reverse_transforms/dedent_string.rs | 32 ++- .../src/transforms/dedent_string.rs | 166 ++++++++------- crates/ruff_python_ast/src/str.rs | 121 ++++++++++- .../fixtures/ruff/based_multiline_strings.by | 7 - .../format@based_multiline_strings.by.snap | 15 -- .../src/parser/expression.rs | 9 + crates/ruff_python_parser/src/string.rs | 191 ++++++++++++++++++ crates/ruff_python_trivia/src/basedpython.rs | 12 +- crates/ty_ide/src/injection.rs | 96 ++++----- crates/ty_ide/src/syntax_outline.rs | 67 +++--- .../mdtest/basedpython_dedent_strings.md | 156 ++++++++++++++ .../src/types/infer/builder.rs | 1 + .../src/types/regex/source.rs | 45 +++-- docs/basedpython/features/dedent-strings.md | 8 + 14 files changed, 697 insertions(+), 229 deletions(-) create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_dedent_strings.md diff --git a/crates/by_transforms/src/reverse_transforms/dedent_string.rs b/crates/by_transforms/src/reverse_transforms/dedent_string.rs index d31f174241..8cd3193987 100644 --- a/crates/by_transforms/src/reverse_transforms/dedent_string.rs +++ b/crates/by_transforms/src/reverse_transforms/dedent_string.rs @@ -81,6 +81,11 @@ impl<'ast> Visitor<'ast> for IndentString<'_> { fn indent_triple_string(raw: &str, line_indent: &str) -> Option { let quote_start = raw.find("\"\"\"").or_else(|| raw.find("'''"))?; let prefix = &raw[..quote_start]; + // a raw string has no line continuation: the backslash and newline after its opening + // quotes are both text + if prefix.contains(['r', 'R']) { + return None; + } let quote = &raw[quote_start..quote_start + 3]; let after_open = &raw[quote_start + 3..]; @@ -101,7 +106,11 @@ fn indent_triple_string(raw: &str, line_indent: &str) -> Option { // form 2: closing """ sits right after the last content character (no trailing newline) // content's leading whitespace is meaningful (string value) — preserve as-is // anything else (bare \n before closing """) is left alone - if let Some(content) = inner.strip_suffix("\\\n") { + // the `\\` before the closing line's newline is a line continuation only when it is not + // the second half of an escaped backslash; an escaped one holds a backslash and a newline + if let Some(content) = inner.strip_suffix("\\\n") + && (content.len() - content.trim_end_matches('\\').len()) % 2 == 0 + { let indent = format!("{line_indent} "); let indented_lines: Vec = content .split('\n') @@ -243,16 +252,19 @@ mod tests { ); } + /// a raw string's backslashes are text, so it holds no line continuation to read #[test] - fn rstring_indent() { - check( - "text = r\"\"\"\\\nhello\\\n\"\"\"\n", - indoc! {r#" - text = r""" - hello - """ - "#}, - ); + fn rstring_unchanged() { + let source = "text = r\"\"\"\\\nhello\\\n\"\"\"\n"; + check(source, source); + } + + /// a `\\` before the closing line is an escaped backslash, and the newline after it + /// is part of the string + #[test] + fn escaped_backslash_before_the_closing_line_unchanged() { + let source = "text = \"\"\"\\\nhello\\\\\n\"\"\"\n"; + check(source, source); } #[test] diff --git a/crates/by_transforms/src/transforms/dedent_string.rs b/crates/by_transforms/src/transforms/dedent_string.rs index 45fb2f74ea..6f6298faf1 100644 --- a/crates/by_transforms/src/transforms/dedent_string.rs +++ b/crates/by_transforms/src/transforms/dedent_string.rs @@ -1,11 +1,11 @@ //! strips common indentation from triple-quoted multiline strings +use ruff_python_ast::str::{Dedent, dedent}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ AnyStringFlags, Expr, InterpolatedStringElement, InterpolatedStringElements, ModModule, Stmt, StringFlags, }; -use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_body}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, PassContext}; @@ -42,23 +42,21 @@ struct State<'src> { } impl State<'_> { - /// dedent the lone triple-quoted literal `string`, whose literal text — the - /// text that is not a replacement field — is `literals`, or all of its body - /// when that is `None` - fn dedent(&mut self, string: TextRange, flags: AnyStringFlags, literals: Option<&[TextRange]>) { - let raw = &self.source[string]; - let literals = literals.map(|literals| { - literals - .iter() - .map(|literal| *literal - string.start()) - .collect::>() - }); - match dedent_triple_string(raw, flags, literals.as_deref()) { - DedentOutcome::Transformed(out) => { - self.edits.push((string, out)); + /// dedent the lone literal `string`, whose literal text — the text that is not a + /// replacement field — is `literals`, or all of its body when that is `None` + fn dedent( + &mut self, + string: TextRange, + flags: AnyStringFlags, + literals: Option>, + ) { + match dedent(self.source, string, flags) { + Dedent::Unchanged => {} + Dedent::Invalid(error) => self.errors.push(error.to_string()), + Dedent::Runs { runs, .. } => { + let written = dedented_literal(self.source, string, flags, &runs, literals); + self.edits.push((string, written)); } - DedentOutcome::Error(msg) => self.errors.push(msg), - DedentOutcome::Unchanged => {} } } } @@ -71,26 +69,20 @@ impl<'ast> Visitor<'ast> for State<'_> { fn visit_expr(&mut self, expr: &'ast Expr) { match expr { Expr::StringLiteral(string_expr) => { - if let Some(part) = string_expr.as_single_part_string() - && part.flags.is_triple_quoted() - { + if let Some(part) = string_expr.as_single_part_string() { self.dedent(part.range(), part.flags.into(), None); } } Expr::FString(string_expr) => { - if let Some(fstring) = string_expr.as_single_part_fstring() - && fstring.flags.is_triple_quoted() - { + if let Some(fstring) = string_expr.as_single_part_fstring() { let literals = literal_ranges(&fstring.elements); - self.dedent(fstring.range(), fstring.flags.into(), Some(&literals)); + self.dedent(fstring.range(), fstring.flags.into(), Some(literals)); } } Expr::TString(string_expr) => { - if let Some(tstring) = string_expr.as_single_part_tstring() - && tstring.flags.is_triple_quoted() - { + if let Some(tstring) = string_expr.as_single_part_tstring() { let literals = literal_ranges(&tstring.elements); - self.dedent(tstring.range(), tstring.flags.into(), Some(&literals)); + self.dedent(tstring.range(), tstring.flags.into(), Some(literals)); } } _ => {} @@ -100,7 +92,7 @@ impl<'ast> Visitor<'ast> for State<'_> { } /// the ranges of an f- or t-string's literal text, including the literal text of -/// its format specs, which a raw prefix makes raw too +/// its format specs, which a raw prefix makes raw too, in source order fn literal_ranges(elements: &InterpolatedStringElements) -> Vec { let mut ranges = Vec::new(); let mut pending = vec![elements]; @@ -116,73 +108,65 @@ fn literal_ranges(elements: &InterpolatedStringElements) -> Vec { } } } + ranges.sort_unstable_by_key(Ranged::start); ranges } -enum DedentOutcome { - Unchanged, - Transformed(String), - Error(String), -} - -/// dedent the triple-quoted literal `raw`, prefix and quotes included +/// the literal `string` written so that it holds the text of `runs`, the runs of the +/// source its dedent keeps /// -/// the dedented text is written between a quote ending in a line continuation -/// and a line continuation before the closing quote, so the literal keeps the -/// lines it was written on. a raw literal has no line continuation — its -/// backslash and newline are both text — so it is written as the plain literal -/// that holds the same text instead: every backslash and quote of its literal -/// text, `literals` (relative to `raw`, or all of the body when `None`), escaped -fn dedent_triple_string( - raw: &str, +/// the text is written between a quote ending in a line continuation and a line +/// continuation before the closing quote, so the literal keeps the lines it was written +/// on. a raw literal has no line continuation — its backslash and newline are both +/// text — so it is written as the plain literal that holds the same text instead: every +/// backslash and quote of its literal text, `literals` (in source order, or all of the +/// body when `None`), escaped +fn dedented_literal( + source: &str, + string: TextRange, flags: AnyStringFlags, - literals: Option<&[TextRange]>, -) -> DedentOutcome { - if flags.is_unclosed() { - return DedentOutcome::Unchanged; - } - let opener = flags.opener_len().to_usize(); - let Some(body) = raw.get(opener..raw.len() - flags.closer_len().to_usize()) else { - return DedentOutcome::Unchanged; - }; + runs: &[TextRange], + literals: Option>, +) -> String { let quote = flags.quote_str(); - - let dedent = dedent_triple_quoted_body(body); - if dedent == TripleQuotedDedent::ClosingOverIndented { - return DedentOutcome::Error(format!( - "closing `{quote}` is indented more than the content of the triple-quoted string" - )); - } - let Some(runs) = dedent.runs() else { - return DedentOutcome::Unchanged; - }; - - let raw_prefix = flags.prefix().is_raw(); - let mut prefix = raw[..opener - quote.len()].to_string(); - if raw_prefix { + let opener = &source[TextRange::at(string.start(), flags.opener_len())]; + let mut prefix = opener[..opener.len() - quote.len()].to_owned(); + let raw = flags.prefix().is_raw(); + if raw { prefix.retain(|c| !matches!(c, 'r' | 'R')); } - let body_start = TextSize::try_from(opener).unwrap_or_default(); - let is_literal = |offset: TextSize| { - literals.is_none_or(|literals| literals.iter().any(|literal| literal.contains(offset))) - }; let quote_char = flags.quote_style().as_char(); - let mut content = String::with_capacity(body.len()); + let mut content = String::new(); + // the runs and the literal text are both in source order, so one pass over each + // finds the literal text every character of the runs is in + let mut literals = literals.map(|literals| literals.into_iter().peekable()); for run in runs { - if !raw_prefix { - content.push_str(&body[run]); + if !raw { + content.push_str(&source[*run]); continue; } - for (index, c) in body[run].char_indices() { - let offset = body_start + run.start() + TextSize::try_from(index).unwrap_or_default(); - if (c == '\\' || c == quote_char) && is_literal(offset) { + for (index, c) in source[*run].char_indices() { + let offset = run.start() + TextSize::try_from(index).unwrap_or_default(); + let is_literal = match &mut literals { + None => true, + Some(literals) => { + while literals + .next_if(|literal| literal.end() <= offset) + .is_some() + {} + literals + .peek() + .is_some_and(|literal| literal.contains(offset)) + } + }; + if (c == '\\' || c == quote_char) && is_literal { content.push('\\'); } content.push(c); } } - DedentOutcome::Transformed(format!("{prefix}{quote}\\\n{content}\\\n{quote}")) + format!("{prefix}{quote}\\\n{content}\\\n{quote}") } #[cfg(test)] @@ -408,6 +392,32 @@ mod tests { assert!(err.contains("indented more"), "got: {err}"); } + /// the newline a `\\` ending the last line continues past is the one the dedent drops + #[test] + fn a_line_continuation_ending_the_last_line_errors() { + for source in [ + "text = \"\"\"\n a\\\n \"\"\"\n", + "text = \"\"\"\n a\\\\\\\n \"\"\"\n", + "text = f\"\"\"\n {a}\\\n \"\"\"\n", + ] { + let err = transpile(source, &Config::test_default()).unwrap_err(); + assert!(err.contains("line continuation"), "{source:?} got: {err}"); + } + } + + /// an escaped backslash is no line continuation, and a raw string has none + #[test] + fn a_backslash_ending_the_last_line_as_text() { + check( + "text = \"\"\"\n a\\\\\n \"\"\"\n", + "text = \"\"\"\\\na\\\\\\\n\"\"\"\n", + ); + check( + "text = r\"\"\"\n a\\\n \"\"\"\n", + "text = \"\"\"\\\na\\\\\\\n\"\"\"\n", + ); + } + #[test] fn closing_equal_indent_ok() { // closing at exactly the content's min indent — fine, dedents to col 0 diff --git a/crates/ruff_python_ast/src/str.rs b/crates/ruff_python_ast/src/str.rs index a24c51a87f..7243654604 100644 --- a/crates/ruff_python_ast/src/str.rs +++ b/crates/ruff_python_ast/src/str.rs @@ -2,7 +2,126 @@ use aho_corasick::{AhoCorasick, AhoCorasickKind, Anchored, Input, MatchKind, Sta use std::fmt; use std::sync::LazyLock; -use ruff_text_size::{TextLen, TextRange}; +use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_body}; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; + +use crate::{AnyStringFlags, StringFlags}; + +/// basedpython: the runs of the string written at `range` of `source` that its +/// [auto-dedent](https://docs.basedpython.org/features/dedent-strings) keeps, as +/// ranges of `source` in order, or `None` when it is not dedented or cannot be +/// (see [`dedent`]) +pub fn dedented_runs( + source: &str, + range: TextRange, + flags: AnyStringFlags, +) -> Option> { + match dedent(source, range, flags) { + Dedent::Runs { runs, .. } => Some(runs), + Dedent::Unchanged | Dedent::Invalid(_) => None, + } +} + +/// basedpython: how the string written at `range` of `source` is +/// [auto-dedented](https://docs.basedpython.org/features/dedent-strings), the one +/// rule the parser, the checker and the transpiler all read a dedented string by +/// +/// only a lone triple-quoted literal is dedented, never one of several adjacent +/// ones, which is for the caller to know; a bytes literal never is, whatever its quotes +pub fn dedent(source: &str, range: TextRange, flags: AnyStringFlags) -> Dedent { + if !flags.is_triple_quoted() || flags.is_unclosed() || flags.is_byte_string() { + return Dedent::Unchanged; + } + let body = TextRange::new( + range.start() + flags.opener_len(), + range.end() - flags.closer_len(), + ); + let Some(text) = source.get(body.start().to_usize()..body.end().to_usize()) else { + return Dedent::Unchanged; + }; + let dedent = dedent_triple_quoted_body(text); + if dedent == TripleQuotedDedent::ClosingOverIndented { + return Dedent::Invalid(DedentError { + kind: DedentErrorKind::ClosingOverIndented { + quote: flags.quote_str(), + }, + range: TextRange::new(body.end(), range.end()), + }); + } + let TripleQuotedDedent::Dedents { indent, .. } = dedent else { + return Dedent::Unchanged; + }; + let indent = indent.text_len(); + let Some(runs) = dedent.runs() else { + return Dedent::Unchanged; + }; + let runs: Vec = runs.into_iter().map(|run| run + body.start()).collect(); + // a `\` ending the last line continues the string past the newline after it, which is + // the newline the dedent drops, so there is nothing left for it to continue onto. a raw + // string has no line continuation: its backslash is text + if !flags.prefix().is_raw() + && let Some(last) = runs.last() + && let trailing = source[*last].len() - source[*last].trim_end_matches('\\').len() + && trailing % 2 == 1 + { + return Dedent::Invalid(DedentError { + kind: DedentErrorKind::LineContinuationAtEnd, + range: TextRange::at(last.end() - '\\'.text_len(), '\\'.text_len()), + }); + } + Dedent::Runs { runs, indent } +} + +/// basedpython: how a string is [auto-dedented](dedent) +#[derive(Debug, PartialEq, Eq)] +pub enum Dedent { + /// the string is not dedented, and is read as written + Unchanged, + /// the string is dedented by stripping `indent` from each of its lines + Runs { + /// the runs of the source the dedent keeps, in order: the text the string is read + /// from is these runs, one after another + runs: Vec, + /// the length of the indent stripped from each line that is not blank + indent: TextSize, + }, + /// the string has the shape of a dedented one, but no reading of it as one + Invalid(DedentError), +} + +/// basedpython: why a string that has the shape of a dedented one cannot be read as one +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DedentError { + kind: DedentErrorKind, + range: TextRange, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum DedentErrorKind { + ClosingOverIndented { quote: &'static str }, + LineContinuationAtEnd, +} + +impl Ranged for DedentError { + fn range(&self) -> TextRange { + self.range + } +} + +impl fmt::Display for DedentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + DedentErrorKind::ClosingOverIndented { quote } => write!( + f, + "closing `{quote}` is indented more than the content of the triple-quoted string" + ), + DedentErrorKind::LineContinuationAtEnd => f.write_str( + "a line continuation cannot end the last line of a dedented string, \ + whose newline the dedent drops", + ), + } + } +} /// Enumeration of the two kinds of quotes that can be used /// for Python string/f/t-string/bytestring literals diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/based_multiline_strings.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/based_multiline_strings.by index 014bd47dc6..e89aff8d72 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/based_multiline_strings.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/based_multiline_strings.by @@ -44,13 +44,6 @@ def reindent(): """ return z -# the closing quotes are indented past the content, which has no consistent -# dedent depth — left for the transpiler to reject -def closing_past_the_content(): - return """ - a - """ - # the closing quotes share a line with content, so nothing is dedented and # nothing may be reindented def closing_on_the_content_line(): diff --git a/crates/ruff_python_formatter/tests/snapshots/format@based_multiline_strings.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@based_multiline_strings.by.snap index 6d5cc7733b..c6d23dca3b 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@based_multiline_strings.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@based_multiline_strings.by.snap @@ -50,13 +50,6 @@ def reindent(): """ return z -# the closing quotes are indented past the content, which has no consistent -# dedent depth — left for the transpiler to reject -def closing_past_the_content(): - return """ - a - """ - # the closing quotes share a line with content, so nothing is dedented and # nothing may be reindented def closing_on_the_content_line(): @@ -139,14 +132,6 @@ def reindent(): return z -# the closing quotes are indented past the content, which has no consistent -# dedent depth — left for the transpiler to reject -def closing_past_the_content(): - return """ - a - """ - - # the closing quotes share a line with content, so nothing is dedented and # nothing may be reindented def closing_on_the_content_line(): diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index b7fc3f823d..728773969d 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2607,6 +2607,15 @@ impl<'src> Parser<'src> { let first = self.parse_string(); if !self.at_ts(STRING_START_SET) { + // basedpython dedents a lone triple-quoted string, so its value is + // what is left once the indent is stripped + if self.options.is_basedpython { + let (string, error) = first.dedented(self.source); + if let Some(error) = error { + self.add_error(ParseErrorType::BasedPythonOnly(error.to_string()), error); + } + return string.into(); + } return first.into(); } diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index bd7e9961ab..a918f5f871 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -3,6 +3,7 @@ use bstr::ByteSlice; use std::fmt; +use ruff_python_ast::str::{Dedent, DedentError, dedent}; use ruff_python_ast::token::TokenKind; use ruff_python_ast::{self as ast, AnyStringFlags, AtomicNodeIndex, Expr, StringFlags}; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -39,6 +40,96 @@ impl From for Expr { } } +impl StringType { + /// basedpython: this string as the program holds it once a lone triple-quoted + /// string is [auto-dedented](https://docs.basedpython.org/features/dedent-strings), + /// and why it cannot be when it has the shape of a dedented string but no reading as + /// one, in which case it is left as written + /// + /// the transpiler strips the indent from the text the string is written as, + /// and python reads what is left, so the value here is the one read from the + /// text the dedent keeps. only values change: every node keeps the range it + /// was written at, since the text the dedent drops is still in the file. + /// a bytes literal is not dedented, and neither is a string written as + /// several adjacent literals, so the caller passes only a lone one + pub(crate) fn dedented(self, source: &str) -> (Self, Option) { + let (range, flags) = match &self { + // a string whose escapes do not read has already been reported, and has no + // value to dedent + StringType::Str(literal) if literal.flags.is_invalid() => return (self, None), + StringType::Str(literal) => (literal.range, AnyStringFlags::from(literal.flags)), + StringType::FString(fstring) => (fstring.range, AnyStringFlags::from(fstring.flags)), + StringType::TString(tstring) => (tstring.range, AnyStringFlags::from(tstring.flags)), + StringType::Bytes(_) => return (self, None), + }; + let runs = match dedent(source, range, flags) { + Dedent::Unchanged => return (self, None), + Dedent::Invalid(error) => return (self, Some(error)), + Dedent::Runs { runs, .. } => runs, + }; + let dedented = match self { + StringType::Str(mut literal) => { + let content = literal.content_range(); + let text = text_of(source, &runs, content); + if let Ok(StringType::Str(dedented)) = + StringParser::new(&text, flags, content.start(), literal.range).parse() + { + literal.value = dedented.value; + } + StringType::Str(literal) + } + StringType::FString(mut fstring) => { + dedent_literal_elements(source, &runs, flags, &mut fstring.elements); + StringType::FString(fstring) + } + StringType::TString(mut tstring) => { + dedent_literal_elements(source, &runs, flags, &mut tstring.elements); + StringType::TString(tstring) + } + StringType::Bytes(_) => self, + }; + (dedented, None) + } +} + +/// the text of `within` that `runs` keep +fn text_of(source: &str, runs: &[TextRange], within: TextRange) -> String { + runs.iter() + .filter_map(|run| run.intersect(within)) + .map(|kept| &source[kept]) + .collect() +} + +/// read again each literal element of an f- or t-string, the literal text of its +/// format specs included, from the text the dedent keeps of it. the replacement +/// fields are code, which the indent does not change the meaning of +fn dedent_literal_elements( + source: &str, + runs: &[TextRange], + flags: AnyStringFlags, + elements: &mut ast::InterpolatedStringElements, +) { + for element in elements.iter_mut() { + match element { + ast::InterpolatedStringElement::Literal(literal) => { + let text = text_of(source, runs, literal.range); + if text != source[literal.range] + && let Ok(dedented) = + StringParser::new(&text, flags, literal.range.start(), literal.range) + .parse_interpolated_string_middle() + { + literal.value = dedented.value; + } + } + ast::InterpolatedStringElement::Interpolation(interpolation) => { + if let Some(spec) = &mut interpolation.format_spec { + dedent_literal_elements(source, runs, flags, &mut spec.elements); + } + } + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InterpolatedStringKind { FString, @@ -538,6 +629,106 @@ mod tests { const MAC_EOL: &str = "\r"; const UNIX_EOL: &str = "\n"; + /// the value of each string statement of `source`, its literal text for an + /// f-string, parsed as a `.by` file when `basedpython` + fn string_values(source: &str, basedpython: bool) -> Vec { + let source_type = if basedpython { + ruff_python_ast::PySourceType::BasedPython + } else { + ruff_python_ast::PySourceType::Python + }; + let parsed = crate::parse_unchecked_source(source, source_type); + parsed + .suite() + .iter() + .filter_map(|stmt| match &*stmt.as_expr_stmt()?.value { + ruff_python_ast::Expr::StringLiteral(string) => Some(string.value.to_string()), + ruff_python_ast::Expr::FString(string) => Some( + string + .value + .elements() + .filter_map(|element| Some(&*element.as_literal()?.value)) + .collect(), + ), + _ => None, + }) + .collect() + } + + #[test] + fn basedpython_reads_a_lone_triple_quoted_string_dedented() { + let source = r#"""" + a + \t + """ +r''' + \d + ''' +f""" + {x} + b + """ +""" + a + """ "b" +"#; + assert_eq!( + string_values(source, true), + ["a\n \t", r"\d", "\nb", "\n a\n b"] + ); + assert_eq!( + string_values(source, false), + [ + "\n a\n \t\n ", + "\n \\d\n ", + "\n \n b\n ", + "\n a\n b" + ] + ); + } + + /// the errors parsing `source` as a `.by` file reports, each with the text it points at + fn basedpython_errors(source: &str) -> Vec<(String, &str)> { + crate::parse_unchecked_source(source, ruff_python_ast::PySourceType::BasedPython) + .errors() + .iter() + .map(|error| (error.error.to_string(), &source[error.location])) + .collect() + } + + #[test] + fn basedpython_rejects_a_string_it_cannot_dedent() { + assert_eq!( + basedpython_errors("'''\n a\n '''\n"), + [( + "closing `'''` is indented more than the content of the triple-quoted string" + .to_owned(), + "'''" + )] + ); + let continuation = "a line continuation cannot end the last line of a dedented string, \ + whose newline the dedent drops"; + for source in [ + "\"\"\"\n a\\\n \"\"\"\n", + "\"\"\"\n a\\\\\\\n \"\"\"\n", + "f\"\"\"\n {x}\\\n \"\"\"\n", + "t\"\"\"\n {x}\\\n \"\"\"\n", + ] { + assert_eq!( + basedpython_errors(source), + [(continuation.to_owned(), "\\")], + "{source:?}" + ); + } + for source in [ + "\"\"\"\n a\\\\\n \"\"\"\n", + "r\"\"\"\n a\\\n \"\"\"\n", + "\"\"\"\n a\\\n b\n \"\"\"\n", + ] { + assert_eq!(basedpython_errors(source), [], "{source:?}"); + } + } + fn parse_suite(source: &str) -> Result { parse_module(source).map(Parsed::into_suite) } diff --git a/crates/ruff_python_trivia/src/basedpython.rs b/crates/ruff_python_trivia/src/basedpython.rs index d16c5644a0..50afd960ec 100644 --- a/crates/ruff_python_trivia/src/basedpython.rs +++ b/crates/ruff_python_trivia/src/basedpython.rs @@ -60,15 +60,17 @@ pub fn dedent_triple_quoted_body(body: &str) -> TripleQuotedDedent<'_> { } impl TripleQuotedDedent<'_> { - /// The runs of the body this was read from that the dedent keeps, as ranges - /// of the body in order, or `None` when the body is not dedented. + /// the runs of the body this was read from that the dedent keeps, as ranges + /// of the body in order, or `None` when the body is not dedented /// - /// The text the string holds is these runs read one after another: each line + /// the text the string holds is these runs read one after another: each line /// after the indent the lines share, with the newline that ends it, except /// that a blank line keeps only its newline, the opening newline is dropped, /// and so is the line the closing quotes sit on together with the newline - /// before it. A run is never empty. What is dropped is whitespace and nothing - /// else, so an escape sequence never straddles two runs. + /// before it. a run is never empty. what is dropped is whitespace and nothing + /// else, so an escape sequence never straddles two runs, save for a `\` ending + /// the last line, whose newline is dropped: a reader of the string rejects that + /// one (`ruff_python_ast::str::dedent`) pub fn runs(&self) -> Option> { let Self::Dedents { content, indent } = self else { return None; diff --git a/crates/ty_ide/src/injection.rs b/crates/ty_ide/src/injection.rs index 81c217f508..1d2877fc67 100644 --- a/crates/ty_ide/src/injection.rs +++ b/crates/ty_ide/src/injection.rs @@ -50,13 +50,13 @@ use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_db::source::source_text; +use ruff_python_ast::str::dedented_runs; use ruff_python_ast::token::TokenKind; use ruff_python_ast::visitor::source_order::{ SourceOrderVisitor, TraversalSignal, walk_body, walk_expr, walk_stmt, }; -use ruff_python_ast::{self as ast, AnyNodeRef, StringFlags}; -use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_body}; -use ruff_text_size::{Ranged, TextRange, TextSize}; +use ruff_python_ast::{self as ast, AnyNodeRef}; +use ruff_text_size::{Ranged, TextRange}; use rustc_hash::FxHashMap; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind}; @@ -409,6 +409,11 @@ impl<'db> SourceOrderVisitor<'db> for InjectionFinder<'_, 'db> { /// `dedent_string`: a lone triple-quoted literal. A fragment written as several /// adjacent literals is left as written there, so it is left as written here /// too, and the two agree about what the program holds. +/// +/// the runs are [`dedented_runs`]', the rule the transpiler and the parser read +/// the string by. a string shaped like a dedented one that has no reading as one, +/// its closing quotes indented past its content or a `\` ending its last line, is +/// an error there and is left as written, so it is left as written here too fn content_ranges( string: &ast::ExprStringLiteral, source: &str, @@ -416,9 +421,9 @@ fn content_ranges( ) -> Box<[TextRange]> { if dedents && let Some(part) = string.as_single_part_string() - && part.flags.is_triple_quoted() + && let Some(runs) = dedented_runs(source, part.range, part.flags.into()) { - return text_ranges(part.content_range(), source); + return runs.into_boxed_slice(); } string .value @@ -427,59 +432,6 @@ fn content_ranges( .collect() } -/// The runs of `body` that survive basedpython's dedent, or `body` itself when -/// nothing is stripped from it. -/// -/// The dedent rule is [`dedent_triple_quoted_body`]'s, which is the one the -/// transpiler applies (`by_transforms`'s `dedent_string`), so a fragment reads -/// here exactly as it will at runtime. Empty runs are dropped: a blank line -/// contributes only the newline that follows it, and the range before that -/// newline is nothing at all. -pub(crate) fn text_ranges(body: TextRange, source: &str) -> Box<[TextRange]> { - let Some(text) = source.get(body.start().to_usize()..body.end().to_usize()) else { - return Box::from([body]); - }; - let (content, indent) = match dedent_triple_quoted_body(text) { - TripleQuotedDedent::Dedents { content, indent } => (content, indent), - // Nothing is stripped, so the text is the body exactly as written. - TripleQuotedDedent::Unchanged => return Box::from([body]), - // The transpiler refuses this one rather than guessing a depth, so - // there is no dedented text to point at: report the body as written and - // let the error the transpiler raises be the thing that gets fixed. - TripleQuotedDedent::ClosingOverIndented => return Box::from([body]), - }; - - // `content` is `text` without its opening newline and without the line the - // closing quotes sit on, so it starts one character in. - let start = body.start() + TextSize::from(1); - let mut ranges = Vec::new(); - let mut offset = start; - let mut lines = content.split('\n').peekable(); - while let Some(line) = lines.next() { - // A blank line is erased entirely, exactly as the transpiler erases it; - // what is left of it is the newline that ends it. - let stripped = if line.trim().is_empty() { - TextSize::of(line) - } else { - TextSize::of(indent) - }; - let line_end = offset + TextSize::of(line); - let text_start = offset + stripped; - // The newline between two lines is part of the text; the last line has - // none, its newline having been consumed as the closing line's. - let end = if lines.peek().is_some() { - line_end + TextSize::from(1) - } else { - line_end - }; - if text_start < end { - ranges.push(TextRange::new(text_start, end)); - } - offset = line_end + TextSize::from(1); - } - ranges.into_boxed_slice() -} - /// The language the parameter named `name` of the function `definition` defines /// expects: either because it says so, or because it hands the value on to a /// parameter that does. @@ -925,6 +877,34 @@ def render(): "#); } + /// a `\` ending the last line of a dedented string would continue it onto the + /// newline the dedent drops, so the string has no dedented reading: the + /// transpiler reports it and the parser keeps it as written, and so does the + /// fragment + #[test] + fn a_string_the_dedent_cannot_read_is_reported_as_written() { + let test = injection_test( + "\ +def render(): + # language=html + page = \"\"\" +

\\ + \"\"\" +", + ); + + assert_snapshot!(test.injections(), @r#" + info[injection]: html (comment) + --> main.by:3:15 + | + 3 | page = """ + | _______________^ + 4 | |
\ + 5 | | """ + | |____^ + "#); + } + /// python does not dedent a triple-quoted string, so in a python file the /// text the literal stands for is the whole run between its quotes, /// indentation and both newlines included. Reporting the dedented runs there diff --git a/crates/ty_ide/src/syntax_outline.rs b/crates/ty_ide/src/syntax_outline.rs index a2d91175d4..aff090b6cb 100644 --- a/crates/ty_ide/src/syntax_outline.rs +++ b/crates/ty_ide/src/syntax_outline.rs @@ -11,6 +11,7 @@ use ruff_db::PythonFile; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; +use ruff_python_ast::str::{Dedent, dedent}; use ruff_python_ast::token::{TokenKind, Tokens, parenthesized_range}; use ruff_python_ast::visitor::source_order::{ SourceOrderVisitor, walk_expr, walk_f_string, walk_t_string, @@ -19,12 +20,10 @@ use ruff_python_ast::{ self as ast, AnyNodeRef, AnyStringFlags, Expr, InterpolatedStringElement, InterpolatedStringElements, Stmt, StringFlags, }; -use ruff_python_trivia::basedpython::{TripleQuotedDedent, dedent_triple_quoted_body}; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::Db; use crate::comma_list::comma_lists; -use crate::injection::text_ranges; use crate::method_kind::{MethodKind, method_kind}; /// The block structure and string literals of one file. @@ -73,7 +72,7 @@ pub struct OutlineStatement { /// static method has none pub receiver: Option, /// for a simple statement, the expression Enter may break across lines by wrapping it in - /// parentheses; see [`wrappable`] + /// parentheses; see `wrappable` pub wrap: Option, } @@ -92,7 +91,7 @@ pub struct OutlineClause { /// The statements of the suite. pub body: Vec, /// the expression of the header Enter may break across lines by wrapping it in parentheses; - /// see [`wrappable`] + /// see `wrappable` pub wrap: Option, } @@ -126,10 +125,11 @@ pub struct OutlineString { /// How many leading characters basedpython strips from each line of this literal, when it /// strips any. /// - /// The rule is [`dedent_triple_quoted_body`]'s, the one the transpiler applies, so this is - /// set exactly when the program's string differs from the literal as written: a lone + /// the rule is [`dedent`]'s, the one the transpiler and the parser apply, so this is set + /// exactly when the program's string differs from the literal as written: a lone /// triple-quoted string in a basedpython file whose content starts on the line after the - /// opening quotes and whose closing quotes sit on a line of their own. + /// opening quotes and whose closing quotes sit on a line of their own. a string of that + /// shape the dedent has no reading of, an error there, is left as written pub stripped_indent: Option, /// The `{...}` interpolations of an f-string or t-string, braces included. pub interpolations: Vec, @@ -873,26 +873,14 @@ impl StringCollector<'_> { interpolations: Vec, literal_runs: &[TextRange], ) { - let stripped_indent = if self.dedents - && self.dedentable == Some(range) - && flags.is_triple_quoted() - && !flags.is_unclosed() - && !flags.is_byte_string() - { - let body = TextRange::new( - range.start() + flags.opener_len(), - range.end() - flags.closer_len(), - ); - match self - .source - .get(body.to_std_range()) - .map(dedent_triple_quoted_body) - { - Some(TripleQuotedDedent::Dedents { indent, .. }) => Some(TextSize::of(indent)), - _ => None, - } + let dedent = if self.dedents && self.dedentable == Some(range) { + dedent(self.source, range, flags) } else { - None + Dedent::Unchanged + }; + let (stripped_indent, dedented_runs) = match dedent { + Dedent::Runs { runs, indent } => (Some(indent), Some(runs)), + Dedent::Unchanged | Dedent::Invalid(_) => (None, None), }; let mut escapes = Vec::new(); @@ -902,13 +890,7 @@ impl StringCollector<'_> { } } - let text = Self::text( - range, - flags, - stripped_indent.is_some(), - &interpolations, - self.source, - ); + let text = Self::text(range, flags, dedented_runs, &interpolations); self.strings.push(OutlineString { range, @@ -919,15 +901,14 @@ impl StringCollector<'_> { }); } - /// See [`OutlineString::text`]. `stripped` says whether basedpython dedents this literal, - /// whose runs are then the ones the dedent leaves, by [`text_ranges`] — the rule the - /// language-injection fragments are reported by, so a string reads the same either way. + /// see [`OutlineString::text`]. `dedented_runs` are the runs basedpython's dedent leaves + /// of this literal, when it dedents it — the rule the language-injection fragments are + /// reported by, so a string reads the same either way fn text( range: TextRange, flags: AnyStringFlags, - stripped: bool, + dedented_runs: Option>, interpolations: &[TextRange], - source: &str, ) -> Vec { let start = range.start() + flags.opener_len(); let closer = if flags.is_unclosed() { @@ -937,11 +918,7 @@ impl StringCollector<'_> { }; let end = range.end().checked_sub(closer).unwrap_or(start).max(start); let body = TextRange::new(start, end); - let runs = if stripped { - text_ranges(body, source).into_vec() - } else { - vec![body] - }; + let runs = dedented_runs.unwrap_or_else(|| vec![body]); // An interpolation is an expression rather than text, so each run is cut around it. let mut text = Vec::with_capacity(runs.len()); @@ -1412,6 +1389,9 @@ def f(): closing_past_the_content = \"\"\" a \"\"\" + continued_last_line = \"\"\" + a\\ + \"\"\" concatenated = \"\"\" a \"\"\" \"b\" @@ -1428,6 +1408,7 @@ def f(): \"\"\"Summary.⏎ more⏎ \"\"\" \"\"\"⏎ text\"\"\" \"\"\"⏎ a⏎ \"\"\" +\"\"\"⏎ a\\⏎ \"\"\" escape `\\⏎` \"\"\"⏎ a⏎ \"\"\" \"b\" b\"\"\"⏎ a⏎ \"\"\" diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_dedent_strings.md b/crates/ty_python_semantic/resources/mdtest/basedpython_dedent_strings.md new file mode 100644 index 0000000000..59a6170ec4 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_dedent_strings.md @@ -0,0 +1,156 @@ +# dedented triple-quoted strings + +A lone triple-quoted string that opens with a newline and closes on a line of its own is dedented: +the transpiler strips the indent its lines share, and drops the opening newline and the line the +closing quotes sit on. Its value is the text that is left, so that is the literal we infer. + +## the value is the dedented text + +```by +def f(): + text = """ + hello + world + """ + reveal_type(text) # revealed: "hello\n world" +``` + +A blank line keeps only its newline, however much whitespace it was written with. + +```by +text = """ + a + + b + """ +reveal_type(text) # revealed: "a\n\nb" +``` + +Escapes are read from the dedented text, and a raw string keeps its backslashes. + +```by +escaped = """ + \t(\ + x + """ +reveal_type(escaped) # revealed: "\t(x" + +raw = r""" + \d+ + """ +reveal_type(raw) # revealed: "\\d+" +``` + +## f-strings + +The literal text of an f-string is dedented the same way. + +```by +def f(name: str): + # revealed: "hello\n there" + reveal_type(f""" + hello + there + """) +``` + +## t-strings + +```toml +[environment] +python-version = "3.14" +``` + +So is the literal text of a t-string: its `strings` are read from the dedented text. + +```by +from string.templatelib import Template + +def greet(name: str) -> Template: + return t""" + hello {name} + there + """ + +assert greet("you").strings == ("hello ", "\n there") +``` + +## strings that are not dedented + +A string written on the line of its opening quotes, a string whose lines share no indent, a bytes +literal and a string written as several adjacent literals are all left as written. + +```by +inline = """one line""" +reveal_type(inline) # revealed: "one line" + +unindented = """ +a +""" +reveal_type(unindented) # revealed: "\na\n" + +data = b""" + a + """ +reveal_type(data) # revealed: b"\n a\n " + +joined = """ + a + """ "b" +reveal_type(joined) # revealed: "\n a\n b" +``` + +## strings a dedent cannot read + +The closing quotes set the depth the content is written at, so content written shallower than them +has no consistent dedent, and we reject the string. + +```by +# error: [invalid-syntax] "is indented more than the content of the triple-quoted string" +text = """ + a + """ +``` + +A `\` ending the last line would continue the string past the newline before the closing quotes, +which is the newline the dedent drops, so we reject that too. A raw string has no line continuation, +and keeps its backslash. + +```by +# error: [invalid-syntax] "a line continuation cannot end the last line of a dedented string, whose newline the dedent drops" +text = """ + a\ + """ + +# error: [invalid-syntax] +formatted = f""" + a\ + """ + +raw = r""" + a\ + """ +reveal_type(raw) # revealed: "a\\" +``` + +## a dedented name in `__all__` + +`__all__` holds what the program does, so a name written in a dedented string is exported. + +`exporter.by`: + +```by +def f(): ... +def g(): ... + +__all__ = [""" + f + """] +``` + +```by +from exporter import * + +f() +g() # error: [unresolved-reference] +``` diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 83f8c9a16b..ef80998798 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -12426,6 +12426,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { regex::pattern_range( pattern, &source_text(db, self.file()), + self.file().source_type(db).is_basedpython(), &text, error.span(), ) diff --git a/crates/ty_python_semantic/src/types/regex/source.rs b/crates/ty_python_semantic/src/types/regex/source.rs index 26bf02cb6d..db39758ec0 100644 --- a/crates/ty_python_semantic/src/types/regex/source.rs +++ b/crates/ty_python_semantic/src/types/regex/source.rs @@ -9,6 +9,7 @@ use std::ops::Range; use ruff_python_ast as ast; +use ruff_python_ast::str::dedented_runs; use ruff_text_size::{TextRange, TextSize}; /// where in `source` the characters `span` of `pattern` were written, when @@ -16,26 +17,34 @@ use ruff_text_size::{TextRange, TextSize}; /// /// a bytes pattern is `pattern` read as latin-1, one character a byte. `span` /// is clamped to the pattern, so a span past its end — an error at the end — -/// is its last character. `None` for an expression that is not a literal, or a -/// literal whose value does not read as `pattern`, which leaves the caller to -/// point at the expression as a whole +/// is its last character. `dedents` says whether `source` is basedpython, which +/// reads a lone triple-quoted string from the text its dedent keeps. `None` for +/// an expression that is not a literal, or a literal whose value does not read +/// as `pattern`, which leaves the caller to point at the expression as a whole pub(crate) fn pattern_range( expr: &ast::Expr, source: &str, + dedents: bool, pattern: &str, span: Range, ) -> Option { let mut written = Vec::new(); match expr { ast::Expr::StringLiteral(string) => { + let dedented = string + .as_single_part_string() + .filter(|_| dedents) + .and_then(|part| dedented_runs(source, part.range, part.flags.into())); for part in &string.value { - read_part( - source, - part.content_range(), - part.flags.prefix().is_raw(), - false, - &mut written, - )?; + let raw = part.flags.prefix().is_raw(); + match &dedented { + Some(runs) => { + for run in runs { + read_part(source, *run, raw, false, &mut written)?; + } + } + None => read_part(source, part.content_range(), raw, false, &mut written)?, + } } } ast::Expr::BytesLiteral(bytes) => { @@ -82,7 +91,8 @@ struct Written { range: TextRange, } -/// read the text between one literal's quotes, `content`, as python does +/// read the text `content` of a literal as python does: the text between its +/// quotes, or one run of it its dedent keeps, which never splits an escape fn read_part( source: &str, content: TextRange, @@ -194,7 +204,7 @@ mod tests { fn written(literal: &str, pattern: &str, span: std::ops::Range) -> Option { let parsed = parse_expression(literal).expect("a literal"); let expr: &ast::Expr = parsed.expr(); - pattern_range(expr, literal, pattern, span).map(|range| literal[range].to_string()) + pattern_range(expr, literal, false, pattern, span).map(|range| literal[range].to_string()) } #[test] @@ -233,6 +243,17 @@ mod tests { ); } + /// basedpython reads a lone triple-quoted string from the text its dedent + /// keeps, so a character of the pattern is found in that text + #[test] + fn a_dedented_literal_is_read_from_what_its_dedent_keeps() { + let literal = "\"\"\"\n a\n (b\n \"\"\""; + let parsed = parse_expression(literal).expect("a literal"); + let range = pattern_range(parsed.expr(), literal, true, "a\n(b", 2..3); + assert_eq!(range.map(|range| &literal[range]), Some("(")); + assert_eq!(range.map(|range| range.start().to_u32()), Some(14)); + } + #[test] fn a_line_continuation_holds_nothing() { assert_eq!(written("\"a\\\n(\"", "a(", 1..2).as_deref(), Some("(")); diff --git a/docs/basedpython/features/dedent-strings.md b/docs/basedpython/features/dedent-strings.md index 26c1c4e4f6..afc21cd5d2 100644 --- a/docs/basedpython/features/dedent-strings.md +++ b/docs/basedpython/features/dedent-strings.md @@ -15,6 +15,11 @@ def doc(): the logical content is `"hello\n world"` — the common leading indent of each interior line is stripped +that is the value the checker sees too: the literal type of `doc()`'s string is +`"hello\n world"`, a regular expression written this way is checked as the +pattern `re` will compile, and a name written this way in `__all__` is the name +exported + ## trigger conditions dedenting fires when *all* of these hold: @@ -73,3 +78,6 @@ pattern = r""" # \\d+ \"units\"\ # """ ``` + +a `\` ending the last line of the content of a string that is not raw is +rejected, since it would continue onto the newline the dedent drops From 90738ea18eba2e928623bd18d42756366572ab2d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:03:26 +1000 Subject: [PATCH 062/110] report a bad escape in a regex pattern, \q, [\A], an incomplete \x4, an octal escape past \377 and a backreference to a group not yet closed, by re's own escape rules for the python version checked against and for str or bytes, which has no \u, \U or \N, at the whole escape, instead of reading any backslash and the character after it as a valid escape --- .../resources/mdtest/basedpython_regex.md | 52 ++- .../src/types/infer/builder.rs | 8 +- crates/ty_python_semantic/src/types/regex.rs | 13 +- .../src/types/regex/parse.rs | 371 +++++++++++++++++- docs/basedpython/features/regex-groups.md | 7 + 5 files changed, 425 insertions(+), 26 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md index e362848da7..c6435425fd 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md @@ -326,6 +326,38 @@ re.findall("(", "") # error: [invalid-regex] "missing ), unterminated subpatter re.sub("(", "", "") # error: [invalid-regex] "missing ), unterminated subpattern at position 0" ``` +## bad escapes + +An ASCII letter or digit after a backslash that is not an escape `re` knows is an error, not the +letter: python keeps them for escapes a later version may add. A `bytes` pattern has no `\u`, `\U` +or `\N` escape, and a backreference has to name a group that is already closed. + +```py +import re + +re.compile(r"a\q") # error: [invalid-regex] "bad escape \q at position 1" +re.compile(r"[\A]") # error: [invalid-regex] "bad escape \A at position 1" +re.compile(r"\x4") # error: [invalid-regex] "incomplete escape \x4 at position 0" +re.compile(rb"\u0041") # error: [invalid-regex] "bad escape \u at position 0" +re.compile(r"(a)\2") # error: [invalid-regex] "invalid group reference 2 at position 4" +re.compile(r"(a\1)") # error: [invalid-regex] "cannot refer to an open group at position 2" + +re.compile(r"\A\d\w+[\b\s]\u0041\N{EM DASH}\x41\0(a)\1\Z") # no diagnostic +``` + +`\z` is an anchor from python 3.14, and a bad escape before it. + +```toml +[environment] +python-version = "3.13" +``` + +```py +import re + +re.compile(r"a\z") # error: [invalid-regex] "bad escape \z at position 1" +``` + ## an invalid pattern's error points at the character it is about python names the position in the pattern where it gave up, which is the character the error is @@ -401,6 +433,22 @@ error[invalid-regex]: redefinition of group name 'name' as group 2; was group 1 | ^^^^ ``` +a bad escape is all of the escape. + +```py +import re + +re.compile(r"a\q") # snapshot +``` + +```snapshot +error[invalid-regex]: bad escape \q at position 1 + --> src/mdtest_snippet.py:18:15 + | +18 | re.compile(r"a\q") # snapshot + | ^^ +``` + a pattern that is not written where it is passed is pointed at as what was passed. ```py @@ -412,9 +460,9 @@ re.compile(pattern) # snapshot ```snapshot error[invalid-regex]: missing ), unterminated subpattern at position 1 - --> src/mdtest_snippet.py:19:12 + --> src/mdtest_snippet.py:22:12 | -19 | re.compile(pattern) # snapshot +22 | re.compile(pattern) # snapshot | ^^^^^^^ ``` diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index ef80998798..0aabaf9af8 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -12398,7 +12398,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { }; (groups, any_str) } else { - let Some((text, any_str)) = regex::pattern_source(db, env, pattern_ty) else { + let Some((text, any_str, dialect)) = regex::pattern_source(db, env, pattern_ty) else { return; }; let flags = overload @@ -12413,7 +12413,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let Some(verbose) = self.regex_verbose_flag(flags) else { return; }; - match regex::analyze(&text, verbose) { + match regex::analyze(&text, verbose, dialect) { regex::PatternAnalysis::Groups(parsed) => { (regex::RegexGroups::from_parsed(db, &parsed), any_str) } @@ -12593,11 +12593,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if let Some(groups) = regex::groups_of(db, pattern_ty) { return Some(groups); } - let (text, _) = regex::pattern_source(db, env, pattern_ty)?; + let (text, _, dialect) = regex::pattern_source(db, env, pattern_ty)?; // no signature has been matched yet, so the `flags` parameter is // located by its position in `re.sub`/`re.subn` directly let verbose = self.regex_verbose_flag(arguments.find_argument_value("flags", 4))?; - match regex::analyze(&text, verbose) { + match regex::analyze(&text, verbose, dialect) { regex::PatternAnalysis::Groups(parsed) => { Some(regex::RegexGroups::from_parsed(db, &parsed)) } diff --git a/crates/ty_python_semantic/src/types/regex.rs b/crates/ty_python_semantic/src/types/regex.rs index 248976a246..9ea3cd0103 100644 --- a/crates/ty_python_semantic/src/types/regex.rs +++ b/crates/ty_python_semantic/src/types/regex.rs @@ -27,7 +27,7 @@ use crate::types::{ UnionType, }; -pub(crate) use parse::{PatternAnalysis, analyze}; +pub(crate) use parse::{Dialect, PatternAnalysis, analyze}; pub(crate) use source::pattern_range; /// one capture group of a statically-known pattern @@ -356,22 +356,27 @@ pub(crate) fn is_regex_class(known: Option) -> bool { } /// the pattern text of a literal `re` pattern argument, with the `str`/`bytes` -/// type the resulting match is over +/// type the resulting match is over and the dialect `re` reads it in pub(crate) fn pattern_source<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, -) -> Option<(String, Type<'db>)> { +) -> Option<(String, Type<'db>, Dialect)> { + let dialect = |bytes| Dialect { + version: env.python_version(db), + bytes, + }; match ty.as_literal_value_kind()? { LiteralValueTypeKind::String(literal) => Some(( literal.value(db).to_string(), KnownClass::Str.to_instance(db, env), + dialect(false), )), LiteralValueTypeKind::Bytes(literal) => { // read the bytes as latin-1 so one byte stays one character, which // keeps the offsets in python's own error messages right let text = literal.value(db).iter().copied().map(char::from).collect(); - Some((text, KnownClass::Bytes.to_instance(db, env))) + Some((text, KnownClass::Bytes.to_instance(db, env), dialect(true))) } _ => None, } diff --git a/crates/ty_python_semantic/src/types/regex/parse.rs b/crates/ty_python_semantic/src/types/regex/parse.rs index 4cdaa2d740..9e003e6658 100644 --- a/crates/ty_python_semantic/src/types/regex/parse.rs +++ b/crates/ty_python_semantic/src/types/regex/parse.rs @@ -12,6 +12,7 @@ use std::ops::Range; +use ruff_python_ast::PythonVersion; use ruff_python_ast::name::Name; use rustc_hash::FxHashMap; @@ -80,15 +81,25 @@ pub(crate) enum PatternAnalysis { Unknown, } +/// which `re` reads a pattern: the python version's, for a `str` or a `bytes` +/// pattern +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Dialect { + pub(crate) version: PythonVersion, + /// a `bytes` pattern has no `\u`, `\U` or `\N` escape + pub(crate) bytes: bool, +} + /// analyse `pattern` as python regular-expression source /// /// `verbose` is the `re.VERBOSE`/`re.X` flag from the call site; a pattern that /// turns verbose mode on itself with a global `(?x)` is re-parsed from the /// start, since the flag changes how the text before it tokenizes -pub(crate) fn analyze(pattern: &str, verbose: bool) -> PatternAnalysis { - match Parser::new(pattern, verbose).run() { +pub(crate) fn analyze(pattern: &str, verbose: bool, dialect: Dialect) -> PatternAnalysis { + match Parser::new(pattern, verbose, dialect).run() { Ok(groups) => PatternAnalysis::Groups(groups), - Err(Failure::RestartVerbose) if !verbose => match Parser::new(pattern, true).run() { + Err(Failure::RestartVerbose) if !verbose => match Parser::new(pattern, true, dialect).run() + { Ok(groups) => PatternAnalysis::Groups(groups), Err(Failure::Invalid(error)) => PatternAnalysis::Invalid(error), Err(_) => PatternAnalysis::Unknown, @@ -135,8 +146,15 @@ struct Parser { chars: Vec, pos: usize, verbose: bool, + dialect: Dialect, depth: u32, group_count: u32, + /// whether each group, by number less one, has been closed, which a + /// backreference to it needs + closed: Vec, + /// inside a lookbehind, the number of the first group opened in it, which no + /// backreference in it may refer to + lookbehind_groups: Option, /// group name to the number it was first bound to, for redefinition checks names: FxHashMap, /// group number to name, built as groups are opened @@ -144,13 +162,16 @@ struct Parser { } impl Parser { - fn new(pattern: &str, verbose: bool) -> Self { + fn new(pattern: &str, verbose: bool, dialect: Dialect) -> Self { Self { chars: pattern.chars().collect(), pos: 0, verbose, + dialect, depth: 0, group_count: 0, + closed: Vec::new(), + lookbehind_groups: None, names: FxHashMap::default(), group_names: Vec::new(), } @@ -367,7 +388,7 @@ impl Parser { Ok(Some(Node::Atom)) } Some('\\') => { - self.parse_escape()?; + self.parse_escape(false)?; Ok(Some(Node::Atom)) } Some(_) => { @@ -378,16 +399,195 @@ impl Parser { } } - fn parse_escape(&mut self) -> Result<(), Failure> { + /// consume an escape, `in_class` saying whether it is written in a `[…]` + /// + /// the rules are `sre_parse`'s `_escape` and `_class_escape`: an ASCII letter + /// or digit that is not an escape is an error, rather than the letter, so + /// that a later python can give it a meaning + fn parse_escape(&mut self, in_class: bool) -> Result<(), Failure> { let start = self.pos; self.pos += 1; - if self.peek().is_none() { + let Some(escaped) = self.peek() else { return Err(Failure::Invalid(RegexError::new( "bad escape (end of pattern)", start, ))); - } + }; self.pos += 1; + let text = !self.dialect.bytes; + match escaped { + // the escapes of special characters, `\b` a backspace in a set + '\\' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => Ok(()), + // the character classes, which a set may hold + 'd' | 'D' | 's' | 'S' | 'w' | 'W' => Ok(()), + // the anchors, which it may not + 'A' | 'B' | 'Z' if !in_class => Ok(()), + 'z' if !in_class && self.dialect.version >= PythonVersion::PY314 => Ok(()), + 'x' => self.hex_escape(start, 2), + 'u' if text => self.hex_escape(start, 4), + 'U' if text => { + self.hex_escape(start, 8)?; + // `chr` refuses a code point past unicode's last + if u32::from_str_radix(&self.text(start + 2..self.pos), 16) + .is_ok_and(|code| code <= 0x0010_ffff) + { + Ok(()) + } else { + Err(self.bad_escape(start)) + } + } + 'N' if text && self.dialect.version >= PythonVersion::PY38 => self.named_escape(), + '0'..='7' if in_class => { + self.eat_octal_digits(2); + self.octal_in_range(start) + } + '0' => { + self.eat_octal_digits(2); + Ok(()) + } + '1'..='9' if !in_class => self.group_reference(start), + escaped if escaped.is_ascii_alphanumeric() => Err(self.bad_escape(start)), + _ => Ok(()), + } + } + + fn bad_escape(&self, start: usize) -> Failure { + Failure::Invalid(RegexError::spanning( + format_args!("bad escape {}", self.text(start..self.pos)), + start, + start..self.pos, + )) + } + + fn text(&self, range: Range) -> String { + self.chars[range].iter().collect() + } + + /// the `digits` hex digits of a `\x`, `\u` or `\U` escape starting at `start` + fn hex_escape(&mut self, start: usize, digits: usize) -> Result<(), Failure> { + let first = self.pos; + while self.pos - first < digits && self.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + self.pos += 1; + } + if self.pos - first == digits { + Ok(()) + } else { + Err(Failure::Invalid(RegexError::spanning( + format_args!("incomplete escape {}", self.text(start..self.pos)), + start, + start..self.pos, + ))) + } + } + + /// the `{NAME}` of a `\N{NAME}`, whose name is not looked up: python's + /// unicode database is not ours, so a name it lacks is a missed error rather + /// than a valid pattern reported as invalid + fn named_escape(&mut self) -> Result<(), Failure> { + if !self.eat('{') { + return Err(Failure::Invalid(RegexError::new("missing {", self.pos))); + } + let name = self.pos; + loop { + match self.peek() { + None if self.pos == name => { + return Err(Failure::Invalid(RegexError::new( + "missing character name", + self.pos, + ))); + } + None => { + return Err(Failure::Invalid(RegexError::spanning( + "missing }, unterminated name", + name, + name..self.pos, + ))); + } + Some('}') if self.pos == name => { + return Err(Failure::Invalid(RegexError::new( + "missing character name", + self.pos, + ))); + } + Some('}') => { + self.pos += 1; + return Ok(()); + } + Some(_) => self.pos += 1, + } + } + } + + fn eat_octal_digits(&mut self, most: usize) { + for _ in 0..most { + if !self.peek().is_some_and(|c| c.is_digit(8)) { + break; + } + self.pos += 1; + } + } + + /// an octal escape starting at `start` names a byte at most + fn octal_in_range(&self, start: usize) -> Result<(), Failure> { + let digits = self.text(start + 1..self.pos); + if u32::from_str_radix(&digits, 8).is_ok_and(|value| value <= 0o377) { + Ok(()) + } else { + Err(Failure::Invalid(RegexError::spanning( + format_args!( + "octal escape value {} outside of range 0-0o377", + self.text(start..self.pos) + ), + start, + start..self.pos, + ))) + } + } + + /// `\1` to `\99`, a backreference, unless it is three octal digits, which + /// is an octal escape + fn group_reference(&mut self, start: usize) -> Result<(), Failure> { + if self.peek().is_some_and(|c| c.is_ascii_digit()) { + self.pos += 1; + if self.chars[start + 1..self.pos] + .iter() + .all(|c| c.is_digit(8)) + && self.peek().is_some_and(|c| c.is_digit(8)) + { + self.pos += 1; + return self.octal_in_range(start); + } + } + let group = self + .text(start + 1..self.pos) + .parse::() + .unwrap_or(u32::MAX); + if group > self.group_count { + return Err(Failure::Invalid(RegexError::spanning( + format_args!("invalid group reference {group}"), + start + 1, + start + 1..self.pos, + ))); + } + if !self + .closed + .get(group as usize - 1) + .copied() + .unwrap_or_default() + { + return Err(Failure::Invalid(RegexError::spanning( + "cannot refer to an open group", + start, + start..self.pos, + ))); + } + if self.lookbehind_groups.is_some_and(|first| group >= first) { + return Err(Failure::Invalid(RegexError::spanning( + "cannot refer to group defined in the same lookbehind subpattern", + self.pos, + start..self.pos, + ))); + } Ok(()) } @@ -413,7 +613,7 @@ impl Parser { self.pos += 1; return Ok(()); } - Some('\\') => self.parse_escape()?, + Some('\\') => self.parse_escape(true)?, Some(_) => self.pos += 1, } } @@ -435,7 +635,7 @@ impl Parser { if !self.eat('?') { let index = self.open_capturing_group(None, open)?; let node = self.parse_alternation()?; - self.close_group(open)?; + self.close_capturing_group(open, index)?; return Ok(Some(Node::Group { index: Some(index), node: Box::new(node), @@ -462,8 +662,13 @@ impl Parser { Some('=' | '!') => { let negative = self.peek_at(1) == Some('!'); self.pos += 2; - self.parse_plain_group(open, None) - .map(|node| Some(uncertain_if(node, negative))) + // the groups a backreference inside may not refer to are the + // ones opened in the outermost lookbehind + let outer = self.lookbehind_groups; + self.lookbehind_groups = outer.or(Some(self.group_count + 1)); + let node = self.parse_plain_group(open, None); + self.lookbehind_groups = outer; + node.map(|node| Some(uncertain_if(node, negative))) } Some(char) => Err(Failure::Invalid(RegexError::new( format_args!("unknown extension ?<{char}"), @@ -531,6 +736,14 @@ impl Parser { } } + fn close_capturing_group(&mut self, open: usize, index: u32) -> Result<(), Failure> { + self.close_group(open)?; + if let Some(closed) = self.closed.get_mut(index as usize - 1) { + *closed = true; + } + Ok(()) + } + /// `(?P…)` and `(?P=name)` fn parse_named_extension(&mut self, open: usize) -> Result, Failure> { self.pos += 1; @@ -541,7 +754,7 @@ impl Parser { let name = Self::validate_group_name(&name, start)?; let index = self.open_capturing_group(Some(name), start)?; let node = self.parse_alternation()?; - self.close_group(open)?; + self.close_capturing_group(open, index)?; Ok(Some(Node::Group { index: Some(index), node: Box::new(node), @@ -724,6 +937,7 @@ impl Parser { ))); } self.group_names.push(name); + self.closed.push(false); Ok(index) } } @@ -793,15 +1007,26 @@ fn collect_certainty(node: &Node, certain: bool, out: &mut [bool]) { #[cfg(test)] mod tests { - use super::{ParsedGroup, PatternAnalysis, analyze}; + use ruff_python_ast::PythonVersion; + + use super::{Dialect, ParsedGroup, PatternAnalysis, analyze}; + + const STR: Dialect = Dialect { + version: PythonVersion::PY313, + bytes: false, + }; /// `!` marks an optional group, `?` an unnamed one, e.g. `"?, !name"` fn shape(pattern: &str) -> String { - shape_with(pattern, false) + shape_in(pattern, false, STR) } fn shape_with(pattern: &str, verbose: bool) -> String { - match analyze(pattern, verbose) { + shape_in(pattern, verbose, STR) + } + + fn shape_in(pattern: &str, verbose: bool, dialect: Dialect) -> String { + match analyze(pattern, verbose, dialect) { PatternAnalysis::Groups(groups) => groups .iter() .map( @@ -1080,6 +1305,120 @@ mod tests { ); } + /// what python 3.13's `re` says of each, for a `str` pattern, and of the + /// same pattern as `bytes` + #[test] + fn escapes() { + let bytes = Dialect { bytes: true, ..STR }; + let both = |pattern: &str, expected: &str| { + assert_eq!(shape(pattern), expected, "{pattern} as str"); + assert_eq!( + shape_in(pattern, false, bytes), + expected, + "{pattern} as bytes" + ); + }; + both(r"\q", r"invalid: bad escape \q at position 0"); + both(r"a\q", r"invalid: bad escape \q at position 1"); + both(r"[\q]", r"invalid: bad escape \q at position 1"); + both(r"[\z]", r"invalid: bad escape \z at position 1"); + both(r"[\A]", r"invalid: bad escape \A at position 1"); + both(r"[\8]", r"invalid: bad escape \8 at position 1"); + both(r"\x4", r"invalid: incomplete escape \x4 at position 0"); + both(r"\xg", r"invalid: incomplete escape \x at position 0"); + both( + r"\777", + r"invalid: octal escape value \777 outside of range 0-0o377 at position 0", + ); + both( + r"[\777]", + r"invalid: octal escape value \777 outside of range 0-0o377 at position 1", + ); + both(r"\8", "invalid: invalid group reference 8 at position 1"); + both(r"\12", "invalid: invalid group reference 12 at position 1"); + both(r"(a)\2", "invalid: invalid group reference 2 at position 4"); + both( + r"(a\1)", + "invalid: cannot refer to an open group at position 2", + ); + both( + r"(?<=(a)\1)", + "invalid: cannot refer to group defined in the same lookbehind subpattern at position 9", + ); + for valid in [ + r"\A\b\B\d\D\s\S\w\W\Z", + r"[\b\d\D\s\S\w\W]", + r"\a\f\n\r\t\v\\", + r"\x41", + r"\0", + r"\07777", + r"\177", + r"[\1]", + r"\-", + r"[\-]", + r"\_", + r"\ ", + "\\\u{e9}", + ] { + both(valid, ""); + } + both(r"(a)\1", "?"); + both(r"(a)(?<=\1)", "?"); + + // the escapes a `bytes` pattern does not have + assert_eq!(shape(r"\u0041\U0010ffff"), ""); + assert_eq!(shape(r"\N{EM DASH}"), ""); + assert_eq!( + shape_in(r"\u0041", false, bytes), + r"invalid: bad escape \u at position 0" + ); + assert_eq!( + shape_in(r"\U0010ffff", false, bytes), + r"invalid: bad escape \U at position 0" + ); + assert_eq!( + shape_in(r"\N{EM DASH}", false, bytes), + r"invalid: bad escape \N at position 0" + ); + assert_eq!( + shape(r"\u12"), + r"invalid: incomplete escape \u12 at position 0" + ); + assert_eq!( + shape(r"\U0011ffff"), + r"invalid: bad escape \U0011ffff at position 0" + ); + assert_eq!(shape(r"\N"), "invalid: missing { at position 2"); + assert_eq!(shape(r"\Nx"), "invalid: missing { at position 2"); + assert_eq!( + shape(r"\N{"), + "invalid: missing character name at position 3" + ); + assert_eq!( + shape(r"\N{}"), + "invalid: missing character name at position 3" + ); + assert_eq!( + shape(r"\N{ab"), + "invalid: missing }, unterminated name at position 3" + ); + } + + /// `\z` is an anchor from python 3.14, and a bad escape before it + #[test] + fn escapes_by_version() { + assert_eq!(shape(r"\z"), r"invalid: bad escape \z at position 0"); + let py314 = Dialect { + version: PythonVersion::PY314, + ..STR + }; + assert_eq!(shape_in(r"\z", false, py314), ""); + assert_eq!( + shape_in(r"[\z]", false, py314), + r"invalid: bad escape \z at position 1" + ); + } + #[test] fn deeply_nested_patterns_are_unknown_not_invalid() { let pattern = format!("{}{}", "(".repeat(200), ")".repeat(200)); diff --git a/docs/basedpython/features/regex-groups.md b/docs/basedpython/features/regex-groups.md index 64a31e340f..3a60c287c1 100644 --- a/docs/basedpython/features/regex-groups.md +++ b/docs/basedpython/features/regex-groups.md @@ -120,6 +120,13 @@ escape, a pattern written as adjacent literals at the literal that has the character, and an error about a construct as a whole, `{2,1}` or a group's redefined name, at all of it. a pattern passed by name is reported at the name. +escapes are read by `re`'s own rules for the python version checked against: an +ASCII letter or digit after a backslash that is no escape, `\q`, is a bad escape +rather than the letter, a `bytes` pattern has no `\u`, `\U` or `\N`, `\z` is an +anchor only from 3.14, and a backreference has to name a group already closed. +each is reported at the whole escape. the name in a `\N{...}` is not looked up, +so a name python's unicode database lacks goes unreported + a construct the pattern reader does not model produces neither a refinement nor a diagnostic: a pattern reported invalid when `re.compile` would have accepted it is much worse than one left alone. From 90ddc4347e3d71bd9d656af8d65c70d2cedde554 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:06:49 +1000 Subject: [PATCH 063/110] answer by/prepareMoveSymbol and by/moveSymbol anywhere in a top-level definition, its body included, with that definition, as a move from inside a function moves the function, instead of null where the position names nothing movable --- crates/ty_ide/src/refactor/move_symbol.rs | 84 +++++++++++++++++++++-- crates/ty_server/tests/e2e/move_symbol.rs | 9 ++- docs/basedpython/features/editor.md | 20 ++++-- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/crates/ty_ide/src/refactor/move_symbol.rs b/crates/ty_ide/src/refactor/move_symbol.rs index bf064db5d0..c7d1f70c3d 100644 --- a/crates/ty_ide/src/refactor/move_symbol.rs +++ b/crates/ty_ide/src/refactor/move_symbol.rs @@ -38,7 +38,7 @@ use ruff_python_ast::token::TokenKind; use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; use ruff_python_ast::{self as ast, AnyNodeRef, Expr, Stmt}; use ruff_python_codegen::Stylist; -use ruff_source_file::LineRanges; +use ruff_source_file::{LineRanges, UniversalNewlines}; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustc_hash::{FxHashMap, FxHashSet}; use ty_module_resolver::{ImportingFile, ModuleName, file_to_module}; @@ -87,9 +87,9 @@ pub struct SymbolMove { pub warnings: Vec, } -/// The top-level definition named at `offset` — at its definition or at a reference to it — when -/// it is one [`move_symbol`] can move: a function, a class, or a variable a single name is -/// assigned to, at the top level of a module of the project. +/// the top-level definition named at `offset` — at its definition or at a reference to it — or +/// else the one `offset` is anywhere in, when it is one [`move_symbol`] can move: a function, a +/// class, or a variable a single name is assigned to, at the top level of a module of the project pub fn movable_symbol( db: &dyn Db, file: ProgramFile<'_>, @@ -502,7 +502,9 @@ fn disjoint(db: &dyn Db, edits: Vec) -> Result, String> } /// The file and offset of the top-level declaration named at `offset`: `offset` itself when it is -/// in a declaration's name, or else where the name at `offset` is defined. +/// in a declaration's name, or else where the name at `offset` is defined, or else the name of the +/// top-level declaration `offset` is anywhere in, as a move from inside a definition's body moves +/// the definition fn declaration_site<'db>( db: &'db dyn Db, file: ProgramFile<'db>, @@ -512,6 +514,19 @@ fn declaration_site<'db>( if Declaration::at(parsed.suite(), offset).is_some() { return Some((file, offset)); } + if let Some(site) = referenced_declaration(db, file, offset) { + return Some(site); + } + let declaration = Declaration::enclosing(parsed.suite(), offset)?; + Some((file, declaration.name_range.start())) +} + +/// the file and offset of the top-level declaration of the project the name at `offset` refers to +fn referenced_declaration<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + offset: TextSize, +) -> Option<(ProgramFile<'db>, TextSize)> { let targets = crate::goto_definition(db, file, offset)?; let mut targets = targets.value.into_iter(); let target = targets.next()?; @@ -537,6 +552,22 @@ struct Declaration<'a> { } impl<'a> Declaration<'a> { + /// the top-level declaration `offset` is anywhere in: its decorators, its signature, its body + /// or its value + fn enclosing(suite: &'a [Stmt], offset: TextSize) -> Option { + let statement = suite + .iter() + .find(|statement| statement.range().contains_inclusive(offset))?; + let name = match statement { + Stmt::FunctionDef(function) => function.name.start(), + Stmt::ClassDef(class) => class.name.start(), + Stmt::Assign(assign) => assign.targets.first()?.start(), + Stmt::AnnAssign(assign) => assign.target.start(), + _ => return None, + }; + Self::at(suite, name) + } + /// The top-level declaration whose name `offset` is in. fn at(suite: &'a [Stmt], offset: TextSize) -> Option { suite.iter().enumerate().find_map(|(index, statement)| { @@ -1054,7 +1085,13 @@ fn appended_text( for _ in trailing_blank_lines.min(blank_lines)..blank_lines { text.push_str(line_ending); } - text.push_str(definition); + // the definition is written in the line ending of the module it was cut from + for line in definition.universal_newlines() { + text.push_str(line.as_str()); + if line.line_ending().is_some() { + text.push_str(line_ending); + } + } text } @@ -1574,6 +1611,41 @@ X = os.sep assert_snapshot!(test.move_to_dst(), @"refused: mv.src and mv.dst would import each other: `area` uses `RATE`, and mv.src still uses `area`"); } + /// anywhere in a definition that is not a name of something else, a move moves the + /// definition, as it does from its name + #[test] + fn a_move_from_inside_a_body_moves_the_definition() { + let inside = + project(&SRC.replace(" return OrderedDict", " return OrderedDict")); + let at_name = project(&SRC.replace("def free(x)", "def free(x)")); + assert_eq!(inside.move_to_dst(), at_name.move_to_dst()); + assert!(inside.move_to_dst().contains("def free(x)")); + } + + /// a definition moved out of a module written with `\r\n` line endings into one written with + /// `\n` is written with `\n`, as the rest of the target is + #[test] + fn a_moved_definition_takes_the_targets_line_endings() { + let test = project( + &SRC.replace("def free(x)", "def free(x)") + .replace('\n', "\r\n"), + ); + let moved = test.move_to_dst(); + let dst_path = system_path_to_file(&test.db, "mv/dst.py") + .expect("dst to exist") + .path(&test.db) + .to_string(); + let dst = moved + .split("--- ") + .find(|file| file.starts_with(&dst_path)) + .expect("the move to change mv/dst.py"); + assert!( + dst.contains("def free(x):\n return OrderedDict(x=x)\n"), + "{dst:?}" + ); + assert!(!dst.contains('\r'), "{dst:?}"); + } + #[test] fn a_definition_that_reads_a_private_name_is_refused() { let test = project(&SRC.replace("def uses_private", "def uses_private")); diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index 3644fe112a..4d0486001b 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -7,6 +7,7 @@ use anyhow::Result; use lsp_types::{LspRequestMethod, MessageDirection, Request}; use ruff_db::system::SystemPath; +use ruff_source_file::LineEnding; use serde_json::json; use crate::{TestServer, TestServerBuilder}; @@ -113,16 +114,20 @@ fn a_move_answers_the_edit_to_every_file_it_changes() -> Result<()> { }]), "{answer:#}" ); + // geometry.py is empty, so what the move writes there takes the platform's line ending + let newline = LineEnding::default().as_str(); assert_eq!( changes[geometry.as_str()], json!([ { "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, - "newText": "from shapes.util import RATE\n", + "newText": format!("from shapes.util import RATE{newline}"), }, { "range": { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 } }, - "newText": "\n\ndef area(r: float) -> float:\n return r * r * RATE\n", + "newText": format!( + "{newline}{newline}def area(r: float) -> float:{newline} return r * r * RATE{newline}" + ), }, ]), "{answer:#}" diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 77b3a38f1f..d24b9b7517 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -166,7 +166,7 @@ a name the file has not imported is offered for import only when what you have typed starts where one of its words does — at its start, after a `_`, a `.` or any other character that is not a letter or a digit, at an upper-case letter that follows a lower-case one, at the last of a run of capitals that a lower-case letter -follows (`ser` offers `HTTPServer`), or at a letter that follows a digit (`sum` +follows (`set` offers `HTTPServer`), or at a letter that follows a digit (`sum` offers `md5sum`) — and the rest follows in order. `ret` offers `reveal_type` and `NoReturn` to import, but not `secrets` or `urlretrieve`, which merely contain the letters, and which would otherwise crowd out `return` and the names in scope @@ -216,13 +216,17 @@ written through it, and one from a module with a `from ... import` is added to t statement. how imports are sorted, grouped and split is not an editor setting: it is the project's isort configuration, which `buff` reads and fixes -an import any of these write — a completion's, a quick fix's, a pasted name's, an -inlay hint's — goes where `buff check --select I --fix` would put it, so adding one -never leaves the block unsorted. it joins its section of the file's first block of +an import any of these write — a completion's, an unresolved name's quick fix's, a +pasted name's, an inlay hint's — goes where `buff check --select I --fix` would put +it, so adding one never leaves the block unsorted. it joins its section of the file's first block of imports (future, standard library, third party, first party, local, or the sections the configuration defines) in the order isort sorts that section, with a blank line around a section the block did not have yet, and a member added to an -existing `from` import is listed where isort orders the members: +existing `from` import is listed where isort orders the members. a new statement, +and a `from` import a new member makes too long, is wrapped to the line length as +isort wraps it, a section new to the block gets the heading `import-heading` +configures for it, and two members of a module the file does not import yet become +one `from` import: ```py import json @@ -241,7 +245,8 @@ resolves it — the closest `ruff.toml`, `.ruff.toml` or `pyproject.toml` with a `no-lines-before`, `lines-between-types`, `force-single-line` and the rest apply as they do to `buff`. the import has to run before the name it binds is used, so a file whose imports all come after that point, or that has none, still gets it at -the top, below a docstring and any `from __future__` import. `buff` sorts each cell +the top, below a docstring and any `from __future__` import, followed by the blank +lines isort writes after a block. `buff` sorts each cell of a notebook on its own, so in a notebook the block an import joins, and the top it otherwise goes at, are those of the cell the name is used in @@ -733,7 +738,8 @@ the move is refused, with the reason, when the program would not run as before: nor python into basedpython, which reads it differently `by/prepareMoveSymbol` takes `textDocument` and `position` — in a top-level -definition's name, or in a reference to one — and answers `{name, kind, module, uri, range}`: the name, `function`, `class` or `variable`, the dotted name of the module +definition's name, in a reference to one, or anywhere else in a top-level +definition, its body included, which is then the one moved — and answers `{name, kind, module, uri, range}`: the name, `function`, `class` or `variable`, the dotted name of the module it is defined in, and its name's range in the file `uri`, which need not be the document asked about; `null` where there is nothing to move. `by/moveSymbol` takes the same and `target`, the URI of an existing `.py` or `.by` file of the project From 8f9a4dbec27bbf46fa6c8d01a90507968efe6842 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:10:11 +1000 Subject: [PATCH 064/110] register for workspace/didCreateFiles on every module and folder and take each file it reports into the session, so a module an editor has just created to move a definition into is a by/moveSymbol target as soon as it says so, rather than a file that does not exist wherever the server had looked for it before --- .../ruff_linter/src/rules/isort/placement.rs | 42 ++++++------- crates/ruff_python_importer/src/order.rs | 8 +-- crates/ty_ide/src/diff_ignored.rs | 18 +++--- crates/ty_ide/src/import_order.rs | 62 +++++++++---------- crates/ty_ide/src/importer.rs | 54 ++++++++-------- crates/ty_ide/src/lib.rs | 2 +- crates/ty_ide/src/merge.rs | 8 +-- crates/ty_python_semantic/src/importer.rs | 48 +++++++------- crates/ty_server/src/capabilities.rs | 14 +++++ crates/ty_server/src/server/api.rs | 5 +- .../ty_server/src/server/api/notifications.rs | 2 + .../api/notifications/did_create_files.rs | 51 +++++++++++++++ .../server/api/requests/merge_conflicts.rs | 20 +++--- crates/ty_server/tests/e2e/merge_conflicts.rs | 2 +- crates/ty_server/tests/e2e/move_symbol.rs | 43 ++++++++++++- .../e2e__initialize__initialization.snap | 18 ++++++ ...ialize__initialization_with_workspace.snap | 18 ++++++ docs/basedpython/features/editor.md | 11 +++- 18 files changed, 290 insertions(+), 136 deletions(-) create mode 100644 crates/ty_server/src/server/api/notifications/did_create_files.rs diff --git a/crates/ruff_linter/src/rules/isort/placement.rs b/crates/ruff_linter/src/rules/isort/placement.rs index 6e65628ae5..c40d7dbef1 100644 --- a/crates/ruff_linter/src/rules/isort/placement.rs +++ b/crates/ruff_linter/src/rules/isort/placement.rs @@ -5,7 +5,7 @@ //! leaves every other line alone. to leave the block as `I001` would write it, the statement //! has to land in its section, in the order isort sorts that section, separated from its //! neighbours by the blank lines isort puts there. [`ImportOrder`] answers those questions -//! with the same keys, sections and settings [`super::format_imports`] sorts with, so the two +//! with the same keys, sections and settings `format_imports` sorts with, so the two //! cannot disagree use std::cell::RefCell; @@ -47,12 +47,12 @@ pub struct ImportPosition<'a> { impl ImportPosition<'_> { /// the index of the statement's section in the configured section order - pub fn section(&self) -> usize { + fn section(&self) -> usize { self.section } /// the position, owning the names it compares - pub fn into_owned(self) -> ImportPosition<'static> { + fn into_owned(self) -> ImportPosition<'static> { ImportPosition { section: self.section, group: self.group, @@ -62,8 +62,8 @@ impl ImportPosition<'_> { } } -/// The ordering isort applies to the imports of one file, looking modules up in `files` to tell -/// whether they are first-party. +/// the ordering isort applies to the imports of one file, looking modules up in `files` to tell +/// whether they are first-party pub struct ImportOrder { settings: Arc, package: Option, @@ -76,7 +76,7 @@ pub struct ImportOrder { impl ImportOrder { /// the ordering for a file in `package` (as `detect_package_root` finds it), under the - /// project's linter `settings` and the Python version it targets. + /// project's linter `settings` and the Python version it targets pub fn new( settings: Arc, package: Option, @@ -96,7 +96,7 @@ impl ImportOrder { &self.settings.isort } - /// The index of the section `module_name` sorts in, in the configured section order. + /// the index of the section `module_name` sorts in, in the configured section order fn section_index(&self, module_name: String, is_relative: bool) -> usize { if let Some(section) = self.sections.borrow().get(&module_name) { return *section; @@ -124,7 +124,7 @@ impl ImportOrder { } /// where `import` sorts - pub fn position<'b>(&self, import: &SortedImport<'b>) -> ImportPosition<'b> { + fn position<'b>(&self, import: &SortedImport<'b>) -> ImportPosition<'b> { let settings = self.isort(); let (module_name, is_relative) = match import { SortedImport::Straight { name, .. } => ((*name).to_string(), false), @@ -175,7 +175,7 @@ impl ImportOrder { /// how many blank lines isort writes between a statement at `before` and the statement at /// `after` that follows it - pub fn blank_lines_between(&self, before: &ImportPosition, after: &ImportPosition) -> usize { + fn blank_lines_between(&self, before: &ImportPosition, after: &ImportPosition) -> usize { let settings = self.isort(); if before.section != after.section { // `format_import_block` separates two sections with a blank line unless every @@ -206,15 +206,15 @@ impl ImportOrder { /// the comment isort writes above the first import of the section at `position`, if one is /// configured - pub fn section_heading(&self, position: &ImportPosition) -> Option<&str> { + fn section_heading(&self, position: &ImportPosition) -> Option<&str> { let settings = self.isort(); let section = settings.section_order.get(position.section)?; settings.import_headings.get(section).map(String::as_str) } - /// Whether `comment` is the heading isort writes above a section, which it takes into the - /// block when it is directly above the block's first import. - pub fn is_heading(&self, comment: &str) -> bool { + /// whether `comment` is the heading isort writes above a section, which it takes into the + /// block when it is directly above the block's first import + fn is_heading(&self, comment: &str) -> bool { self.isort() .import_headings .values() @@ -222,7 +222,7 @@ impl ImportOrder { } /// how isort orders two members of one `from` import - pub fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { + fn compare_members(&self, a: (&str, Option<&str>), b: (&str, Option<&str>)) -> Ordering { let settings = self.isort(); MemberKey::from_member(a.0, a.1, settings).cmp(&MemberKey::from_member(b.0, b.1, settings)) } @@ -230,16 +230,16 @@ impl ImportOrder { /// whether isort writes `member` (imported `as asname`, if given) from `module` in a `from` /// import of its own rather than among the members of an existing one, under /// `force-single-line` or, for an aliased member, without `combine-as-imports` - pub fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { + fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool { let settings = self.isort(); (settings.force_single_line && module.is_none_or(|module| !settings.single_line_exclusions.contains(module))) || (asname.is_some() && !settings.combine_as_imports) } - /// How many blank lines isort writes between the last import of a block and the statement - /// after it, as `format_imports` writes them after a block in a source file. - pub fn blank_lines_after_block(&self, before_definition: bool) -> usize { + /// how many blank lines isort writes between the last import of a block and the statement + /// after it, as `format_imports` writes them after a block in a source file + fn blank_lines_after_block(&self, before_definition: bool) -> usize { match usize::try_from(self.isort().lines_after_imports) { Ok(lines) => lines, Err(_) if before_definition => 2, @@ -248,8 +248,8 @@ impl ImportOrder { } /// `source`, a run of import statements, as isort writes them in `stylist`'s style, without - /// the headings of their sections; see [`sort_import_statements`]. - pub fn write_statements(&self, source: &str, stylist: &Stylist) -> Option { + /// the headings of their sections; see [`sort_import_statements`] + fn write_statements(&self, source: &str, stylist: &Stylist) -> Option { let written = write_import_statements( source, &self.settings, @@ -313,7 +313,7 @@ impl ImportOrdering for ImportOrder { /// writes an import block with the same settings: one block, deduplicated, `from` imports of one /// module combined, in sections, each wrapped to the line length. `None` for source with /// anything other than import statements in it, or a comment, which isort keeps beside the import -/// it is on and this cannot place. +/// it is on and this cannot place pub fn sort_import_statements( source: &str, settings: &LinterSettings, diff --git a/crates/ruff_python_importer/src/order.rs b/crates/ruff_python_importer/src/order.rs index f284d409e9..3a60a8a7d1 100644 --- a/crates/ruff_python_importer/src/order.rs +++ b/crates/ruff_python_importer/src/order.rs @@ -55,8 +55,8 @@ pub trait ImportOrdering { /// is configured fn section_heading(&self, position: &Self::Position) -> Option<&str>; - /// Whether `comment`, a whole comment line, is a heading the sorter writes above a section. - /// The sorter rewrites the headings directly above the first import of a block. + /// whether `comment`, a whole comment line, is a heading the sorter writes above a section. + /// the sorter rewrites the headings directly above the first import of a block fn is_heading(&self, comment: &str) -> bool; /// how the sorter orders two members of one `from` import, each a name and its alias @@ -66,8 +66,8 @@ pub trait ImportOrdering { /// `from` import of its own rather than among the members of an existing one fn imports_member_alone(&self, module: Option<&str>, asname: Option<&str>) -> bool; - /// How many blank lines the sorter writes between the last import of a block and the - /// statement after it, a function or class definition or another statement. + /// how many blank lines the sorter writes between the last import of a block and the + /// statement after it, a function or class definition or another statement fn blank_lines_after_block(&self, before_definition: bool) -> usize; /// `source`, a run of import statements, as the sorter writes them in `stylist`'s style, diff --git a/crates/ty_ide/src/diff_ignored.rs b/crates/ty_ide/src/diff_ignored.rs index 9f136ea8d3..3e773c2c33 100644 --- a/crates/ty_ide/src/diff_ignored.rs +++ b/crates/ty_ide/src/diff_ignored.rs @@ -1,12 +1,12 @@ -//! The parts of a file a diff may ignore when asked to ignore imports and formatting. +//! the parts of a file a diff may ignore when asked to ignore imports and formatting //! -//! Imports are every import statement, wherever it is. Formatting is the text python's tokenizer +//! imports are every import statement, wherever it is. formatting is the text python's tokenizer //! discards without its being significant: whitespace between two tokens of a logical line, the //! line breaks and indentation of a line continued inside brackets or after a backslash, and blank -//! lines. The indentation a logical line starts with is not formatting, since it says which block +//! lines. the indentation a logical line starts with is not formatting, since it says which block //! the line is in, and nor is anything inside an f-string or a t-string, where `f"{x = }"` prints -//! the spaces around its `=`. Two versions that differ only in what is ignored here tokenize to -//! the same tokens. +//! the spaces around its `=`. two versions that differ only in what is ignored here tokenize to +//! the same tokens use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; use ruff_python_ast::token::TokenKind; @@ -14,12 +14,12 @@ use ruff_python_ast::{PySourceType, Stmt}; use ruff_python_parser::parse_unchecked_source; use ruff_text_size::{Ranged, TextRange, TextSize}; -/// What a diff ignoring imports and formatting ignores in one text. +/// what a diff ignoring imports and formatting ignores in one text #[derive(Debug, Clone, PartialEq, Eq)] pub struct DiffIgnoredRanges { - /// Each import statement. + /// each import statement pub imports: Vec, - /// Each run of text that is only formatting, in order. + /// each run of text that is only formatting, in order pub formatting: Vec, } @@ -101,7 +101,7 @@ mod tests { use super::diff_ignored_ranges; - /// `text` with what is ignored as formatting written as `~`, and imports as `I`. + /// `text` with what is ignored as formatting written as `~`, and imports as `I` fn marked(text: &str) -> String { let ignored = diff_ignored_ranges(text, PySourceType::BasedPython); let mut out: Vec = text.chars().collect(); diff --git a/crates/ty_ide/src/import_order.rs b/crates/ty_ide/src/import_order.rs index f119f862f2..e1938153d7 100644 --- a/crates/ty_ide/src/import_order.rs +++ b/crates/ty_ide/src/import_order.rs @@ -7,18 +7,18 @@ //! defaults with the `target-version` the closest `requires-python` implies. this resolves it //! with the same `ruff_workspace` code, so the two agree on sections, known modules and order //! -//! The configuration files are read through the database's [`System`], as everything else the +//! the configuration files are read through the database's [`System`], as everything else the //! project reads is, but not tracked by it: they need not be in a directory the server watches, -//! as the user's own configuration is not. A resolved configuration is cached instead with the +//! as the user's own configuration is not. a resolved configuration is cached instead with the //! revision of every file resolving it looked at, including the ones it found missing, and used //! again only while each is unchanged, so an edit to a configuration file, to a file it //! `extend`s, or a configuration file created closer to the file is seen by the next import //! added //! -//! Whether a module is the project's own is decided, as buff decides it, by looking for it in -//! the configured `src` directories. That lookup goes through the database's files, which keep +//! whether a module is the project's own is decided, as buff decides it, by looking for it in +//! the configured `src` directories. that lookup goes through the database's files, which keep //! what they found until the server reports a change, so a completion offering hundreds of -//! imports doesn't look at the file system for each. +//! imports doesn't look at the file system for each use std::cell::RefCell; use std::io; @@ -57,8 +57,8 @@ pub(crate) fn importer<'a>( Importer::new(db, file, parsed).sorted_by(move || order(db, file)) } -/// The order buff's isort configuration sorts the imports of `file` in, or `None` for a file -/// that is not on disk. A notebook's cells are each sorted in this order on their own +/// the order buff's isort configuration sorts the imports of `file` in, or `None` for a file +/// that is not on disk. a notebook's cells are each sorted in this order on their own fn order<'a>(db: &'a dyn Db, file: ProgramFile<'a>) -> Option>> { let path = file.file(db).path(db).as_system_path()?.as_std_path(); let files = ProjectFiles(db); @@ -109,7 +109,7 @@ impl<'a> IsortConfig<'a> { Self::new(system, &SystemFiles(system), directory, None, project_root) } - /// The configuration of a file in `directory`, its package found in `files`. + /// the configuration of a file in `directory`, its package found in `files` fn new( system: &'a dyn System, files: &dyn PackageFiles, @@ -140,7 +140,7 @@ impl<'a> IsortConfig<'a> { } /// `source`, a run of import statements, as buff's `I001` fix sorts and writes them, or - /// `None` for source with anything else in it, a comment among them. + /// `None` for source with anything else in it, a comment among them pub(crate) fn sort(&self, source: &str, source_type: PySourceType) -> Option { sort_import_statements_in( source, @@ -153,8 +153,8 @@ impl<'a> IsortConfig<'a> { } } -/// The files of the database, which remember what they found at a path until the server reports -/// a change to it. +/// the files of the database, which remember what they found at a path until the server reports +/// a change to it pub(crate) struct ProjectFiles<'db>(&'db dyn Db); impl PackageFiles for ProjectFiles<'_> { @@ -173,7 +173,7 @@ impl PackageFiles for ProjectFiles<'_> { } } -/// The files of a [`System`], looked at each time they are asked about. +/// the files of a [`System`], looked at each time they are asked about struct SystemFiles<'a>(&'a dyn System); impl PackageFiles for SystemFiles<'_> { @@ -186,8 +186,8 @@ impl PackageFiles for SystemFiles<'_> { } } -/// The configuration files of a [`System`], recording the revision of every file looked at: -/// `None` for one that is missing. +/// the configuration files of a [`System`], recording the revision of every file looked at: +/// `None` for one that is missing struct RecordedFiles<'a> { system: &'a dyn System, revisions: RefCell)>>, @@ -235,7 +235,7 @@ impl ConfigurationFiles for RecordedFiles<'_> { } } -/// The revision of the file at `path`, or `None` when there is none. +/// the revision of the file at `path`, or `None` when there is none fn file_revision(system: &dyn System, path: &Path) -> Option { let metadata = system .path_metadata(SystemPath::from_std_path(path)?) @@ -243,8 +243,8 @@ fn file_revision(system: &dyn System, path: &Path) -> Option { metadata.file_type().is_file().then(|| metadata.revision()) } -/// Sets the target version the closest `requires-python` implies, which takes precedence over -/// the user's own configuration, as buff gives it. +/// sets the target version the closest `requires-python` implies, which takes precedence over +/// the user's own configuration, as buff gives it struct FallbackTargetVersion(Option); impl ConfigurationTransformer for FallbackTargetVersion { @@ -264,21 +264,21 @@ impl ConfigurationTransformer for Unchanged { } } -/// The settings resolved for a directory, and the revision of every file resolving them looked -/// at. +/// the settings resolved for a directory, and the revision of every file resolving them looked +/// at struct Resolved { revisions: Vec<(PathBuf, Option)>, settings: Arc, } -/// The resolved settings, by the directory, project root and user configuration directory they -/// were resolved for. +/// the resolved settings, by the directory, project root and user configuration directory they +/// were resolved for static RESOLVED: LazyLock>> = LazyLock::new(Mutex::default); type ResolvedKey = (PathBuf, PathBuf, Option); -/// The linter settings buff reads for a file in `directory`, from the cache while no file -/// resolving them looked at has changed. +/// the linter settings buff reads for a file in `directory`, from the cache while no file +/// resolving them looked at has changed fn settings_for(system: &dyn System, directory: &Path, project_root: &Path) -> Arc { let user_config_directory = system.user_config_directory(); let key = ( @@ -322,9 +322,9 @@ fn settings_for(system: &dyn System, directory: &Path, project_root: &Path) -> A settings } -/// The linter settings buff reads for a file in `directory`, from the configuration files of -/// `files`. A configuration file that can't be read or is invalid is passed over for the next, -/// as buff falls back on the next configuration too. +/// the linter settings buff reads for a file in `directory`, from the configuration files of +/// `files`. a configuration file that can't be read or is invalid is passed over for the next, +/// as buff falls back on the next configuration too fn resolve( files: &RecordedFiles, directory: &Path, @@ -374,7 +374,7 @@ mod tests { use super::IsortConfig; - /// The line length buff reads for `/project/main.py` of `system`. + /// the line length buff reads for `/project/main.py` of `system` fn line_length(system: &TestSystem) -> usize { IsortConfig::for_file(system, Path::new("/project/main.py"), Path::new("/project")) .settings @@ -392,7 +392,7 @@ mod tests { system } - /// A configuration file `extend`s another: an edit to the other is seen. + /// a configuration file `extend`s another: an edit to the other is seen #[test] fn edit_to_an_extended_file_is_seen() { let system = system(&[ @@ -408,7 +408,7 @@ mod tests { assert_eq!(line_length(&system), 50); } - /// A configuration file created where none was found is read. + /// a configuration file created where none was found is read #[test] fn configuration_created_later_is_read() { let system = system(&[("/project/main.py", "")]); @@ -420,7 +420,7 @@ mod tests { assert_eq!(line_length(&system), 40); } - /// The user's own configuration is read from the system's configuration directory. + /// the user's own configuration is read from the system's configuration directory #[test] fn user_configuration_from_the_system() { let system = system(&[ @@ -435,7 +435,7 @@ mod tests { } /// `src` globs are matched against the system's files: a module in a matched directory is - /// the project's own. + /// the project's own #[test] fn src_globs_match_the_systems_files() { let system = system(&[ diff --git a/crates/ty_ide/src/importer.rs b/crates/ty_ide/src/importer.rs index b4df8dc5ee..5171eef93e 100644 --- a/crates/ty_ide/src/importer.rs +++ b/crates/ty_ide/src/importer.rs @@ -85,10 +85,10 @@ mod tests { source } - /// The file with everything `requests` import added at once, as + /// the file with everything `requests` import added at once, as /// [`Importer::combined_edits`] adds them, with each request's symbol written on a line - /// of its own at the cursor. A file buff accepts the imports of with the symbols written - /// in still has imports it accepts once they are added. + /// of its own at the cursor. a file buff accepts the imports of with the symbols written + /// in still has imports it accepts once they are added fn add_all(&self, requests: &[ImportRequest<'_>]) -> String { let (with_symbols, source) = self.add_all_with(&self.importer(), requests); if self.unsorted_imports(&with_symbols) == 0 { @@ -101,8 +101,8 @@ mod tests { source } - /// The file with the symbols `requests` are referred to by written at the cursor, and - /// the file with `importer`'s imports of them added too. + /// the file with the symbols `requests` are referred to by written at the cursor, and + /// the file with `importer`'s imports of them added too fn add_all_with( &self, importer: &Importer<'_>, @@ -131,7 +131,7 @@ mod tests { (with_symbols, source) } - /// How many `I001` diagnostics buff's isort check reports for `source` as the contents + /// how many `I001` diagnostics buff's isort check reports for `source` as the contents /// of the cursor's file, with the configuration buff reads for it. buff sorts each cell /// of a notebook on its own, so of a notebook only the cursor's cell is checked, which /// is where an import is added @@ -449,8 +449,8 @@ import sys "); } - /// A test whose project configures buff with `ruff_toml`, a `ruff.toml`, and whose file is - /// `main`. + /// a test whose project configures buff with `ruff_toml`, a `ruff.toml`, and whose file is + /// `main` fn configured_test(ruff_toml: &str, main: &str) -> CursorTest { CursorTest::builder() .source("ruff.toml", ruff_toml) @@ -464,9 +464,9 @@ standard-library = "Standard Library" third-party = "Third Party" "#; - /// A statement of a section before the block's first goes above the heading of that first + /// a statement of a section before the block's first goes above the heading of that first /// section, under a heading of its own: isort takes the headings directly above a block's - /// first import into the block and writes them again. + /// first import into the block and writes them again #[test] fn heading_of_a_new_first_section() { let test = configured_test( @@ -489,7 +489,7 @@ import attr "); } - /// A statement that joins the block's first section goes under its heading. + /// a statement that joins the block's first section goes under its heading #[test] fn heading_of_the_first_section_stays_first() { let test = configured_test( @@ -516,7 +516,7 @@ import attr "); } - /// A new section after the others is written with its heading. + /// a new section after the others is written with its heading #[test] fn heading_of_a_new_last_section() { let test = configured_test( @@ -539,8 +539,8 @@ import os "); } - /// Two members added from a module the file doesn't import yet are one `from` import, with - /// no import block to join. + /// two members added from a module the file doesn't import yet are one `from` import, with + /// no import block to join #[test] fn members_combined_without_an_import_block() { let test = cursor_test( @@ -561,7 +561,7 @@ x = 1 "); } - /// And without an order to follow, as for a file buff has no configuration for. + /// and without an order to follow, as for a file buff has no configuration for #[test] fn members_combined_without_an_order() { let test = cursor_test( @@ -594,7 +594,7 @@ x = 1 "); } - /// A member that makes an import too long for its line wraps it, as isort writes it. + /// a member that makes an import too long for its line wraps it, as isort writes it #[test] fn member_wraps_an_import_past_the_line_length() { let test = configured_test( @@ -614,7 +614,7 @@ from collections import OrderedDict, deque "); } - /// A new statement too long for its line is written wrapped. + /// a new statement too long for its line is written wrapped #[test] fn new_statement_wrapped_to_the_line_length() { let test = configured_test( @@ -640,7 +640,7 @@ import os "); } - /// Between two imports of one line, a statement is written on that line, in order. + /// between two imports of one line, a statement is written on that line, in order #[test] fn between_imports_of_one_line() { let test = cursor_test( @@ -657,7 +657,7 @@ import os; import sys "); } - /// Under `force-single-line`, a member is imported by a `from` import of its own. + /// under `force-single-line`, a member is imported by a `from` import of its own #[test] fn force_single_line() { let test = configured_test( @@ -674,8 +674,8 @@ from collections import OrderedDict "); } - /// Without `combine-as-imports`, an aliased member is imported by a `from` import of its - /// own. + /// without `combine-as-imports`, an aliased member is imported by a `from` import of its + /// own #[test] fn aliased_member_alone_without_combine_as_imports() { let test = cursor_test( @@ -691,7 +691,7 @@ from pathlib import Path "); } - /// With it, an aliased member joins the others. + /// with it, an aliased member joins the others #[test] fn aliased_member_joins_with_combine_as_imports() { let test = configured_test( @@ -707,7 +707,7 @@ from pathlib import Path "); } - /// A section listed in `no-lines-before` follows the section before it without a blank line. + /// a section listed in `no-lines-before` follows the section before it without a blank line #[test] fn no_lines_before() { let test = configured_test( @@ -726,7 +726,7 @@ import os "); } - /// `lines-between-types` sets the `from` imports of a section apart from its plain ones. + /// `lines-between-types` sets the `from` imports of a section apart from its plain ones #[test] fn lines_between_types() { let test = configured_test( @@ -746,7 +746,7 @@ import os "); } - /// `from-first` puts a section's `from` imports before its plain ones. + /// `from-first` puts a section's `from` imports before its plain ones #[test] fn from_first() { let test = configured_test( @@ -765,7 +765,7 @@ import os "); } - /// An eager import goes before a section's lazy ones, and a lazy one after its eager ones. + /// an eager import goes before a section's lazy ones, and a lazy one after its eager ones #[test] fn eager_before_lazy() { let test = cursor_test( @@ -785,7 +785,7 @@ lazy import json "); } - /// Statements added at once that go in different places of the block are each placed. + /// statements added at once that go in different places of the block are each placed #[test] fn several_places_at_once() { let test = cursor_test( diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index d8f5a9522d..e92d9ab9b9 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -108,7 +108,7 @@ pub use inlay_hints::{ InlayHintKind, InlayHintLabel, InlayHintSettings, InlayHintTextEdit, inlay_hints, }; pub use markup::MarkupKind; -pub use merge::{ConflictLines, MergeTexts, resolve_file_merge_conflicts, resolve_merge_conflicts}; +pub use merge::{ConflictLines, MergeTexts, resolve_file_merge_conflicts}; pub use method_kind::MethodKind; pub use module_rename::{ FileMove, ModuleRenameEdits, SkipReason, SkippedImport, module_rename_edits, diff --git a/crates/ty_ide/src/merge.rs b/crates/ty_ide/src/merge.rs index 0e2ffa19c6..b98306f5f4 100644 --- a/crates/ty_ide/src/merge.rs +++ b/crates/ty_ide/src/merge.rs @@ -75,7 +75,7 @@ pub struct MergeTexts<'a> { /// /// `sort_imports` writes a run of import statements as the project's isort configuration sorts /// them, and answers `None` for anything else -pub fn resolve_merge_conflicts( +pub(crate) fn resolve_merge_conflicts( texts: MergeTexts<'_>, conflicts: &[ConflictLines], source_type: PySourceType, @@ -101,7 +101,7 @@ pub fn resolve_merge_conflicts( .collect() } -/// [`resolve_merge_conflicts`] for the file at `path` in `db`'s project, whose extension says +/// `resolve_merge_conflicts` for the file at `path` in `db`'s project, whose extension says /// how it is parsed, and whose imports are sorted with the isort configuration buff reads for it: /// a project file's, or with no `path` the project root directory's pub fn resolve_file_merge_conflicts( @@ -1588,8 +1588,8 @@ mod tests { use super::{ConflictLines, MergeTexts, resolve_merge_conflicts}; - /// The file a merge left with git's diff3 conflict markers in it, each conflict replaced by - /// what resolves it, or left as git wrote it where nothing does. + /// the file a merge left with git's diff3 conflict markers in it, each conflict replaced by + /// what resolves it, or left as git wrote it where nothing does fn resolved(conflicted: &str) -> String { let (mut base, mut ours, mut theirs) = (String::new(), String::new(), String::new()); let mut conflicts = Vec::new(); diff --git a/crates/ty_python_semantic/src/importer.rs b/crates/ty_python_semantic/src/importer.rs index 829dcbd200..ea08eeed09 100644 --- a/crates/ty_python_semantic/src/importer.rs +++ b/crates/ty_python_semantic/src/importer.rs @@ -472,7 +472,7 @@ impl<'a> Importer<'a> { /// statements of one section are sorted among each other and separated as the sorter /// separates them, two members added to one `from` import are sorted, and two members added /// from a module the file doesn't import yet are listed by one new `from` import, with or - /// without a sorter's order to follow. + /// without a sorter's order to follow pub fn combined_edits<'b>( &self, actions: impl IntoIterator, @@ -531,9 +531,9 @@ impl<'a> Importer<'a> { Some(start..end) } - /// The edits that make the additions in `placements`: where the file's import sorter would + /// the edits that make the additions in `placements`: where the file's import sorter would /// make them, or without an order to follow, statements at the top of the file and members - /// after an import's last. + /// after an import's last fn placed_edits(&self, placements: &[Placement]) -> Vec { if let Some(sorting) = &self.sorting && let Some(edits) = sorting.placed_edits(self, placements) @@ -590,7 +590,7 @@ impl<'a> Importer<'a> { /// import statements the module's body starts its imports with, or in a notebook the body /// of `at`'s cell, which the sorter sorts on its own, and where `order` sorts each of them, /// worked out once into `blocks`. `None` when it starts at or after `at`, where the imports - /// have to be available. + /// have to be available fn import_block<'b, O: ImportOrdering>( &self, order: &O, @@ -650,7 +650,7 @@ impl<'a> Importer<'a> { Some(edits) } - /// `statement` as `order`'s sorter writes it, wrapped to its line length. + /// `statement` as `order`'s sorter writes it, wrapped to its line length fn written(&self, order: &O, statement: &NewStatement) -> String { let line_ending = self.stylist.line_ending().as_str(); let text = statement.text(); @@ -667,8 +667,8 @@ impl<'a> Importer<'a> { /// the edits adding `statements` to the file's import block, each in its section, sorted /// among the section's imports and each other, with the blank lines and section headings - /// the sorter writes around them. Without an import block before `at`, the statements are - /// one of their own, where a statement goes without an order. + /// the sorter writes around them. without an import block before `at`, the statements are + /// one of their own, where a statement goes without an order fn placed_statements( &self, order: &O, @@ -759,7 +759,7 @@ impl<'a> Importer<'a> { None => positions.len().checked_sub(1), }; - // Between two imports of one line, as in `import os; import sys`, the statements + // between two imports of one line, as in `import os; import sys`, the statements // can only be written on that line if let (Some(before), Some(after)) = (before, after) && !source @@ -774,11 +774,11 @@ impl<'a> Importer<'a> { continue; } - // The sorter takes the headings directly above the block's first import into the + // the sorter takes the headings directly above the block's first import into the // block and writes them again, and moves the other comments directly above an import - // with it. So the lines between the imports the statements go between, which are + // with it. so the lines between the imports the statements go between, which are // regenerated with them, run from the line after the import before them to the - // headings and blank lines above the import after them. + // headings and blank lines above the import after them let start = before.map_or_else( || { let mut start = source.line_start(block[0].start()); @@ -836,7 +836,7 @@ impl<'a> Importer<'a> { /// `added`, sorted, written as the sorter writes them after the import at `previous` and /// before the import at `next`: each preceded by the blank lines the sorter separates it /// from the statement before with, and by its section's heading when it starts the section, - /// and followed by the blank lines and heading the sorter writes above `next`. + /// and followed by the blank lines and heading the sorter writes above `next` fn separated<'p, O: ImportOrdering>( &self, order: &O, @@ -874,9 +874,9 @@ impl<'a> Importer<'a> { } /// the edits listing `added`, sorted, among the members of the existing `from` import - /// `from`, each where `order` sorts it. When the import was written as the sorter writes it + /// `from`, each where `order` sorts it. when the import was written as the sorter writes it /// and the sorter would write it differently with the members added, as it wraps an import - /// that no longer fits on its line, the import is written again as the sorter writes it. + /// that no longer fits on its line, the import is written again as the sorter writes it fn placed_members( &self, order: &O, @@ -1350,11 +1350,11 @@ impl Placement { } } -/// A member of a `from` import, and the name it is imported as. +/// a member of a `from` import, and the name it is imported as type Member<'a> = (&'a str, Option<&'a str>); -/// The statements added before the import of the block at the index, or after the block's last -/// import for `None`, each with where it sorts. +/// the statements added before the import of the block at the index, or after the block's last +/// import for `None`, each with where it sorts type StatementSlot<'s, P> = (Option, Vec<(P, &'s NewStatement<'s>)>); /// `member`, or `member as asname`, as a `from` import lists it @@ -1365,8 +1365,8 @@ fn imported_member((member, asname): Member) -> String { } } -/// The edit that changes `old`, the text at `start`, to `new`, touching only what differs. -/// `None` when nothing does. +/// the edit that changes `old`, the text at `start`, to `new`, touching only what differs. +/// `None` when nothing does fn minimal_edit(old: &str, new: &str, start: TextSize) -> Option { let prefix = old .char_indices() @@ -1397,11 +1397,11 @@ fn minimal_edit(old: &str, new: &str, start: TextSize) -> Option { }) } -/// The statements and members `placements` add, combined: two members a sorter lists together +/// the statements and members `placements` add, combined: two members a sorter lists together /// that a module the file doesn't import yet adds are one statement listing both, and the -/// members added to one existing `from` import, by its start, are listed together. Each +/// members added to one existing `from` import, by its start, are listed together. each /// statement's and import's members are sorted by `compare_members`, and `imports_member_alone` -/// says which member the sorter writes in a `from` import of its own. +/// says which member the sorter writes in a `from` import of its own fn combined<'p>( placements: &'p [Placement], order: &impl MemberOrder, @@ -1474,7 +1474,7 @@ fn combined<'p>( (statements, members) } -/// How a sorter lists the members of a `from` import, as [`combined`] needs to know it. +/// how a sorter lists the members of a `from` import, as [`combined`] needs to know it trait MemberOrder { fn compare_members(&self, a: Member, b: Member) -> std::cmp::Ordering; @@ -1491,7 +1491,7 @@ impl MemberOrder for O { } } -/// No sorter: members are listed in the order they are added, all in one `from` import. +/// no sorter: members are listed in the order they are added, all in one `from` import struct Unordered; impl MemberOrder for Unordered { diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index 9d8711be70..959d3a61d6 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -723,6 +723,7 @@ pub(crate) fn server_capabilities( }), file_operations: Some(lsp_types::FileOperationOptions { will_rename: Some(will_rename_registration()), + did_create: Some(did_create_registration()), ..Default::default() }), ..Default::default() @@ -746,6 +747,19 @@ pub(crate) fn server_capabilities( /// name. So every folder rename is asked about, and the ones that turn out not to be packages cost /// one request that answers with no edits. fn will_rename_registration() -> lsp_types::FileOperationRegistrationOptions { + module_and_package_filters() +} + +/// which creations the client should report once they are on disk: the same modules and packages +/// a rename is asked about, since what a request may name next is a module the client just made — +/// the module a definition is moved into. the server's own watcher reports a creation too, but +/// only some time after it, and a request the client sends in between is answered from a file +/// system on which the module is not there yet +fn did_create_registration() -> lsp_types::FileOperationRegistrationOptions { + module_and_package_filters() +} + +fn module_and_package_filters() -> lsp_types::FileOperationRegistrationOptions { fn filter( glob: &str, kind: lsp_types::FileOperationPatternKind, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 6f8cc0e324..5310e9fdf8 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -163,7 +163,7 @@ pub(super) fn request(req: server::Request) -> Task { requests::EndpointsHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } - // A merge's and a diff's texts, which are versions no document holds. + // a merge's and a diff's texts, which are versions no document holds requests::ResolveMergeConflictsHandler::METHOD => background_request_task::< requests::ResolveMergeConflictsHandler, >(req, BackgroundSchedule::Worker), @@ -470,6 +470,9 @@ pub(super) fn notification(notif: server::Notification) -> Task { notifications::DidChangeWatchedFiles::METHOD => { sync_notification_task::(notif) } + notifications::DidCreateFiles::METHOD => { + sync_notification_task::(notif) + } notifications::DidChangeWorkspaceFoldersHandler::METHOD => { sync_notification_task::(notif) } diff --git a/crates/ty_server/src/server/api/notifications.rs b/crates/ty_server/src/server/api/notifications.rs index 7fbb63a644..0bc95a02a1 100644 --- a/crates/ty_server/src/server/api/notifications.rs +++ b/crates/ty_server/src/server/api/notifications.rs @@ -5,6 +5,7 @@ mod did_change_watched_files; mod did_change_workspace_folders; mod did_close; mod did_close_notebook; +mod did_create_files; mod did_open; mod did_open_notebook; mod did_save; @@ -16,6 +17,7 @@ pub(super) use did_change_watched_files::DidChangeWatchedFiles; pub(super) use did_change_workspace_folders::DidChangeWorkspaceFoldersHandler; pub(super) use did_close::DidCloseTextDocumentHandler; pub(super) use did_close_notebook::DidCloseNotebookHandler; +pub(super) use did_create_files::DidCreateFiles; pub(super) use did_open::DidOpenTextDocumentHandler; pub(super) use did_open_notebook::DidOpenNotebookHandler; pub(super) use did_save::DidSaveTextDocumentHandler; diff --git a/crates/ty_server/src/server/api/notifications/did_create_files.rs b/crates/ty_server/src/server/api/notifications/did_create_files.rs new file mode 100644 index 0000000000..c288149d39 --- /dev/null +++ b/crates/ty_server/src/server/api/notifications/did_create_files.rs @@ -0,0 +1,51 @@ +//! `workspace/didCreateFiles`: the client created files, through a refactoring or its own user +//! interface, and says so once they are on disk +//! +//! the session's own watcher sees a write after it happens, but not before the client goes on to +//! ask about what it created: a module made to move a definition into is asked about at once. the +//! server answers from what it has read of the file system, so a path it had found nothing at is +//! still nothing until a change reaches it. this notification is that change, in order with the +//! requests after it + +use lsp_types::{self as types, DidCreateFilesNotification}; +use ty_project::watch::{ChangeEvent, CreatedKind, ExistingPathKind}; + +use crate::document::DocumentKey; +use crate::server::Result; +use crate::server::api::changes; +use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler}; +use crate::session::Session; +use crate::session::client::Client; +use crate::system::AnySystemPath; + +pub(crate) struct DidCreateFiles; + +impl NotificationHandler for DidCreateFiles { + type NotificationType = DidCreateFilesNotification; +} + +impl SyncNotificationHandler for DidCreateFiles { + fn run(session: &mut Session, client: &Client, params: types::CreateFilesParams) -> Result<()> { + let system = session.system(); + let changes: Vec<_> = params + .files + .iter() + .filter_map( + |created| match DocumentKey::from_uri(&created.uri).into_file_path() { + AnySystemPath::System(path) => Some(ChangeEvent::Created { + kind: CreatedKind::from(ExistingPathKind::from_system(system, &path)), + path, + }), + AnySystemPath::SystemVirtual(path) => { + tracing::debug!("Ignoring a created virtual path: `{path}`"); + None + } + }, + ) + .collect(); + + changes::apply(session, client, &changes); + + Ok(()) + } +} diff --git a/crates/ty_server/src/server/api/requests/merge_conflicts.rs b/crates/ty_server/src/server/api/requests/merge_conflicts.rs index 63ddfb7aa2..7ec9fb9416 100644 --- a/crates/ty_server/src/server/api/requests/merge_conflicts.rs +++ b/crates/ty_server/src/server/api/requests/merge_conflicts.rs @@ -1,11 +1,11 @@ //! `by/resolveMergeConflicts` and `by/diffIgnoredRanges`: what an editor's merge and diff tools -//! ask a language about the texts they compare. +//! ask a language about the texts they compare //! -//! Both take texts rather than documents: a merge's base, ours and theirs, and a diff's two sides, -//! are versions no file on disk holds. A `uri`, where one is given, names the file the texts are +//! both take texts rather than documents: a merge's base, ours and theirs, and a diff's two sides, +//! are versions no file on disk holds. a `uri`, where one is given, names the file the texts are //! versions of, which says how they are parsed and which isort configuration sorts their imports. -//! See [`ty_ide::resolve_merge_conflicts`] and [`ty_ide::diff_ignored_ranges`] for what is -//! answered. +//! see [`ty_ide::resolve_file_merge_conflicts`] and [`ty_ide::diff_ignored_ranges`] for what is +//! answered use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, Uri}; use ruff_db::system::SystemPathBuf; @@ -41,7 +41,7 @@ pub(crate) struct ResolveMergeConflictsParams { uri: Option, } -/// The lines a conflict spans in each version. +/// the lines a conflict spans in each version #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct Conflict { @@ -83,8 +83,8 @@ impl RequestHandler for ResolveMergeConflictsHandler { } impl BackgroundRequestHandler for ResolveMergeConflictsHandler { - /// The text each conflict resolves to, or `null` for one that is not resolved: all `null` - /// without a project to read the configuration of. + /// the text each conflict resolves to, or `null` for one that is not resolved: all `null` + /// without a project to read the configuration of fn run( snapshot: &SessionSnapshot, _client: &Client, @@ -154,9 +154,9 @@ pub(crate) struct DiffIgnoredRangesParams { #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct DiffIgnoredRanges { - /// Each import statement. + /// each import statement imports: Vec, - /// Each run of text that is only formatting. + /// each run of text that is only formatting formatting: Vec, } diff --git a/crates/ty_server/tests/e2e/merge_conflicts.rs b/crates/ty_server/tests/e2e/merge_conflicts.rs index 3a86b8aa7f..d36aec3746 100644 --- a/crates/ty_server/tests/e2e/merge_conflicts.rs +++ b/crates/ty_server/tests/e2e/merge_conflicts.rs @@ -1,4 +1,4 @@ -//! `by/resolveMergeConflicts` and `by/diffIgnoredRanges` over the wire: texts in, no document. +//! `by/resolveMergeConflicts` and `by/diffIgnoredRanges` over the wire: texts in, no document use anyhow::Result; use lsp_server::ErrorCode; diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index 4d0486001b..a978da377a 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -5,7 +5,10 @@ //! workspace edit of a move, and the reason a move is refused use anyhow::Result; -use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use lsp_types::{ + CreateFilesParams, DidCreateFilesNotification, FileCreate, LspRequestMethod, MessageDirection, + Request, +}; use ruff_db::system::SystemPath; use ruff_source_file::LineEnding; use serde_json::json; @@ -374,3 +377,41 @@ fn an_extracted_superclass_takes_the_class_s_bases_and_becomes_its_base() -> Res ); Ok(()) } + +/// a module the client has just made is a target once the client reports it with +/// `workspace/didCreateFiles`, even where the server had found nothing at its path before +#[test] +fn a_module_the_client_created_is_a_target_once_it_says_so() -> Result<()> { + let mut server = server()?; + let main = server.file_uri(SystemPath::new("src/main.py")); + let fresh = SystemPath::new("src/shapes/fresh.py"); + let move_to_fresh = |server: &mut TestServer| { + server + .send_request_await::(json!({ + "textDocument": { "uri": main }, + "position": { "line": 2, "character": 7 }, + "target": server.file_uri(fresh), + })) + .expect("an answer") + }; + assert_eq!( + move_to_fresh(&mut server)["refused"], + json!("the target is not a file that exists") + ); + + server.write_file(fresh, "")?; + server.send_notification::(CreateFilesParams { + files: vec![FileCreate { + uri: server.file_uri(fresh), + }], + }); + let answer = move_to_fresh(&mut server); + assert!(answer.get("refused").is_none(), "{answer:#}"); + assert!( + answer["edit"]["changes"] + .get(server.file_uri(fresh).as_str()) + .is_some(), + "{answer:#}" + ); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 69d1fc49d2..2b639cc041 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -145,6 +145,24 @@ expression: initialization_result "changeNotifications": true }, "fileOperations": { + "didCreate": { + "filters": [ + { + "scheme": "file", + "pattern": { + "glob": "**/*.{py,pyi,by,byi}", + "matches": "file" + } + }, + { + "scheme": "file", + "pattern": { + "glob": "**", + "matches": "folder" + } + } + ] + }, "willRename": { "filters": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 69d1fc49d2..2b639cc041 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -145,6 +145,24 @@ expression: initialization_result "changeNotifications": true }, "fileOperations": { + "didCreate": { + "filters": [ + { + "scheme": "file", + "pattern": { + "glob": "**/*.{py,pyi,by,byi}", + "matches": "file" + } + }, + { + "scheme": "file", + "pattern": { + "glob": "**", + "matches": "folder" + } + } + ] + }, "willRename": { "filters": [ { diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index d24b9b7517..ebc9b8cd9f 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -742,12 +742,19 @@ definition's name, in a reference to one, or anywhere else in a top-level definition, its body included, which is then the one moved — and answers `{name, kind, module, uri, range}`: the name, `function`, `class` or `variable`, the dotted name of the module it is defined in, and its name's range in the file `uri`, which need not be the document asked about; `null` where there is nothing to move. `by/moveSymbol` takes -the same and `target`, the URI of an existing `.py` or `.by` file of the project -(an editor creates a new module before asking), and answers `{edit, warnings}`, a +the same and `target`, the URI of an existing `.py` or `.by` file of the project, +and answers `{edit, warnings}`, a `WorkspaceEdit` of `changes` to every file the move changes and the references it left as they were, each with its file and line, or `{refused, warnings}`, the reason nothing is moved. two edits at one position of a file apply in the order listed +an editor moving into a new module creates it first and reports it with the +protocol's `workspace/didCreateFiles`, which the server registers for every +`.py`, `.pyi`, `.by` and `.byi` file and every folder, before it asks: the server +answers from the file system as it last read it, so without the notification a +module made where it had found nothing is refused as a file that does not exist. +`workspace/didChangeWatchedFiles` reporting it `Created` does the same + ### pulling members up and pushing them down a class's members — a method, with the accessors or overloads that share its name, From 36078c403b5ac19c97eb31bb2e1f7c7dd100bff3 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:12:13 +1000 Subject: [PATCH 065/110] test that by/extractSuperclass refuses a name that is not an identifier, a keyword and a name its module binds already, and say so where its refusals are listed --- crates/ty_ide/src/refactor/class_members.rs | 12 ++++++++++++ docs/basedpython/features/editor.md | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index 39e2401348..c1d2ea588e 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -1981,6 +1981,18 @@ class Leaf(Mid): "); } + /// the new class's name has to be one a class statement can be written with, and one the + /// module does not bind already, to a class or to anything else + #[test] + fn extracting_under_a_name_a_class_cannot_have_is_refused() { + let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); + assert_snapshot!(test.extract(&["plain"], "1Shape"), @"refused: `1Shape` is not a name a class can have"); + assert_snapshot!(test.extract(&["plain"], "My Shape"), @"refused: `My Shape` is not a name a class can have"); + assert_snapshot!(test.extract(&["plain"], ""), @"refused: `` is not a name a class can have"); + assert_snapshot!(test.extract(&["plain"], "class"), @"refused: `class` is not a name a class can have"); + assert_snapshot!(test.extract(&["plain"], "Base"), @"refused: cls.shapes has a `Base` already"); + } + #[test] fn extracting_what_reads_an_attribute_left_behind_is_refused() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index ebc9b8cd9f..e0655cddc2 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -839,8 +839,9 @@ started. it is refused, with the reason, for a member that reads an attribute of left behind), a name-mangled member, a method that reads `__class__`, which would be the new class, an attribute of a decorated class (a data class's field, say), a generic class, a class whose base python treats specially — -`Protocol`, `NamedTuple`, `TypedDict`, `Generic`, an enum — and a name the module -binds already. it takes `textDocument`, `position` (anywhere in the class), +`Protocol`, `NamedTuple`, `TypedDict`, `Generic`, an enum — a class that passes a +keyword other than `metaclass` when it is made, a name that is not an +identifier or is a keyword, and a name the module binds already. it takes `textDocument`, `position` (anywhere in the class), `members`, the names to move, as `by/prepareMoveMembers` lists them, and `name`, and answers as `by/moveSymbol` does From 494b7270c1a71ff9dc3ff6bb0f48fa103e882324 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:18:24 +1000 Subject: [PATCH 066/110] rename a function with by/changeSignature's name, wherever the project refers to it, and rewrite its parameters' annotations and its return annotation, each given as source, null to remove and absent to keep, answering by/prepareChangeSignature with the return annotation too, refusing a name the function cannot have or its scope binds and an annotation that is not an expression --- crates/ty_ide/src/lib.rs | 8 +- crates/ty_ide/src/refactor.rs | 4 +- .../ty_ide/src/refactor/change_signature.rs | 1273 +++++++++++++++-- crates/ty_ide/src/refactor/flow.rs | 2 +- .../server/api/requests/change_signature.rs | 99 +- .../src/server/api/requests/move_symbol.rs | 65 +- .../ty_server/tests/e2e/change_signature.rs | 80 ++ docs/basedpython/features/editor.md | 27 +- 8 files changed, 1316 insertions(+), 242 deletions(-) diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index e92d9ab9b9..6666d11427 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -119,15 +119,15 @@ pub use override_members::{ override_members, }; pub use pasted_imports::{CopiedCode, CopiedImport, PastedImports, copied_imports, pasted_imports}; +pub use refactor::{ + AnnotationChange, FunctionSignature, NewParameter, ParameterKind, SignatureChange, + SignatureParameter, change_signature, function_signature, +}; pub use refactor::{ ClassMember, HierarchyClass, MemberKind, MovableKind, MovableMembers, MovableSymbol, Refactor, RefactorKind, RefactorOffer, SymbolMove, extract_superclass, movable_members, movable_symbol, move_symbol, pull_members_up, push_members_down, refactor, refactors, }; -pub use refactor::{ - FunctionSignature, NewParameter, ParameterKind, SignatureParameter, change_signature, - function_signature, -}; pub use references::ReferencesMode; pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; pub use selection_range::selection_range; diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 7d26d198f0..6e93352cb3 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -44,8 +44,8 @@ use ty_python_semantic::SemanticModel; use crate::FileEdit; pub use change_signature::{ - FunctionSignature, NewParameter, ParameterKind, SignatureParameter, change_signature, - function_signature, + AnnotationChange, FunctionSignature, NewParameter, ParameterKind, SignatureChange, + SignatureParameter, change_signature, function_signature, }; pub use class_members::{ ClassMember, HierarchyClass, MemberKind, MovableMembers, extract_superclass, movable_members, diff --git a/crates/ty_ide/src/refactor/change_signature.rs b/crates/ty_ide/src/refactor/change_signature.rs index d5066d4ac2..e43951881a 100644 --- a/crates/ty_ide/src/refactor/change_signature.rs +++ b/crates/ty_ide/src/refactor/change_signature.rs @@ -1,4 +1,4 @@ -//! Changing a function's parameters, and every call of it in the project with them. +//! changing a function's parameters, and every call of it in the project with them //! //! ```py //! def area(width, height=1): def area(height, width, *, scale=1.0): @@ -8,59 +8,75 @@ //! area(width=2) area(width=2, height=1) # when `height` loses its default //! ``` //! -//! The new parameter list is given in full: each parameter by its name, the parameter it was -//! (none for a new one), its kind, its default, and for a new one the value each call passes. -//! A renamed parameter is renamed where the body reads it and where calls pass it by keyword. +//! the new parameter list is given in full: each parameter by its name, the parameter it was +//! (none for a new one), its kind, its default, its annotation, and for a new one the value each +//! call passes. a renamed parameter is renamed where the body reads it and where calls pass it by +//! keyword. the function can be given a new name, renamed everywhere it is referred to, and a new +//! return annotation //! -//! Every call is rebound the way python binds it — its positional arguments to the positional +//! every call is rebound the way python binds it — its positional arguments to the positional //! parameters, its keywords by name — and written again for the new list: by position while the -//! parameters still line up, by keyword after that, and keeping a keyword a keyword. The change is -//! refused, with the reason, where a call cannot be rewritten so that it means what it meant: +//! parameters still line up, by keyword after that, and keeping a keyword a keyword. arguments are +//! evaluated in the order they are written, so where the new order would evaluate one before +//! another written before it, and either could change what the other does, the call passes them +//! by keyword in the order it wrote them. calls through a name an import gives the function, `from +//! m import area as a`, are calls of it too. the change is refused, with the reason, where a call +//! cannot be rewritten so that it means what it meant: //! //! - a call that unpacks `*args` or `**kwargs` into the function, whose arguments cannot be told //! apart, unless the change leaves every existing call valid as it is; -//! - an argument the change would drop, or evaluate after one written after it, when evaluating it -//! could have an effect; +//! - an argument the change would drop when evaluating it could have an effect, and one whose +//! order of evaluation cannot be kept because it can only be passed by position; +//! - a keyword a call passes into `**kwargs` that a parameter of the new list would take; //! - a call with nothing to pass for a parameter without a default; //! - the function used other than called — passed as a value, say — unless every existing call //! stays valid; //! - a removed parameter the body still reads, and a list python would not accept: a parameter //! without a default after one with a default, two of a name, kinds out of order; -//! - a method another class overrides or that overrides another's, whose signatures would part. +//! - a new parameter name the body already means something by: one it binds, or reads from outside +//! the function, and a renamed parameter a nested scope that binds its new name reads; +//! - a default, an annotation or a value that is not one expression where it is written; +//! - a function its decorators bind to something other than itself, and one defined more than +//! once where it is defined, as an `@overload` set is; +//! - a method another class overrides or that overrides another's, whose signatures would part use ruff_db::files::File; -use ruff_db::parsed::parsed_module; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_db::source::source_text; use ruff_diagnostics::Edit; use ruff_python_ast::find_node::covering_node; use ruff_python_ast::{self as ast, AnyNodeRef, Expr}; +use ruff_python_parser::{Mode, ParseOptions}; use ruff_python_stdlib::identifiers::is_identifier; use ruff_python_stdlib::keyword::is_keyword; use ruff_text_size::{Ranged, TextRange, TextSize}; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use ty_project::Db; -use ty_python_core::ProgramFile; +use ty_python_core::scope::{FileScopeId, NodeWithScopeRef}; +use ty_python_core::{ProgramFile, SemanticIndex, semantic_index}; use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::decorated_function; use ty_python_semantic::{HasType, SemanticModel}; -use super::evaluation::is_effect_free; +use super::evaluation::{is_effect_free, is_pure}; +use super::flow::{bindings_of_symbol, descendants}; use super::move_symbol::{SymbolMove, line_number}; use crate::code_action::FileEdit; use crate::goto::find_goto_target; use crate::method_kind::{MethodKind, resolved_method_kind}; use crate::references::{ReferencesMode, references}; -/// How a parameter is passed. +/// how a parameter is passed #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum ParameterKind { - /// Before a `/`: only by position. + /// before a `/`: only by position PositionalOnly, PositionalOrKeyword, - /// `*args`. + /// `*args` Variadic, - /// After a `*` or `*args`: only by keyword. + /// after a `*` or `*args`: only by keyword KeywordOnly, - /// `**kwargs`. + /// `**kwargs` VariadicKeyword, } @@ -74,42 +90,65 @@ impl ParameterKind { } } -/// A parameter of a function as it is. +/// a parameter of a function as it is #[derive(Debug, Clone, PartialEq, Eq)] pub struct SignatureParameter { pub name: String, pub kind: ParameterKind, - /// Its default, as the source writes it. + /// its default, as the source writes it pub default: Option, - /// Its annotation, as the source writes it. + /// its annotation, as the source writes it pub annotation: Option, } -/// A function whose signature [`change_signature`] can change. +/// a function whose signature [`change_signature`] can change #[derive(Debug, Clone, PartialEq, Eq)] pub struct FunctionSignature { - /// The file it is defined in, which need not be the one asked about. + /// the file it is defined in, which need not be the one asked about pub file: File, pub name: String, pub name_range: TextRange, - /// Its parameters, without the `self` or `cls` a method is called with, which stays as it - /// is. + /// its parameters, without the `self` or `cls` a method is called with, which stays as it + /// is pub parameters: Vec, + /// its return annotation, as the source writes it + pub returns: Option, } -/// A parameter of the list [`change_signature`] writes. +/// what an annotation becomes +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum AnnotationChange { + /// as it is written: none, for a new parameter + #[default] + Keep, + /// the annotation given, written as source, or none + Set(Option), +} + +/// the whole of the signature [`change_signature`] gives a function +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SignatureChange { + /// its new name; `None` keeps the one it has + pub name: Option, + /// its parameter list, in order, without a method's `self` or `cls` + pub parameters: Vec, + pub returns: AnnotationChange, +} + +/// a parameter of the list [`change_signature`] writes #[derive(Debug, Clone, PartialEq, Eq)] pub struct NewParameter { pub name: String, - /// The parameter it is, for one the function has; `None` for a new one. + /// the parameter it is, for one the function has; `None` for a new one pub old_name: Option, pub kind: ParameterKind, pub default: Option, - /// For a new parameter, what each call passes it; none leaves it to its default. + pub annotation: AnnotationChange, + /// for a new parameter, what each call passes it; none leaves it to its default pub value: Option, } -/// The function defined or named at `offset`, and its parameters. +/// the function defined or named at `offset`, and its parameters pub fn function_signature( db: &dyn Db, file: ProgramFile<'_>, @@ -127,17 +166,23 @@ pub fn function_signature( .into_iter() .map(|old| old.parameter) .collect(), + returns: function + .returns + .as_ref() + .map(|returns| source[returns.range()].to_string()), }) } -/// Gives the function defined or named at `offset` the parameters `parameters`, and rewrites every -/// call of it in the project to pass them. `Err` says why it cannot be changed. +/// gives the function defined or named at `offset` the signature `change`, and rewrites every call +/// of it in the project to pass its parameters, and every reference to it to use its new name. +/// `Err` says why it cannot be changed pub fn change_signature( db: &dyn Db, file: ProgramFile<'_>, offset: TextSize, - parameters: &[NewParameter], + change: &SignatureChange, ) -> Result { + let parameters = change.parameters.as_slice(); let (file, offset) = definition_site(db, file, offset).ok_or_else(|| "there is no function here".to_string())?; let parsed = parsed_module(db, file.python_file(db)).load(db); @@ -150,6 +195,82 @@ pub fn change_signature( let receiver_name = receiver.then(|| first_parameter_name(function)).flatten(); validate(name, &old, parameters, receiver_name.as_deref())?; + let basedpython = file.file(db).source_type(db).is_basedpython(); + for parameter in parameters { + if let AnnotationChange::Set(Some(annotation)) = ¶meter.annotation + && !Place::Annotation.fits(annotation, basedpython) + { + return Err(format!("`{annotation}` is not an annotation")); + } + if let Some(default) = ¶meter.default + && !Place::Default.fits(default, basedpython) + { + return Err(format!("`{default}` is not an expression a default can be")); + } + if let Some(value) = ¶meter.value + && !Place::Argument.fits(value, basedpython) + { + return Err(format!("`{value}` is not an expression a call can pass")); + } + } + if let AnnotationChange::Set(Some(returns)) = &change.returns + && !Place::Returns.fits(returns, basedpython) + { + return Err(format!("`{returns}` is not an annotation")); + } + + let model = SemanticModel::new(db, file); + let index = semantic_index(db, file); + let Some(definition) = index + .try_definitions(AnyNodeRef::from(function)) + .and_then(<[_]>::first) + .copied() + else { + return Err(format!("`{name}` cannot be searched for")); + }; + // a decorator that binds the name to something other than the function decides what its + // calls pass, and an `@overload` set or a redefinition has calls that may reach another `def` + if decorated_function(db, definition) + .is_none_or(|decorated| decorated.decorated != decorated.undecorated) + { + return Err(format!( + "`{name}` is decorated, and its decorators make what its calls reach something other \ + than the function itself" + )); + } + let defining_scope = definition.scope(db).file_scope_id(db); + let defining_table = index.place_table(defining_scope); + if let Some(symbol) = defining_table.symbol_id(name) + && bindings_of_symbol( + db, + index, + &parsed, + defining_scope, + symbol, + file.file(db).is_stub(db), + ) + .len() + > 1 + { + return Err(format!( + "`{name}` is defined more than once where it is defined, as an `@overload` set or a \ + redefinition, and its calls may reach any of the definitions" + )); + } + let Some(body_scope) = index.try_node_scope(NodeWithScopeRef::Function(function)) else { + return Err(format!("`{name}` cannot be searched for")); + }; + let new_name = change.name.as_deref().filter(|new_name| *new_name != name); + if let Some(new_name) = new_name { + if let Some(reason) = crate::invalid_new_name(new_name) { + return Err(reason); + } + if binds_beside(db, file, function, new_name) { + return Err(format!( + "`{new_name}` is bound already where `{name}` is defined" + )); + } + } if receiver { // what it overrides in the project would part from it; a class of a library or the @@ -178,8 +299,22 @@ pub fn change_signature( let mut moved = SymbolMove::default(); let file_id = file.file(db); + // a name the function's parameters did not have must not be one its body already means + // something else by + for parameter in parameters { + if parameter.old_name.as_deref() == Some(parameter.name.as_str()) + || old + .iter() + .any(|existing| existing.parameter.name == parameter.name) + { + continue; + } + if let Some(reason) = body_meaning_of(index, body_scope, ¶meter.name) { + return Err(reason); + } + } + // the body: renamed parameters renamed where it reads them; removed ones must be unread - let model = SemanticModel::new(db, file); for parameter in &old { let new = parameters .iter() @@ -205,10 +340,22 @@ pub fn change_signature( .map(|reference| reference.range()) .collect(); match new { - Some(new) => moved.edits.extend(reads.into_iter().map(|range| FileEdit { - file: file_id, - edit: Edit::range_replacement(new.name.clone(), range), - })), + Some(new) => { + if let Some(read) = reads.iter().find(|read| { + nested_binding_reads(index, &parsed, body_scope, **read, &new.name) + }) { + return Err(format!( + "`{}` on line {} would read the `{}` a scope inside `{name}` binds", + parameter.parameter.name, + line_number(source, read.start()), + new.name + )); + } + moved.edits.extend(reads.into_iter().map(|range| FileEdit { + file: file_id, + edit: Edit::range_replacement(new.name.clone(), range), + })); + } None => { if let Some(read) = reads.first() { return Err(format!( @@ -222,6 +369,12 @@ pub fn change_signature( } // the definition + if let Some(new_name) = new_name { + moved.edits.push(FileEdit { + file: file_id, + edit: Edit::range_replacement(new_name.to_string(), function.name.range()), + }); + } moved.edits.push(FileEdit { file: file_id, edit: Edit::range_replacement( @@ -229,33 +382,60 @@ pub fn change_signature( function.parameters.range(), ), }); + if let AnnotationChange::Set(returns) = &change.returns { + let edit = match (&function.returns, returns) { + (Some(written), Some(returns)) => { + Some(Edit::range_replacement(returns.clone(), written.range())) + } + (Some(written), None) => Some(Edit::range_deletion(TextRange::new( + function.parameters.end(), + written.end(), + ))), + (None, Some(returns)) => Some(Edit::insertion( + format!(" -> {returns}"), + function.parameters.end(), + )), + (None, None) => None, + }; + moved.edits.extend(edit.map(|edit| FileEdit { + file: file_id, + edit, + })); + } // the calls let compatible = is_compatible(&old, parameters); - let goto = find_goto_target(&model, &parsed, function.name.start()) - .ok_or_else(|| format!("`{name}` cannot be searched for"))?; - let mut by_file: FxHashMap> = FxHashMap::default(); - for reference in - references(db, file, &goto, ReferencesMode::RenameMultiFile).unwrap_or_default() - { - if reference.file() == file_id && reference.range() == function.name.range() { - continue; - } - by_file - .entry(reference.file()) - .or_default() - .push(reference.range()); - } - let mut files: Vec = by_file.keys().copied().collect(); - files.sort_by_key(|file| file.path(db).as_str().to_string()); - for reference_file in files { + let by_file = function_references(db, file, &model, &parsed, function)?; + let mut calls: Vec<(File, TextRange, Vec)> = Vec::new(); + for (reference_file, references) in by_file { let program_file = ProgramFile::new(db, reference_file, file.program(db)); let parsed = parsed_module(db, program_file.python_file(db)).load(db); let text = source_text(db, reference_file); let text = text.as_str(); let model = SemanticModel::new(db, program_file); let path = relative_path(db, reference_file); - for range in &by_file[&reference_file] { + let reference_basedpython = reference_file.source_type(db).is_basedpython(); + if !compatible + && reference_basedpython != basedpython + && let Some(value) = parameters + .iter() + .filter_map(|parameter| parameter.value.as_deref()) + .find(|value| !Place::Argument.fits(value, reference_basedpython)) + { + return Err(format!( + "`{value}` is not an expression a call in {path} can pass" + )); + } + for reference in &references { + let range = &reference.range; + if reference.spells_the_name + && let Some(new_name) = new_name + { + moved.edits.push(FileEdit { + file: reference_file, + edit: Edit::range_replacement(new_name.to_string(), *range), + }); + } let covering = covering_node(parsed.syntax().into(), *range); let call = call_of(&covering); let Some((call, bound)) = @@ -285,24 +465,24 @@ pub fn change_signature( let rewritten = rewrite_call(&model, text, call, bound, &old, parameters) .map_err(|reason| format!("{path}:{} {reason}", line_number(text, call.start())))?; if let Some(rewritten) = rewritten { - moved.edits.push(FileEdit { - file: reference_file, - edit: Edit::range_replacement(rewritten, call.arguments.range()), - }); + calls.push((reference_file, call.arguments.range(), rewritten)); } } } + moved.edits = composed(db, moved.edits, calls); Ok(moved) } -/// A parameter of the function as it is, with where its name is. +/// a parameter of the function as it is, with where its name is struct OldParameter { parameter: SignatureParameter, name_range: TextRange, - /// Its text without its default: its modifiers, name and annotation. + /// its text without its default: its modifiers, name and annotation text: String, - /// The range of `text`'s name within it. + /// the range of `text`'s name within it name_in_text: TextRange, + /// the range of `text`'s annotation within it + annotation_in_text: Option, } fn old_parameters( @@ -326,6 +506,10 @@ fn old_parameters( }, name_range: parameter.name.range(), name_in_text: parameter.name.range() - parameter.start(), + annotation_in_text: parameter + .annotation + .as_ref() + .map(|annotation| annotation.range() - parameter.start()), text, }); }; @@ -371,8 +555,8 @@ fn first_parameter_name(function: &ast::StmtFunctionDef) -> Option { .map(|parameter| parameter.parameter.name.to_string()) } -/// Checks that `parameters` is a list python accepts, made of the function's own parameters and -/// new ones. +/// checks that `parameters` is a list python accepts, made of the function's own parameters and +/// new ones fn validate( name: &str, old: &[OldParameter], @@ -453,18 +637,18 @@ fn validate( Ok(()) } -/// Whether every call of the function as it is stays valid, and means the same, against +/// whether every call of the function as it is stays valid, and means the same, against /// `parameters`: the parameters it has keep their names, kinds, places and defaults, and those -/// added after them all have defaults. +/// added after them all have defaults. a parameter added before `**kwargs` could take a keyword a +/// call passes to `**kwargs`, so only a `*args` can be fn is_compatible(old: &[OldParameter], parameters: &[NewParameter]) -> bool { let mut new = parameters.iter().peekable(); for existing in old { - // added keyword-only ones may come before `**kwargs` while let Some(added) = new.peek() && added.old_name.is_none() && existing.parameter.kind == ParameterKind::VariadicKeyword { - if added.default.is_none() && !added.kind.is_variadic() { + if !added.kind.is_variadic() { return false; } new.next(); @@ -487,7 +671,7 @@ fn is_compatible(old: &[OldParameter], parameters: &[NewParameter]) -> bool { }) } -/// The parenthesized parameter list `parameters` makes, the receiver first. +/// the parenthesized parameter list `parameters` makes, the receiver first fn parameter_list( source: &str, function: &ast::StmtFunctionDef, @@ -534,21 +718,47 @@ fn parameter_list( old.iter() .find(|existing| existing.parameter.name == *old_name) }); + let annotation = match ¶meter.annotation { + AnnotationChange::Keep => { + existing.and_then(|existing| existing.parameter.annotation.as_deref()) + } + AnnotationChange::Set(annotation) => annotation.as_deref(), + }; let mut text = match existing { Some(existing) => { + // the annotation after the name, so the name's range still holds once it changes let mut text = existing.text.clone(); + if let AnnotationChange::Set(_) = ¶meter.annotation { + let after_name = existing.name_in_text.end().to_usize(); + let written = existing + .annotation_in_text + .map_or(after_name..after_name, |range| { + after_name..range.end().to_usize() + }); + text.replace_range( + written, + &annotation + .map(|annotation| format!(": {annotation}")) + .unwrap_or_default(), + ); + } text.replace_range(existing.name_in_text.to_std_range(), ¶meter.name); text } - None => match parameter.kind { - ParameterKind::Variadic => format!("*{}", parameter.name), - ParameterKind::VariadicKeyword => format!("**{}", parameter.name), - _ => parameter.name.clone(), - }, + None => { + let star = match parameter.kind { + ParameterKind::Variadic => "*", + ParameterKind::VariadicKeyword => "**", + _ => "", + }; + match annotation { + Some(annotation) => format!("{star}{}: {annotation}", parameter.name), + None => format!("{star}{}", parameter.name), + } + } }; if let Some(default) = ¶meter.default { - let annotated = - existing.is_some_and(|existing| existing.parameter.annotation.is_some()); + let annotated = annotation.is_some(); text.push_str(if annotated { " = " } else { "=" }); text.push_str(default); } @@ -560,14 +770,59 @@ fn parameter_list( format!("({})", parts.join(", ")) } -/// An argument of a call, as python binds it. +/// an argument of a call, as python binds it #[derive(Clone, Copy)] struct Argument<'a> { value: &'a Expr, keyword: Option<&'a str>, } -/// The arguments of `call` rewritten for `parameters`, or `None` when they stay as they are. +/// an argument as the rewritten call writes it +#[derive(Clone)] +struct WrittenArgument<'a> { + /// what the call passes, or `None` for a new parameter's value + value: Option<&'a Expr>, + piece: Piece, + /// the name it can be passed by keyword under, `None` where it can only be passed by + /// position + keyword: Option, +} + +/// of `evaluated`, in the order a call evaluates them, the first two that are evaluated the other +/// way round from how they are written, written first then written second, where evaluating +/// either could change what the other does +fn reordered_pair<'a>( + model: &SemanticModel<'_>, + evaluated: &[&'a Expr], +) -> Option<(&'a Expr, &'a Expr)> { + evaluated + .iter() + .enumerate() + .flat_map(|(position, first)| { + evaluated[position + 1..] + .iter() + .map(move |second| (*first, *second)) + }) + .find(|(first, second)| { + second.start() < first.start() + && !(is_constant(first) + || is_constant(second) + || (is_effect_free(model, first) && is_effect_free(model, second))) + }) + .map(|(first, second)| (second, first)) +} + +/// whether `expr` is a literal, which reads nothing an effect could change +fn is_constant(expr: &Expr) -> bool { + match expr { + Expr::Name(_) => false, + Expr::Tuple(tuple) => is_pure(expr) && tuple.elts.iter().all(is_constant), + _ => is_pure(expr), + } +} + +/// the arguments of `call` rewritten for `parameters`, parentheses included, or `None` when they +/// stay as they are fn rewrite_call( model: &SemanticModel<'_>, source: &str, @@ -575,7 +830,7 @@ fn rewrite_call( bound: bool, old: &[OldParameter], parameters: &[NewParameter], -) -> Result, String> { +) -> Result>, String> { let arguments = &call.arguments; if arguments .args @@ -644,9 +899,28 @@ fn rewrite_call( } } + // a keyword `**kwargs` took that a parameter of the new list is named for would be taken by + // that parameter instead + if let Some(stolen) = extra_keyword.iter().find_map(|argument| { + argument.keyword.filter(|keyword| { + parameters.iter().any(|parameter| { + parameter.name == *keyword + && matches!( + parameter.kind, + ParameterKind::PositionalOrKeyword | ParameterKind::KeywordOnly + ) + }) + }) + }) { + return Err(format!( + "passes `{stolen}` into `**kwargs`, where the parameter `{stolen}` would take it \ + instead" + )); + } + // write them again for the new list - let mut by_position: Vec<(Option, String)> = Vec::new(); - let mut by_keyword: Vec<(Option, String)> = Vec::new(); + let mut by_position: Vec = Vec::new(); + let mut by_keyword: Vec = Vec::new(); let mut positional_ok = true; let mut used = vec![false; old.len()]; for parameter in parameters { @@ -660,25 +934,20 @@ fn rewrite_call( parameter.name )); } - by_position.extend( - extra_positional - .iter() - .map(|value| (Some(value.start()), source[value.range()].to_string())), - ); + by_position.extend(extra_positional.iter().map(|value| WrittenArgument { + value: Some(value), + piece: Piece::Source(value.range()), + keyword: None, + })); extra_positional.clear(); } continue; } ParameterKind::VariadicKeyword => { - by_keyword.extend(extra_keyword.iter().map(|argument| { - ( - Some(argument.value.start()), - format!( - "{}={}", - argument.keyword.unwrap_or_default(), - &source[argument.value.range()] - ), - ) + by_keyword.extend(extra_keyword.iter().map(|argument| WrittenArgument { + value: Some(argument.value), + piece: Piece::Source(argument.value.range()), + keyword: argument.keyword.map(ToString::to_string), })); extra_keyword.clear(); continue; @@ -693,13 +962,13 @@ fn rewrite_call( used[slot] = true; } let passed = slot.and_then(|slot| bound_to[slot]); - let (start, value, was_keyword) = match (passed, ¶meter.value) { + let (value, piece, was_keyword) = match (passed, ¶meter.value) { (Some(argument), _) => ( - Some(argument.value.start()), - source[argument.value.range()].to_string(), + Some(argument.value), + Piece::Source(argument.value.range()), argument.keyword.is_some(), ), - (None, Some(value)) if slot.is_none() => (None, value.clone(), false), + (None, Some(value)) if slot.is_none() => (None, Piece::Text(value.clone()), false), (None, _) => { if parameter.default.is_none() { return Err(format!("has nothing to pass for `{}`", parameter.name)); @@ -708,24 +977,30 @@ fn rewrite_call( continue; } }; + let argument = WrittenArgument { + value, + piece, + keyword: (parameter.kind != ParameterKind::PositionalOnly) + .then(|| parameter.name.clone()), + }; if positional_ok && parameter.kind.is_positional() && !was_keyword { - by_position.push((start, value)); + by_position.push(argument); } else { - if parameter.kind == ParameterKind::PositionalOnly { + if argument.keyword.is_none() { return Err(format!( "would have to pass `{}` by keyword, which it only takes by position", parameter.name )); } positional_ok = false; - by_keyword.push((start, format!("{}={value}", parameter.name))); + by_keyword.push(argument); } } if !extra_positional.is_empty() || !extra_keyword.is_empty() { return Err("passes extra arguments the new parameters have nowhere to take".to_string()); } - // what is dropped, or moved after something written after it, must have no effect + // what is dropped must have no effect for (slot, argument) in bound_to.iter().enumerate() { if let Some(argument) = argument && !used[slot] @@ -738,37 +1013,82 @@ fn rewrite_call( )); } } - by_keyword.sort_by_key(|(start, _)| start.unwrap_or(TextSize::from(u32::MAX))); - let order: Vec = by_position - .iter() - .chain(&by_keyword) - .filter_map(|(start, _)| *start) - .collect(); - for (earlier, later) in order.iter().zip(order.iter().skip(1)) { - if later < earlier { - let moved = arguments - .args - .iter() - .chain(arguments.keywords.iter().map(|keyword| &keyword.value)) - .find(|value| value.start() == *earlier); - if let Some(moved) = moved - && !is_effect_free(model, moved) - { - return Err(format!( - "would evaluate `{}` after an argument written after it", - &source[moved.range()] - )); + + // python evaluates arguments in the order they are written. the arguments passed by + // position are written in the order of the new parameters, which can evaluate one before an + // argument written before it; where that could be told apart, the arguments from there on + // are passed by keyword instead, written in the order the call wrote them + let in_written_order = |arguments: &mut Vec| { + arguments.sort_by_key(|argument| { + argument + .value + .map_or(TextSize::from(u32::MAX), Ranged::start) + }); + }; + let mut layout = None; + let mut first_reordered = None; + for count in (0..=by_position.len()).rev() { + if by_position[count..] + .iter() + .any(|argument| argument.keyword.is_none()) + { + break; + } + let mut keywords: Vec = by_position[count..] + .iter() + .chain(&by_keyword) + .cloned() + .collect(); + in_written_order(&mut keywords); + let evaluated: Vec<&Expr> = by_position[..count] + .iter() + .chain(&keywords) + .filter_map(|argument| argument.value) + .collect(); + match reordered_pair(model, &evaluated) { + Some(pair) => { + first_reordered.get_or_insert(pair); + } + None => { + layout = Some((count, keywords)); + break; } } } + let Some((count, keywords)) = layout else { + let Some((earlier, later)) = first_reordered else { + return Err("cannot keep the order its arguments are evaluated in".to_string()); + }; + return Err(format!( + "would evaluate `{}` after `{}`, which is written after it, and either could change \ + what the other does", + &source[earlier.range()], + &source[later.range()] + )); + }; - let mut written: Vec = Vec::new(); + let mut written: Vec> = Vec::new(); if let Some(receiver) = receiver_argument { - written.push(source[receiver.range()].to_string()); + written.push(vec![Piece::Source(receiver.range())]); + } + written.extend( + by_position + .into_iter() + .take(count) + .map(|argument| vec![argument.piece]), + ); + written.extend(keywords.into_iter().map(|argument| match argument.keyword { + Some(keyword) => vec![Piece::Text(format!("{keyword}=")), argument.piece], + None => vec![argument.piece], + })); + let mut pieces = vec![Piece::Text("(".to_string())]; + for (position, argument) in written.into_iter().enumerate() { + if position > 0 { + pieces.push(Piece::Text(", ".to_string())); + } + pieces.extend(argument); } - written.extend(by_position.into_iter().map(|(_, text)| text)); - written.extend(by_keyword.into_iter().map(|(_, text)| text)); - let rewritten = format!("({})", written.join(", ")); + pieces.push(Piece::Text(")".to_string())); let original: Vec<&str> = arguments .args .iter() @@ -780,13 +1100,74 @@ fn rewrite_call( .map(|keyword| &source[keyword.range()]), ) .collect(); - if format!("({})", original.join(", ")) == rewritten { + let as_written: String = pieces + .iter() + .map(|piece| match piece { + Piece::Text(text) => text.as_str(), + Piece::Source(range) => &source[*range], + }) + .collect(); + if format!("({})", original.join(", ")) == as_written { return Ok(None); } - Ok(Some(rewritten)) + Ok(Some(pieces)) +} + +/// a piece of the text a call's arguments are written again as +#[derive(Clone)] +enum Piece { + Text(String), + /// the text of the file there, with what the change does inside it + Source(TextRange), } -/// The call whose callee the reference at `covering` names: `f` of `f(...)` or of `x.f(...)`. +/// `edits`, with the calls whose arguments `calls` writes again, as edits that do not overlap, +/// which an editor would refuse: an edit inside an argument a call writes again, a parameter the +/// argument reads or a call of the function, is made in the argument's new text, and one in the +/// rest of the arguments, a keyword the call writes anew, is left out +fn composed( + db: &dyn Db, + mut edits: Vec, + mut calls: Vec<(File, TextRange, Vec)>, +) -> Vec { + // a call inside another's arguments is written first, so the outer one takes it in + calls.sort_by_key(|(_, range, _)| range.len()); + for (file, range, pieces) in calls { + let source = source_text(db, file); + let (inside, outside): (Vec, Vec) = edits + .into_iter() + .partition(|edit| edit.file == file && range.contains_range(edit.edit.range())); + edits = outside; + let mut text = String::new(); + for piece in pieces { + match piece { + Piece::Text(written) => text.push_str(&written), + Piece::Source(piece_range) => { + let mut written = source[piece_range].to_string(); + let mut within: Vec<&FileEdit> = inside + .iter() + .filter(|edit| piece_range.contains_range(edit.edit.range())) + .collect(); + within.sort_by_key(|edit| std::cmp::Reverse(edit.edit.start())); + for edit in within { + written.replace_range( + (edit.edit.range() - piece_range.start()).to_std_range(), + edit.edit.content().unwrap_or_default(), + ); + } + text.push_str(&written); + } + } + } + edits.push(FileEdit { + file, + edit: Edit::range_replacement(text, range), + }); + } + edits +} + +/// the call whose callee the reference at `covering` names: `f` of `f(...)` or of `x.f(...)` fn call_of<'a>( covering: &ruff_python_ast::find_node::CoveringNode<'a>, ) -> Option<&'a ast::ExprCall> { @@ -807,8 +1188,8 @@ fn call_of<'a>( } } -/// Whether `call` passes the method's receiver itself, as a call through an instance does, rather -/// than as its first argument, as a call through the class does. +/// whether `call` passes the method's receiver itself, as a call through an instance does, rather +/// than as its first argument, as a call through the class does fn is_bound_call( model: &SemanticModel<'_>, call: &ast::ExprCall, @@ -828,8 +1209,8 @@ fn is_bound_call( !through_class || resolved_method_kind(model, function) == Some(MethodKind::Class) } -/// The function of `file` whose definition `offset` is in the name or the parameters of, and -/// whether it is a method that is passed its receiver. +/// the function of `file` whose definition `offset` is in the name or the parameters of, and +/// whether it is a method that is passed its receiver fn function_at<'p>( db: &dyn Db, file: ProgramFile<'_>, @@ -864,8 +1245,8 @@ fn is_in_class_body( .is_some_and(|node| matches!(node, AnyNodeRef::StmtClassDef(_))) } -/// The file and offset of the function named at `offset`: `offset` itself when it is in a -/// definition's name or parameters, or else where the name at `offset` is defined. +/// the file and offset of the function named at `offset`: `offset` itself when it is in a +/// definition's name or parameters, or else where the name at `offset` is defined fn definition_site<'db>( db: &'db dyn Db, file: ProgramFile<'db>, @@ -892,7 +1273,224 @@ fn definition_site<'db>( Some((target_file, target.focus_range().start())) } -/// `file`'s path relative to the project's root. +/// where the change writes source it is given +#[derive(Clone, Copy)] +enum Place { + Annotation, + Returns, + Default, + /// an argument of a call, by position or by keyword + Argument, +} + +impl Place { + /// whether `text`, written here, is read as one whole expression, as a file that is + /// basedpython or not reads it: `int, str` is two parameters, and `1)` ends the list early + fn fits(self, text: &str, basedpython: bool) -> bool { + // a keyword argument after another argument rules out what only a lone or a positional + // argument can be: a bare generator, a starred expression, an unparenthesized walrus + let (prefix, suffix) = match self { + Self::Annotation => ("def _(p: ", "): pass"), + Self::Returns => ("def _() -> ", ": pass"), + Self::Default => ("def _(p=", "): pass"), + Self::Argument => ("_(_, p=", ")"), + }; + let code = format!("{prefix}{text}{suffix}\n"); + let Ok(parsed) = ruff_python_parser::parse( + &code, + ParseOptions::from(Mode::Module).with_basedpython(basedpython), + ) else { + return false; + }; + let Some(module) = parsed.syntax().as_module() else { + return false; + }; + let read = match (self, module.body.as_slice()) { + (Self::Argument, [ast::Stmt::Expr(statement)]) => match &*statement.value { + Expr::Call(call) => call + .arguments + .keywords + .first() + .map(|keyword| keyword.value.range()), + _ => None, + }, + (Self::Returns, [ast::Stmt::FunctionDef(function)]) => { + function.returns.as_ref().map(|returns| returns.range()) + } + (Self::Annotation, [ast::Stmt::FunctionDef(function)]) => function + .parameters + .args + .first() + .and_then(|parameter| parameter.parameter.annotation.as_ref()) + .map(|annotation| annotation.range()), + (Self::Default, [ast::Stmt::FunctionDef(function)]) => function + .parameters + .args + .first() + .and_then(|parameter| parameter.default.as_ref()) + .map(|default| default.range()), + _ => None, + }; + read == Some(TextRange::at(TextSize::of(prefix), TextSize::of(text))) + } +} + +/// why the body of the function whose scope is `body` already means something by `name`, which a +/// parameter of that name would change: a name it binds itself, or one it reads from outside the +/// function, as a global, a builtin or an enclosing function's, in its own scope or in a scope +/// nested in it that does not bind the name first +fn body_meaning_of(index: &SemanticIndex<'_>, body: FileScopeId, name: &str) -> Option { + let table = index.place_table(body); + if let Some(symbol) = table.symbol_id(name) { + let symbol = table.symbol(symbol); + return Some( + if symbol.is_local() || symbol.is_global() || symbol.is_nonlocal() { + format!("the body binds `{name}` already") + } else { + format!("the body reads `{name}` from outside the function") + }, + ); + } + descendants(index, body) + .any(|nested| { + index.place_table(nested).symbol_id(name).is_some() + && reaches(index, nested, body, name) + }) + .then(|| format!("the body reads `{name}` from outside the function")) +} + +/// whether the name `name` the scope `from` mentions resolves through `to`, an enclosing function +/// scope, before any scope on the way binds it or sends it to the module with `global` +fn reaches(index: &SemanticIndex<'_>, from: FileScopeId, to: FileScopeId, name: &str) -> bool { + for (scope, _) in index.visible_ancestor_scopes(from) { + if scope == to { + return true; + } + let table = index.place_table(scope); + if let Some(symbol) = table.symbol_id(name) { + let symbol = table.symbol(symbol); + if symbol.is_global() || symbol.is_local() { + return false; + } + } + } + false +} + +/// whether the read at `read` of a parameter of the function whose scope is `body`, renamed to +/// `name`, would resolve to a binding of `name` in a scope nested in the function instead +fn nested_binding_reads( + index: &SemanticIndex<'_>, + parsed: &ParsedModuleRef, + body: FileScopeId, + read: TextRange, + name: &str, +) -> bool { + let covering = covering_node(parsed.syntax().into(), read); + let Some(scope) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::ExprName(expr) => index.try_expression_scope_id(&ast::ExprRef::Name(expr)), + _ => None, + }) else { + return false; + }; + scope != body && !reaches(index, scope, body, name) +} + +/// a reference to the function a change rewrites +struct FunctionReference { + range: TextRange, + /// whether it spells the function's own name, which a new name renames, rather than a name an + /// import binds it to under another name + spells_the_name: bool, +} + +/// every reference to `function` in the project, other than its name where it is defined, by file +/// in the order of their paths. a name +/// an import binds it to under another name, `from m import area as a`, is followed to its own +/// references, wherever they are, since a call through it is a call of the function too +fn function_references( + db: &dyn Db, + file: ProgramFile<'_>, + model: &SemanticModel<'_>, + parsed: &ParsedModuleRef, + function: &ast::StmtFunctionDef, +) -> Result)>, String> { + let goto = find_goto_target(model, parsed, function.name.start()) + .ok_or_else(|| format!("`{}` cannot be searched for", function.name))?; + let mut pending: Vec<(File, TextRange, bool)> = + references(db, file, &goto, ReferencesMode::RenameMultiFile) + .unwrap_or_default() + .into_iter() + .map(|reference| (reference.file(), reference.range(), true)) + .collect(); + let mut seen: FxHashSet<(File, TextRange)> = FxHashSet::default(); + seen.insert((file.file(db), function.name.range())); + let mut by_file: FxHashMap> = FxHashMap::default(); + while let Some((reference_file, range, spells_the_name)) = pending.pop() { + if !seen.insert((reference_file, range)) { + continue; + } + by_file + .entry(reference_file) + .or_default() + .push(FunctionReference { + range, + spells_the_name, + }); + let program_file = ProgramFile::new(db, reference_file, file.program(db)); + let parsed = parsed_module(db, program_file.python_file(db)).load(db); + let covering = covering_node(parsed.syntax().into(), range); + let Some(asname) = covering.ancestors().find_map(|node| match node { + AnyNodeRef::Alias(alias) if alias.name.range() == range => alias.asname.as_ref(), + _ => None, + }) else { + continue; + }; + let model = SemanticModel::new(db, program_file); + let Some(goto) = find_goto_target(&model, &parsed, asname.start()) else { + return Err(format!( + "{}:{} imports `{}` as `{asname}`, which cannot be searched for", + relative_path(db, reference_file), + line_number(source_text(db, reference_file).as_str(), range.start()), + function.name + )); + }; + pending.extend( + references(db, program_file, &goto, ReferencesMode::RenameMultiFile) + .unwrap_or_default() + .into_iter() + .map(|reference| (reference.file(), reference.range(), false)), + ); + } + let mut by_file: Vec<(File, Vec)> = by_file.into_iter().collect(); + by_file.sort_by_key(|(file, _)| file.path(db).as_str().to_string()); + for (_, references) in &mut by_file { + references.sort_by_key(|reference| reference.range.start()); + } + Ok(by_file) +} + +/// whether the scope `function` is defined in binds `name` +fn binds_beside( + db: &dyn Db, + file: ProgramFile<'_>, + function: &ast::StmtFunctionDef, + name: &str, +) -> bool { + let index = ty_python_core::semantic_index(db, file); + let Some(definition) = index + .try_definitions(AnyNodeRef::from(function)) + .and_then(<[_]>::first) + else { + return false; + }; + let table = index.place_table(definition.scope(db).file_scope_id(db)); + table + .symbol_id(name) + .is_some_and(|symbol| table.symbol(symbol).is_bound()) +} + +/// `file`'s path relative to the project's root fn relative_path(db: &dyn Db, file: File) -> String { let path = file.path(db); let root = db.project().root(db); @@ -905,7 +1503,10 @@ fn relative_path(db: &dyn Db, file: File) -> String { mod tests { use insta::assert_snapshot; - use super::{NewParameter, ParameterKind, change_signature, function_signature}; + use super::{ + AnnotationChange, NewParameter, ParameterKind, SignatureChange, change_signature, + function_signature, + }; use crate::refactor::move_symbol::tests::rendered; use crate::tests::{CursorTest, CursorTestBuilder}; @@ -933,13 +1534,14 @@ print(area(width=2)) .build() } - /// `name`, as the parameter `old` was, of `kind`, with `default`. + /// `name`, as the parameter `old` was, of `kind`, with `default` fn kept(name: &str, old: &str, kind: ParameterKind, default: Option<&str>) -> NewParameter { NewParameter { name: name.to_string(), old_name: Some(old.to_string()), kind, default: default.map(ToString::to_string), + annotation: AnnotationChange::Keep, value: None, } } @@ -955,6 +1557,7 @@ print(area(width=2)) old_name: None, kind, default: default.map(ToString::to_string), + annotation: AnnotationChange::Keep, value: value.map(ToString::to_string), } } @@ -963,12 +1566,19 @@ print(area(width=2)) impl CursorTest { fn change(&self, parameters: &[NewParameter]) -> String { + self.change_to(&SignatureChange { + parameters: parameters.to_vec(), + ..SignatureChange::default() + }) + } + + fn change_to(&self, change: &SignatureChange) -> String { let changed = salsa::attach(&self.db, || { change_signature( &self.db, self.program_file(self.cursor.file), self.cursor.offset, - parameters, + change, ) }); rendered(&self.db, changed) @@ -1017,6 +1627,7 @@ print(area(width=2)) annotation: None, }, ], + returns: None, }, ) "#); @@ -1052,9 +1663,9 @@ print(area(width=2)) ); } - /// Without the call that unpacks into it, or the one that leaves `height` to the default it + /// without the call that unpacks into it, or the one that leaves `height` to the default it /// loses, swapping the two parameters swaps what each call passes by position, and a call - /// that passed `height` by keyword passes both by keyword, in the order it wrote them. + /// that passed `height` by keyword passes both by keyword, in the order it wrote them #[test] fn reordering_rewrites_each_call() { let test = project( @@ -1085,7 +1696,7 @@ print(area(width=2)) "); } - /// `forward` unpacks `*args` into `area`, which cannot be rebound. + /// `forward` unpacks `*args` into `area`, which cannot be rebound #[test] fn a_call_that_unpacks_is_refused() { let test = project(GEO); @@ -1096,8 +1707,8 @@ print(area(width=2)) ]), @"refused: sig/geo.py:6 unpacks arguments into the call, which cannot be told apart"); } - /// A parameter added with a default leaves every call valid, so none is touched, the one - /// that unpacks included. + /// a parameter added with a default leaves every call valid, so none is touched, the one + /// that unpacks included #[test] fn adding_a_parameter_with_a_default_changes_no_call() { let test = project(GEO); @@ -1122,7 +1733,7 @@ print(area(width=2)) "); } - /// A required parameter is passed the value given, by position while it can be. + /// a required parameter is passed the value given, by position while it can be #[test] fn adding_a_required_parameter_passes_its_value() { let test = project(&GEO.replace( @@ -1172,7 +1783,7 @@ print(area(width=2)) ]), @"refused: the body still reads `height`, on line 2"); } - /// A rename renames the body's reads and the calls' keywords. + /// a rename renames the body's reads and the calls' keywords #[test] fn renaming_renames_the_body_and_keywords() { let test = project(&GEO.replace( @@ -1197,7 +1808,7 @@ print(area(width=2)) "); } - /// Dropping an argument whose evaluation could have an effect would lose the effect. + /// dropping an argument whose evaluation could have an effect would lose the effect #[test] fn dropping_an_argument_with_an_effect_is_refused() { let test = CursorTestBuilder::default() @@ -1223,8 +1834,8 @@ print(area(width=2)) "); } - /// A method called through an instance is passed its receiver; through the class, its first - /// argument is the receiver. + /// a method called through an instance is passed its receiver; through the class, its first + /// argument is the receiver #[test] fn a_method_s_calls_keep_their_receiver() { let test = CursorTestBuilder::default() @@ -1256,4 +1867,362 @@ Shape.scaled(shape, 2, offset=1) Shape.scaled(shape, factor=2, offset=1) "); } + + /// a new name renames the function where it is defined and everywhere it is referred to, + /// imports included, alongside what the parameters do to its calls + #[test] + fn a_function_is_renamed_and_annotated() { + let test = project(GEO); + let mut width = kept("width", "width", PositionalOrKeyword, None); + width.annotation = AnnotationChange::Set(Some("float".to_string())); + assert_snapshot!(test.change_to(&SignatureChange { + name: Some("surface".to_string()), + parameters: vec![ + width, + kept("height", "height", PositionalOrKeyword, Some("1")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ], + returns: AnnotationChange::Set(Some("float".to_string())), + }), @" + --- /sig/geo.py + def surface(width: float, height=1, *, scale=1.0) -> float: + return width * height * scale + + + def forward(*args, **kwargs): + return surface(*args, **kwargs) + + + print(surface(2, 3)) + print(surface(2, height=3, scale=2.0)) + print(surface(width=2)) + --- /sig/user.py + from sig.geo import surface + + surface(4, 5, scale=3) + "); + } + + /// an annotation set to none is removed, and one kept is written as it was + #[test] + fn annotations_are_removed_and_kept() { + let test = CursorTestBuilder::default() + .source( + "main.py", + "def area(width: int, height: int = 1) -> int:\n return width * height\n", + ) + .build(); + let mut width = kept("width", "width", PositionalOrKeyword, None); + width.annotation = AnnotationChange::Set(None); + let mut height = kept("height", "height", PositionalOrKeyword, Some("2")); + height.annotation = AnnotationChange::Set(Some("float".to_string())); + let mut depth = added("depth", KeywordOnly, Some("0"), None); + depth.annotation = AnnotationChange::Set(Some("int".to_string())); + assert_snapshot!(test.change_to(&SignatureChange { + name: None, + parameters: vec![width, height, depth], + returns: AnnotationChange::Set(None), + }), @" + --- /main.py + def area(width, height: float = 2, *, depth: int = 0): + return width * height + "); + assert_snapshot!(test.change_to(&SignatureChange { + name: Some("area".to_string()), + parameters: vec![ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + ], + returns: AnnotationChange::Keep, + }), @" + --- /main.py + def area(height: int, width: int) -> int: + return width * height + "); + } + + /// a name a function cannot have, or one its scope binds already, and an annotation that is + /// not an expression, are refused + #[test] + fn a_name_or_annotation_that_cannot_be_written_is_refused() { + let test = project(GEO); + let keep = vec![ + kept("width", "width", PositionalOrKeyword, None), + kept("height", "height", PositionalOrKeyword, Some("1")), + kept("scale", "scale", KeywordOnly, Some("1.0")), + ]; + let renamed = |name: &str| SignatureChange { + name: Some(name.to_string()), + parameters: keep.clone(), + returns: AnnotationChange::Keep, + }; + assert_snapshot!(test.change_to(&renamed("class")), @"refused: `class` is a keyword"); + assert_snapshot!(test.change_to(&renamed("2d")), @"refused: `2d` is not a valid name"); + assert_snapshot!(test.change_to(&renamed("forward")), @"refused: `forward` is bound already where `area` is defined"); + assert_snapshot!(test.change_to(&SignatureChange { + name: None, + parameters: keep.clone(), + returns: AnnotationChange::Set(Some("int (".to_string())), + }), @"refused: `int (` is not an annotation"); + } + + fn single(source: &str) -> CursorTest { + CursorTestBuilder::default() + .source("main.py", source) + .build() + } + + /// a parameter renamed to a name the body reads from outside the function would read + /// itself instead + #[test] + fn renaming_to_a_name_the_body_reads_from_outside_is_refused() { + let global = single("b = 1\n\n\ndef f(a):\n return a + b\n\n\nf(2)\n"); + assert_snapshot!(global.change(&[kept("b", "a", PositionalOrKeyword, None)]), @"refused: the body reads `b` from outside the function"); + let builtin = single("def f(a):\n return len(a)\n\n\nf([])\n"); + assert_snapshot!(builtin.change(&[kept("len", "a", PositionalOrKeyword, None)]), @"refused: the body reads `len` from outside the function"); + let nested = single("b = 1\n\n\ndef f(a):\n return lambda: a + b\n\n\nf(2)\n"); + assert_snapshot!(nested.change(&[kept("b", "a", PositionalOrKeyword, None)]), @"refused: the body reads `b` from outside the function"); + let local = single("def f(a):\n b = 1\n return a + b\n\n\nf(2)\n"); + assert_snapshot!(local.change(&[kept("b", "a", PositionalOrKeyword, None)]), @"refused: the body binds `b` already"); + } + + /// a read of the renamed parameter in a nested scope that binds the new name itself would + /// read that scope's own binding + #[test] + fn renaming_to_a_name_a_nested_scope_binds_is_refused() { + let test = single( + "def f(a):\n def g(b):\n return a + b\n return g(1)\n\n\nf(2)\n", + ); + assert_snapshot!(test.change(&[kept("b", "a", PositionalOrKeyword, None)]), @"refused: `a` on line 3 would read the `b` a scope inside `f` binds"); + } + + /// a parameter added under a name the body reads from outside would be read instead + #[test] + fn adding_a_parameter_the_body_reads_from_outside_is_refused() { + let test = single("b = 1\n\n\ndef f(a):\n return a + b\n\n\nf(2)\n"); + assert_snapshot!(test.change(&[ + kept("a", "a", PositionalOrKeyword, None), + added("b", PositionalOrKeyword, Some("0"), None), + ]), @"refused: the body reads `b` from outside the function"); + } + + /// two parameters that swap their names read each other's argument, which is what the + /// rename asks for + #[test] + fn parameters_may_swap_their_names() { + let test = single("def f(a, b):\n return a - b\n\n\nf(1, 2)\n"); + assert_snapshot!(test.change(&[ + kept("b", "a", PositionalOrKeyword, None), + kept("a", "b", PositionalOrKeyword, None), + ]), @" + --- /main.py + def f(b, a): + return b - a + + + f(1, 2) + "); + } + + /// reordering arguments evaluates them in a new order; where an argument's effect could be + /// seen by another, the call keeps their order by passing them by keyword + #[test] + fn reordering_keeps_the_order_effects_happen_in() { + let test = single( + "def bump():\n return 1\n\n\ndef f(a, b, c):\n return a\n\n\nn = 1\nf(bump(), n, 3)\nf(1, 2, bump())\n", + ); + assert_snapshot!(test.change(&[ + kept("b", "b", PositionalOrKeyword, None), + kept("c", "c", PositionalOrKeyword, None), + kept("a", "a", PositionalOrKeyword, None), + ]), @" + --- /main.py + def bump(): + return 1 + + + def f(b, c, a): + return a + + + n = 1 + f(a=bump(), b=n, c=3) + f(2, bump(), 1) + "); + } + + /// an argument only taken by position cannot be passed by keyword to keep its order + #[test] + fn reordering_positional_only_arguments_with_effects_is_refused() { + let test = single( + "def bump():\n return 1\n\n\ndef f(a, b, /):\n return a\n\n\nn = 1\nf(bump(), n)\n", + ); + assert_snapshot!(test.change(&[ + kept("b", "b", ParameterKind::PositionalOnly, None), + kept("a", "a", ParameterKind::PositionalOnly, None), + ]), @"refused: main.py:10 would evaluate `bump()` after `n`, which is written after it, and either could change what the other does"); + } + + /// a call through a name an import gives the function is a call of it too + #[test] + fn a_call_through_an_import_alias_is_rewritten() { + let test = CursorTestBuilder::default() + .source( + "m.py", + "def area(width, height):\n return width * height\n", + ) + .source("user.py", "from m import area as a\n\na(2, 3)\n") + .build(); + assert_snapshot!(test.change(&[ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + ]), @" + --- /m.py + def area(height, width): + return width * height + --- /user.py + from m import area as a + + a(3, 2) + "); + } + + /// a keyword-only parameter added before `**kwargs` would take a keyword a call passes to + /// `**kwargs` + #[test] + fn a_new_parameter_that_takes_a_keyword_from_kwargs_is_refused() { + let test = single("def f(**kw):\n return kw\n\n\nf(x=1)\n"); + assert_snapshot!(test.change(&[ + added("x", KeywordOnly, Some("0"), None), + kept("kw", "kw", ParameterKind::VariadicKeyword, None), + ]), @"refused: main.py:5 passes `x` into `**kwargs`, where the parameter `x` would take it instead"); + assert_snapshot!(test.change(&[ + added("x", KeywordOnly, None, Some("2")), + kept("kw", "kw", ParameterKind::VariadicKeyword, None), + ]), @"refused: main.py:5 passes `x` into `**kwargs`, where the parameter `x` would take it instead"); + let unpacked = single("def f(**kw):\n return kw\n\n\nf(**{})\n"); + assert_snapshot!(unpacked.change(&[ + added("x", KeywordOnly, Some("0"), None), + kept("kw", "kw", ParameterKind::VariadicKeyword, None), + ]), @"refused: main.py:5 unpacks arguments into the call, which cannot be told apart"); + } + + /// a default or a value that is not one expression, as it would be written, is refused + #[test] + fn a_default_or_value_that_is_not_an_expression_is_refused() { + let test = single("def f(a):\n return a\n\n\nf(1)\n"); + assert_snapshot!(test.change(&[ + kept("a", "a", PositionalOrKeyword, None), + added("b", PositionalOrKeyword, Some("1)"), None), + ]), @"refused: `1)` is not an expression a default can be"); + assert_snapshot!(test.change(&[ + kept("a", "a", PositionalOrKeyword, None), + added("b", PositionalOrKeyword, None, Some("1, 2")), + ]), @"refused: `1, 2` is not an expression a call can pass"); + assert_snapshot!(test.change(&[ + kept("a", "a", PositionalOrKeyword, None), + added("b", PositionalOrKeyword, None, Some("*x")), + ]), @"refused: `*x` is not an expression a call can pass"); + let mut annotated = kept("a", "a", PositionalOrKeyword, None); + annotated.annotation = AnnotationChange::Set(Some("int, str".to_string())); + assert_snapshot!(test.change(&[annotated]), @"refused: `int, str` is not an annotation"); + } + + /// a decorator that binds the name to something other than the function itself decides + /// what its calls pass, which the change cannot see + #[test] + fn a_function_a_decorator_replaces_is_refused() { + let test = single( + "def inject(f):\n return lambda *args: f(0, *args)\n\n\n@inject\ndef f(a, b):\n return a\n\n\nf(1)\n", + ); + assert_snapshot!(test.change(&[ + kept("b", "b", PositionalOrKeyword, None), + kept("a", "a", PositionalOrKeyword, None), + ]), @"refused: `f` is decorated, and its decorators make what its calls reach something other than the function itself"); + } + + /// a call of the function in its own body is rewritten as any other is + #[test] + fn a_recursive_call_is_rewritten() { + let test = + single("def f(a, b=0):\n return f(a=a - 1, b=b) if a else b\n\n\nf(3)\n"); + assert_snapshot!(test.change(&[ + kept("x", "a", PositionalOrKeyword, None), + kept("b", "b", PositionalOrKeyword, Some("0")), + ]), @" + --- /main.py + def f(x, b=0): + return f(x=x - 1, b=b) if x else b + + + f(3) + "); + } + + /// a call passed to a call of the function is rewritten inside the outer call's new + /// arguments + #[test] + fn a_call_inside_a_call_is_rewritten() { + let test = single("def f(a, b):\n return a - b\n\n\nf(f(1, 2), 3)\n"); + assert_snapshot!(test.change(&[ + kept("b", "b", PositionalOrKeyword, None), + kept("a", "a", PositionalOrKeyword, None), + ]), @" + --- /main.py + def f(b, a): + return a - b + + + f(3, f(2, 1)) + "); + } + + /// `@staticmethod` and `@classmethod` leave the function as it is called + #[test] + fn a_static_or_class_method_is_changed() { + let test = single( + "class Shape:\n @staticmethod\n def area(width, height):\n return width * height\n\n\nShape.area(2, 3)\n", + ); + assert_snapshot!(test.change(&[ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + ]), @" + --- /main.py + class Shape: + @staticmethod + def area(height, width): + return width * height + + + Shape.area(3, 2) + "); + let test = single( + "class Shape:\n @classmethod\n def area(cls, width, height):\n return width * height\n\n\nShape.area(2, 3)\n", + ); + assert_snapshot!(test.change(&[ + kept("height", "height", PositionalOrKeyword, None), + kept("width", "width", PositionalOrKeyword, None), + ]), @" + --- /main.py + class Shape: + @classmethod + def area(cls, height, width): + return width * height + + + Shape.area(3, 2) + "); + } + + /// one function of an `@overload` set is one of several its calls may reach + #[test] + fn an_overloaded_function_is_refused() { + let test = single( + "from typing import overload\n\n\n@overload\ndef f(a: int) -> int: ...\n@overload\ndef f(a: str) -> str: ...\ndef f(a):\n return a\n\n\nf(1)\n", + ); + assert_snapshot!(test.change(&[ + kept("a", "a", PositionalOrKeyword, None), + added("b", PositionalOrKeyword, Some("0"), None), + ]), @"refused: `f` is defined more than once where it is defined, as an `@overload` set or a redefinition, and its calls may reach any of the definitions"); + } } diff --git a/crates/ty_ide/src/refactor/flow.rs b/crates/ty_ide/src/refactor/flow.rs index 6e75b55ad8..1de2eee0eb 100644 --- a/crates/ty_ide/src/refactor/flow.rs +++ b/crates/ty_ide/src/refactor/flow.rs @@ -155,7 +155,7 @@ pub(crate) fn rebound_from_nested_scope( } /// The scopes nested, at any depth, inside `scope`. -fn descendants<'a>( +pub(crate) fn descendants<'a>( index: &'a SemanticIndex<'_>, scope: FileScopeId, ) -> impl Iterator + 'a { diff --git a/crates/ty_server/src/server/api/requests/change_signature.rs b/crates/ty_server/src/server/api/requests/change_signature.rs index 1e788dd8c4..41838b2c17 100644 --- a/crates/ty_server/src/server/api/requests/change_signature.rs +++ b/crates/ty_server/src/server/api/requests/change_signature.rs @@ -8,11 +8,14 @@ use lsp_types::{ LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, }; use ruff_db::files::FileRange; -use ty_ide::{NewParameter, ParameterKind, change_signature, function_signature}; +use ty_ide::{ + AnnotationChange, NewParameter, ParameterKind, SignatureChange, change_signature, + function_signature, +}; use ty_project::{ProjectDatabase, SemanticDb as _}; -use super::move_symbol::MoveSymbolAnswer; -use crate::document::{FileRangeExt, PositionExt}; +use super::move_symbol::{MoveSymbolAnswer, position}; +use crate::document::FileRangeExt; use crate::server::api::traits::{ BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, }; @@ -87,6 +90,8 @@ pub(crate) struct Signature { range: Range, /// its parameters, without the `self` or `cls` of a method, which stays as it is parameters: Vec, + /// its return annotation, as the source writes it + returns: Option, } #[derive(Debug, serde::Deserialize, serde::Serialize)] @@ -145,6 +150,7 @@ impl BackgroundDocumentRequestHandler for PrepareChangeSignatureRequestHandler { annotation: parameter.annotation, }) .collect(), + returns: signature.returns, })) } } @@ -167,7 +173,40 @@ impl Request for ChangeSignatureRequest { pub(crate) struct ChangeSignatureParams { text_document: TextDocumentIdentifier, position: Position, + /// its new name; absent keeps the one it has + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, parameters: Vec, + /// its return annotation + #[serde(default, skip_serializing_if = "WrittenAnnotation::is_kept")] + returns: WrittenAnnotation, +} + +/// an annotation written as source, where `null` removes the one there is and a field left out +/// keeps it: the field's absence and its `null` say different things, so it is not an `Option` +#[derive(Debug, Default)] +pub(crate) struct WrittenAnnotation(AnnotationChange); + +impl WrittenAnnotation { + fn is_kept(&self) -> bool { + self.0 == AnnotationChange::Keep + } +} + +impl<'de> serde::Deserialize<'de> for WrittenAnnotation { + fn deserialize>(deserializer: D) -> Result { + as serde::Deserialize>::deserialize(deserializer) + .map(|annotation| Self(AnnotationChange::Set(annotation))) + } +} + +impl serde::Serialize for WrittenAnnotation { + fn serialize(&self, serializer: S) -> Result { + match &self.0 { + AnnotationChange::Keep => serializer.serialize_unit(), + AnnotationChange::Set(annotation) => annotation.serialize(serializer), + } + } } #[derive(Debug, serde::Deserialize, serde::Serialize)] @@ -181,6 +220,9 @@ pub(crate) struct ChangedParameter { /// its default, written as source; absent for none #[serde(default)] default: Option, + /// its annotation; absent keeps the one it has, none for a new parameter + #[serde(default, skip_serializing_if = "WrittenAnnotation::is_kept")] + annotation: WrittenAnnotation, /// for a new parameter, what each call passes it, written as source; absent leaves it to its /// default #[serde(default)] @@ -208,21 +250,26 @@ impl BackgroundDocumentRequestHandler for ChangeSignatureRequestHandler { let Some((file, offset)) = position(db, snapshot, params.position) else { return Ok(None); }; - let parameters: Vec = params - .parameters - .into_iter() - .map(|parameter| NewParameter { - name: parameter.name, - old_name: parameter.old_name, - kind: parameter.kind.into(), - default: parameter.default, - value: parameter.value, - }) - .collect(); + let change = SignatureChange { + name: params.name, + parameters: params + .parameters + .into_iter() + .map(|parameter| NewParameter { + name: parameter.name, + old_name: parameter.old_name, + kind: parameter.kind.into(), + default: parameter.default, + annotation: parameter.annotation.0, + value: parameter.value, + }) + .collect(), + returns: params.returns.0, + }; Ok(Some(MoveSymbolAnswer::of( db, snapshot, - change_signature(db, db.program_file(file), offset, ¶meters), + change_signature(db, db.program_file(file), offset, &change), ))) } } @@ -231,25 +278,3 @@ impl RetriableRequestHandler for ChangeSignatureRequestHandler { /// the user is waiting on the change they asked for const RETRY_ON_CANCELLATION: bool = true; } - -/// the file and offset of a position in the snapshot's document, `None` where the request is not -/// answered -fn position( - db: &ProjectDatabase, - snapshot: &DocumentSnapshot, - position: Position, -) -> Option<(ruff_db::files::File, ruff_text_size::TextSize)> { - if snapshot - .workspace_settings() - .is_language_services_disabled() - || snapshot.is_django_template() - { - return None; - } - let file = snapshot.document().to_notebook_or_file(db)?; - if !file.source_type(db).is_py_file() { - return None; - } - let offset = position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding())?; - Some((file, offset)) -} diff --git a/crates/ty_server/src/server/api/requests/move_symbol.rs b/crates/ty_server/src/server/api/requests/move_symbol.rs index d88e6d458d..f2f260ff8a 100644 --- a/crates/ty_server/src/server/api/requests/move_symbol.rs +++ b/crates/ty_server/src/server/api/requests/move_symbol.rs @@ -10,9 +10,9 @@ use lsp_types::{ LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, TextEdit, Uri, WorkspaceEdit, }; -use ruff_db::files::{FileRange, system_path_to_file}; +use ruff_db::files::{File, FileRange, system_path_to_file}; use ruff_db::system::SystemPathBuf; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextSize}; use rustc_hash::FxHashMap; use ty_ide::{MovableKind, SymbolMove, movable_symbol, move_symbol}; use ty_project::{ProjectDatabase, SemanticDb as _}; @@ -77,24 +77,7 @@ impl BackgroundDocumentRequestHandler for PrepareMoveSymbolRequestHandler { _client: &Client, params: PrepareMoveSymbolParams, ) -> crate::server::Result> { - if snapshot - .workspace_settings() - .is_language_services_disabled() - || snapshot.is_django_template() - { - return Ok(None); - } - let Some(file) = snapshot.document().to_notebook_or_file(db) else { - return Ok(None); - }; - if !file.source_type(db).is_py_file() { - return Ok(None); - } - let Some(offset) = - params - .position - .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) - else { + let Some((file, offset)) = position(db, snapshot, params.position) else { return Ok(None); }; let Some(symbol) = movable_symbol(db, db.program_file(file), offset) else { @@ -177,24 +160,7 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { _client: &Client, params: MoveSymbolParams, ) -> crate::server::Result> { - if snapshot - .workspace_settings() - .is_language_services_disabled() - || snapshot.is_django_template() - { - return Ok(None); - } - let Some(file) = snapshot.document().to_notebook_or_file(db) else { - return Ok(None); - }; - if !file.source_type(db).is_py_file() { - return Ok(None); - } - let Some(offset) = - params - .position - .to_text_size(db, file, snapshot.document().uri(), snapshot.encoding()) - else { + let Some((file, offset)) = position(db, snapshot, params.position) else { return Ok(None); }; let Some(target) = params @@ -263,3 +229,26 @@ impl RetriableRequestHandler for MoveSymbolRequestHandler { /// the user is waiting on the move they asked for const RETRY_ON_CANCELLATION: bool = true; } + +/// the file and offset of a position in the snapshot's document, `None` where a refactoring of a +/// definition is not answered: when language services are off, and for a template, a notebook or +/// a file that is not source code +pub(super) fn position( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + position: Position, +) -> Option<(File, TextSize)> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + || snapshot.is_django_template() + { + return None; + } + let file = snapshot.document().to_notebook_or_file(db)?; + if !file.source_type(db).is_py_file() { + return None; + } + let offset = position.to_text_size(db, file, snapshot.document().uri(), snapshot.encoding())?; + Some((file, offset)) +} diff --git a/crates/ty_server/tests/e2e/change_signature.rs b/crates/ty_server/tests/e2e/change_signature.rs index 05df270505..c5d2b1da0e 100644 --- a/crates/ty_server/tests/e2e/change_signature.rs +++ b/crates/ty_server/tests/e2e/change_signature.rs @@ -67,6 +67,7 @@ fn a_call_names_the_function_and_its_parameters() -> Result<()> { { "name": "width", "kind": "positionalOrKeyword", "default": null, "annotation": "float" }, { "name": "height", "kind": "positionalOrKeyword", "default": "1", "annotation": "float" }, ], + "returns": "float", })) ); Ok(()) @@ -118,3 +119,82 @@ fn a_change_answers_the_edit_to_the_definition_and_every_call() -> Result<()> { ); Ok(()) } + +/// `name` renames the function, a parameter's `annotation` replaces its own and `null` removes it, +/// and `returns` does the same for the return annotation; each absent leaves what is written +#[test] +fn a_change_renames_the_function_and_rewrites_its_annotations() -> Result<()> { + let mut server = server()?; + let uri = server.file_uri(SystemPath::new("src/main.py")); + let answer = server + .send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 0, "character": 5 }, + "name": "surface", + "parameters": [ + { "name": "width", "oldName": "width", "kind": "positionalOrKeyword", "annotation": null }, + { "name": "height", "oldName": "height", "kind": "positionalOrKeyword", "default": "1", "annotation": "int" }, + ], + "returns": null, + })) + .expect("an answer"); + assert_eq!( + answer, + json!({ + "edit": { "changes": { uri.as_str(): [ + { + "range": { "start": { "line": 0, "character": 4 }, "end": { "line": 0, "character": 8 } }, + "newText": "surface", + }, + { + "range": { "start": { "line": 0, "character": 8 }, "end": { "line": 0, "character": 41 } }, + "newText": "(width, height: int = 1)", + }, + { + "range": { "start": { "line": 0, "character": 41 }, "end": { "line": 0, "character": 50 } }, + "newText": "", + }, + { + "range": { "start": { "line": 4, "character": 6 }, "end": { "line": 4, "character": 10 } }, + "newText": "surface", + }, + ] } }, + "warnings": [], + }) + ); + Ok(()) +} + +/// a refusal answers why, and no edit +#[test] +fn a_change_that_would_alter_what_the_body_reads_is_refused() -> Result<()> { + const SCALE: &str = "\ +FACTOR = 2 + + +def scale(x): + return x * FACTOR + + +print(scale(3)) +"; + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/scale.py"), SCALE)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/scale.py"), SCALE, 1); + let answer = server.send_request_await::(json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/scale.py")) }, + "position": { "line": 3, "character": 5 }, + "parameters": [{ "name": "FACTOR", "oldName": "x", "kind": "positionalOrKeyword" }], + })); + assert_eq!( + answer, + Some(json!({ + "refused": "the body reads `FACTOR` from outside the function", + "warnings": [], + })) + ); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index e0655cddc2..cb405b76dd 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -848,8 +848,10 @@ and answers as `by/moveSymbol` does ### changing a signature `by/changeSignature` gives a function a new parameter list — parameters added, -removed, reordered, renamed, given or stripped of defaults, moved between kinds — -and rewrites every call of it in the project to pass them. each call is bound the +removed, reordered, renamed, given or stripped of defaults or annotations, moved +between kinds — and rewrites every call of it in the project to pass them. it can +rename the function too, wherever the project refers to it, imports included, and +give it a new return annotation. each call is bound the way python binds it, its positional arguments to the positional parameters and its keywords by name, and written again for the new list: by position while the parameters still line up, by keyword after that, a keyword staying a keyword, and @@ -880,17 +882,26 @@ parameter without a default; the function used other than called, unless every c stays valid. and where the result would not be python, or not the same program: a removed parameter the body still reads, a parameter without a default after one with a default, two of one name, kinds out of order, or a method another class of the -project overrides or that overrides another's, whose signatures would part +project overrides or that overrides another's, whose signatures would part. a new +name that is not an identifier, is a keyword or is bound already in the scope the +function is defined in is refused, and so is a default, a value or an annotation that is not one expression +where it is written. also refused are a new parameter name the body already binds +or reads from outside the function, a function its decorators bind to something +other than itself, and one defined more than once where it is defined, as an +`@overload` set is `by/prepareChangeSignature` takes `textDocument` and `position` — in a function's -name or parameters, or in a name that refers to it — and answers `{name, uri, range, parameters}`, its parameters each `{name, kind, default, annotation}` with `kind` +name or parameters, or in a name that refers to it — and answers `{name, uri, range, parameters, returns}`, `returns` its return annotation as the source writes it or `null`, and its parameters each `{name, kind, default, annotation}` with `kind` one of `positionalOnly`, `positionalOrKeyword`, `variadic` (`*args`), `keywordOnly` and `variadicKeyword` (`**kwargs`), and `default` and `annotation` as the source writes them; a method's `self` or `cls` is not listed and stays as it is. -`by/changeSignature` takes the same position and `parameters`, the whole new list -in order, each `{name, oldName?, kind, default?, value?}`: `oldName` the parameter it -was, absent for a new one; `default` written as source, absent for none; `value` what -each call passes a new one. it answers as `by/moveSymbol` does, `{edit, warnings}` or +`by/changeSignature` takes the same position, `name?`, the function's new name, absent +to keep it, `parameters`, the whole new list in order, each +`{name, oldName?, kind, default?, annotation?, value?}`, and `returns?`: `oldName` the +parameter it was, absent for a new one; `default` written as source, absent for +none; `annotation` and `returns` written as source, `null` to remove the one there +is, and absent to keep it (none, for a new parameter); `value` what each call passes +a new one. it answers as `by/moveSymbol` does, `{edit, warnings}` or `{refused, warnings}` ## unwrap and remove From a33e0bde2d3d8267ee33bfb3c728e89cd7f22c65 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:21:47 +1000 Subject: [PATCH 067/110] answer each member by/prepareMoveMembers lists with its detail, a method or property as its name and its parameters as written on one line, fetch(self, key: str), an attribute as its name, for a client to list them as python's pull members up dialog does --- crates/ty_ide/src/refactor/class_members.rs | 78 ++++++++++++++++++- .../src/server/api/requests/move_members.rs | 4 + crates/ty_server/tests/e2e/move_symbol.rs | 2 + docs/basedpython/features/editor.md | 6 +- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index c1d2ea588e..83595b3419 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -81,6 +81,9 @@ pub struct ClassMember { pub kind: MemberKind, /// The range of its name in its first statement. pub name_range: TextRange, + /// How it reads in a list of members: a method or property as its name and its parameters + /// as its first statement writes them, `fetch(self, key: str)`, an attribute as its name. + pub detail: String, } /// A class of the project, where its name is. @@ -114,6 +117,7 @@ pub fn movable_members( ) -> Option { let class = ClassSite::at(db, file, offset)?; let def = class.def()?; + let source = source_text(db, class.file_id); let mut members: Vec = Vec::new(); for statement in &def.body { let Some((name, kind)) = member_of(statement) else { @@ -132,6 +136,7 @@ pub fn movable_members( name: name.to_string(), kind, name_range: member_name_range(statement)?, + detail: member_detail(source.as_str(), statement), }); } Some(MovableMembers { @@ -964,6 +969,45 @@ fn member_of(statement: &Stmt) -> Option<(&Name, MemberKind)> { } } +/// See [`ClassMember::detail`]. Each parameter is written as the source writes it, but on one +/// line, with the `/` and `*` that separate the kinds. +fn member_detail(source: &str, statement: &Stmt) -> String { + let Stmt::FunctionDef(function) = statement else { + return member_of(statement).map_or_else(String::new, |(name, _)| name.to_string()); + }; + let written = |range: TextRange| { + source[range] + .split_whitespace() + .collect::>() + .join(" ") + }; + let variadic = |star: &str, parameter: &ast::Parameter| { + let text = written(parameter.range()); + if text.starts_with('*') { + text + } else { + format!("{star}{text}") + } + }; + let parameters = &function.parameters; + let mut parts: Vec = Vec::new(); + parts.extend(parameters.posonlyargs.iter().map(|p| written(p.range()))); + if !parameters.posonlyargs.is_empty() { + parts.push("/".to_string()); + } + parts.extend(parameters.args.iter().map(|p| written(p.range()))); + match ¶meters.vararg { + Some(vararg) => parts.push(variadic("*", vararg)), + None if !parameters.kwonlyargs.is_empty() => parts.push("*".to_string()), + None => {} + } + parts.extend(parameters.kwonlyargs.iter().map(|p| written(p.range()))); + if let Some(kwarg) = ¶meters.kwarg { + parts.push(variadic("**", kwarg)); + } + format!("{}({})", function.name, parts.join(", ")) +} + fn member_kind(statement: &Stmt) -> MemberKind { let Stmt::FunctionDef(function) = statement else { return MemberKind::Attribute; @@ -1638,7 +1682,7 @@ class GrandChild(Child): movable .members .iter() - .map(|member| format!("{} ({:?})", member.name, member.kind)) + .map(|member| format!("{} ({:?})", member.detail, member.kind)) .collect::>() .join(", "), movable @@ -1662,12 +1706,42 @@ class GrandChild(Child): let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); assert_snapshot!(test.movable(), @" cls.shapes.Child - members: LIMIT (Attribute), __init__ (Method), area (Method), plain (Method), helper (Method) + members: LIMIT (Attribute), __init__(self, x) (Method), area(self) (Method), plain(self) (Method), helper(self) (Method) superclasses: cls.shapes.Base subclasses: cls.shapes.GrandChild "); } + /// A method reads as its parameters, on one line whatever lines they are written on. + #[test] + fn a_member_reads_as_its_name_and_parameters() { + let test = CursorTestBuilder::default() + .source( + "main.py", + r#"class Store: + SIZE: int = 3 + + def fetch(self, key: str, /, *args, + default=None, **options): + return key + + def only(self, *, strict: bool = True): + return strict + + @property + def size(self): + return self.SIZE +"#, + ) + .build(); + assert_snapshot!(test.movable(), @" + main.Store + members: SIZE (Attribute), fetch(self, key: str, /, *args, default=None, **options) (Method), only(self, *, strict: bool = True) (Method), size(self) (Property) + superclasses: + subclasses: + "); + } + #[test] fn a_method_that_needs_nothing_else_is_pulled_up() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); diff --git a/crates/ty_server/src/server/api/requests/move_members.rs b/crates/ty_server/src/server/api/requests/move_members.rs index 873cd77e64..5d15b77003 100644 --- a/crates/ty_server/src/server/api/requests/move_members.rs +++ b/crates/ty_server/src/server/api/requests/move_members.rs @@ -69,6 +69,9 @@ pub(crate) struct MemberItem { name: String, /// `method`, `classmethod`, `staticmethod`, `property` or `attribute` kind: String, + /// how it reads in a list: a method or property as its name and parameters as written, + /// `fetch(self, key: str)`, an attribute as its name + detail: String, /// its name, in its first statement range: Range, } @@ -134,6 +137,7 @@ impl BackgroundDocumentRequestHandler for PrepareMoveMembersRequestHandler { MemberKind::Attribute => "attribute", } .to_string(), + detail: member.detail, range, }) }) diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index a978da377a..f0170a0db6 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -271,11 +271,13 @@ fn a_class_lists_its_members_and_where_they_could_go() -> Result<()> { { "name": "SIDES", "kind": "attribute", + "detail": "SIDES", "range": { "start": { "line": 5, "character": 4 }, "end": { "line": 5, "character": 9 } }, }, { "name": "describe", "kind": "method", + "detail": "describe(self)", "range": { "start": { "line": 7, "character": 8 }, "end": { "line": 7, "character": 16 } }, }, ], diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index cb405b76dd..621224ceba 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -801,8 +801,10 @@ it once nothing above has the name `by/prepareMoveMembers` takes `textDocument` and `position` (anywhere in a class) and answers `{class, members, superclasses, subclasses}`: the class and the classes -of the project above it, nearest first, and below it, each `{name, qualifiedName, uri, range}` with `range` its name; and its members, each `{name, kind, range}`, -`kind` one of `method`, `classmethod`, `staticmethod`, `property` and `attribute`; +of the project above it, nearest first, and below it, each `{name, qualifiedName, uri, range}` with `range` its name; and its members, each `{name, kind, detail, range}`, +`kind` one of `method`, `classmethod`, `staticmethod`, `property` and `attribute`, +`detail` how the member reads in a list — a method or property as its name and its +parameters as written, on one line, `fetch(self, key: str)`, an attribute as its name; `null` outside a class. `by/pullMembersUp` takes the same position, `members`, the names to move, and `target`, a superclass's `qualifiedName`; `by/pushMembersDown` takes the position and `members`. both answer as `by/moveSymbol` does, From 097e7fe50f95487cf6d8d5f04fc1ffc109b88699 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:30:59 +1000 Subject: [PATCH 068/110] pull a base the class statement names up into the superclass's bases with by/pullMembersUp, listed by by/prepareMoveMembers as a member of kind base, as python's pull members up offers a mixin, refusing it unless the superclass and every class of the project below it find each name where they did and python can still order their bases, the class still reaches the superclass, and the base is bound above the superclass in the class's module --- crates/ty_ide/src/refactor/class_members.rs | 282 ++++++++-- .../src/refactor/class_members/bases.rs | 504 ++++++++++++++++++ crates/ty_ide/src/refactor/move_symbol.rs | 8 +- .../src/types/ide_support.rs | 14 +- .../src/server/api/requests/move_members.rs | 4 +- .../src/server/api/requests/move_symbol.rs | 2 +- crates/ty_server/tests/e2e/move_symbol.rs | 6 + docs/basedpython/features/editor.md | 14 +- 8 files changed, 763 insertions(+), 71 deletions(-) create mode 100644 crates/ty_ide/src/refactor/class_members/bases.rs diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index 83595b3419..9d9da9d56e 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -1,5 +1,5 @@ -//! Pulling a class's members up into one of its superclasses, and pushing them down into its -//! subclasses. +//! pulling a class's members up into one of its superclasses, and pushing them down into its +//! subclasses //! //! ```py //! class Shape: class Shape: @@ -10,11 +10,11 @@ //! return type(self).__name__ pass //! ``` //! -//! A member is every statement of a class body that binds its name: a method (with its property -//! accessors or overloads, which share the name), or a class attribute. It moves with its -//! decorators and the comments directly above it, reindented for the body it lands in. +//! a member is every statement of a class body that binds its name: a method (with its property +//! accessors or overloads, which share the name), or a class attribute. it moves with its +//! decorators and the comments directly above it, reindented for the body it lands in //! -//! Either move is refused, with the reason, where the program would no longer mean what it +//! either move is refused, with the reason, where the program would no longer mean what it //! meant: //! //! - pulled up, a member that reads an attribute of its receiver the superclass does not have, @@ -32,7 +32,7 @@ //! `override` follows the move: a method pulled up overrides nothing where it lands unless the //! superclass inherits the name, so it loses the modifier there, and the methods of the same //! name below the superclass then override it and gain the modifier; a method pushed down keeps -//! it only where something above still has the name. +//! it only where something above still has the name use ruff_db::files::File; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; @@ -64,7 +64,9 @@ use crate::import_order; use crate::method_kind::{MethodKind, method_kind}; use crate::references::{ReferencesMode, references}; -/// What kind of member of a class a member is. +mod bases; + +/// what kind of member of a class a member is #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MemberKind { Method, @@ -72,44 +74,47 @@ pub enum MemberKind { StaticMethod, Property, Attribute, + /// A base the class statement names, which a pull up moves into the superclass's bases. + Base, } -/// A member [`pull_members_up`] or [`push_members_down`] can move. +/// A member [`pull_members_up`] or [`push_members_down`] can move, or a base a pull up can. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ClassMember { + /// Its name, or a base as the class statement writes it. pub name: String, pub kind: MemberKind, - /// The range of its name in its first statement. + /// The range of its name in its first statement, or of the base. pub name_range: TextRange, /// How it reads in a list of members: a method or property as its name and its parameters /// as its first statement writes them, `fetch(self, key: str)`, an attribute as its name. pub detail: String, } -/// A class of the project, where its name is. +/// a class of the project, where its name is #[derive(Debug, Clone, PartialEq, Eq)] pub struct HierarchyClass { pub file: File, pub name: String, - /// Its module's dotted name and its own, the way [`pull_members_up`] is told which - /// superclass to pull into. + /// its module's dotted name and its own, the way [`pull_members_up`] is told which + /// superclass to pull into pub qualified_name: Option, pub name_range: TextRange, } -/// The members of the class at a position, and the classes of the project above and below it -/// they could move to. +/// the members of the class at a position, and the classes of the project above and below it +/// they could move to #[derive(Debug, Clone, PartialEq, Eq)] pub struct MovableMembers { pub class: HierarchyClass, pub members: Vec, - /// The project's classes the class inherits from, nearest first. + /// the project's classes the class inherits from, nearest first pub superclasses: Vec, - /// The project's classes that name the class as a base. + /// the project's classes that name the class as a base pub subclasses: Vec, } -/// The class around `offset`, its members, and where they could go. +/// the class around `offset`, its members, and where they could go pub fn movable_members( db: &dyn Db, file: ProgramFile<'_>, @@ -139,6 +144,16 @@ pub fn movable_members( detail: member_detail(source.as_str(), statement), }); } + members.extend( + bases::written_bases(def, source.as_str()) + .into_iter() + .map(|base| ClassMember { + name: base.text.to_string(), + kind: MemberKind::Base, + name_range: base.expression.range(), + detail: base.text.to_string(), + }), + ); Some(MovableMembers { class: class.item(db), members, @@ -183,12 +198,32 @@ pub fn pull_members_up( .def() .ok_or_else(|| format!("{target} cannot be read"))?; let env = ProgramEnvironment::from_file(class.file); - let super_members = instance_member_names(db, &env, superclass.ty); - let pulled: FxHashSet<&str> = names.iter().map(String::as_str).collect(); + let mut super_members = instance_member_names(db, &env, superclass.ty); let source = source_text(db, class.file.file(db)); let source = source.as_str(); + // a name the class body binds is a member; any other is one of the bases it writes + let (names, pulled_bases): (Vec<&String>, Vec<&String>) = names + .iter() + .partition(|name| !member_statements(def, name).is_empty()); let mut moved = SymbolMove::default(); + if !pulled_bases.is_empty() { + let pulled = bases::pull_bases_up( + db, + &class, + superclass, + &pulled_bases + .iter() + .map(|name| name.as_str()) + .collect::>(), + source, + )?; + // what the bases bring is the superclass's from now on, for the members pulled with them + super_members.extend(pulled.gained); + moved.edits.extend(pulled.edits); + } + let pulled: FxHashSet<&str> = names.iter().map(|name| name.as_str()).collect(); + let mut texts: Vec<(usize, MemberKind, String)> = Vec::new(); let mut removed: Vec = Vec::new(); let mut target_imports: Vec = Vec::new(); @@ -353,9 +388,9 @@ pub fn pull_members_up( Ok(moved) } -/// Moves the members `names` of the class around `offset` into each of the project's classes +/// moves the members `names` of the class around `offset` into each of the project's classes /// that name it as a base, as a copy into each that does not define the member itself. `Err` -/// says why they cannot be moved. +/// says why they cannot be moved pub fn push_members_down( db: &dyn Db, file: ProgramFile<'_>, @@ -742,7 +777,7 @@ fn module_binds(class: &ClassSite<'_>, db: &dyn Db, name: &str) -> bool { .is_some_and(|symbol| globals.symbol(symbol).is_bound()) } -/// A class of the project, parsed. +/// a class of the project, parsed struct ClassSite<'db> { file: ProgramFile<'db>, file_id: File, @@ -754,7 +789,7 @@ struct ClassSite<'db> { } impl<'db> ClassSite<'db> { - /// The innermost class whose definition `offset` is in. + /// the innermost class whose definition `offset` is in fn at(db: &'db dyn Db, file: ProgramFile<'db>, offset: TextSize) -> Option { let parsed = parsed_module(db, file.python_file(db)).load(db); let def = all_classes(&parsed) @@ -766,7 +801,7 @@ impl<'db> ClassSite<'db> { Self::new(db, file, parsed, name_range, ty) } - /// The class whose name is at `name_range` in `file`, of the type `ty`. + /// the class whose name is at `name_range` in `file`, of the type `ty` fn new( db: &'db dyn Db, file: ProgramFile<'db>, @@ -794,7 +829,7 @@ impl<'db> ClassSite<'db> { }) } - /// The class described by `class`, a class of the type hierarchy, of the type `ty`. + /// the class described by `class`, a class of the type hierarchy, of the type `ty` fn of_hierarchy( db: &'db dyn Db, program: ProgramFile<'db>, @@ -829,7 +864,7 @@ impl<'db> ClassSite<'db> { } } - /// The project's classes this class names as its bases. + /// the project's classes this class names as its bases fn bases(&self, db: &'db dyn Db) -> Vec> { let env = ProgramEnvironment::from_file(self.file); class_bases(db, &env, self.ty) @@ -841,7 +876,7 @@ impl<'db> ClassSite<'db> { .collect() } - /// The project's classes this class inherits from, nearest first. + /// the project's classes this class inherits from, nearest first fn ancestors(&self, db: &'db dyn Db) -> Vec> { let mut found: Vec> = Vec::new(); let mut pending = self.bases(db); @@ -858,7 +893,7 @@ impl<'db> ClassSite<'db> { found } - /// Whether this class names `base` as its one base. + /// whether this class names `base` as its one base fn has_only_base(&self, db: &'db dyn Db, base: &ClassSite<'_>) -> bool { let env = ProgramEnvironment::from_file(self.file); let bases = class_bases(db, &env, self.ty); @@ -872,7 +907,7 @@ impl<'db> ClassSite<'db> { .any(|ancestor| ancestor.same(self)) } - /// The project's classes that name this class as a base. + /// the project's classes that name this class as a base fn subclasses(&self, db: &'db dyn Db) -> Vec> { let env = ProgramEnvironment::from_file(self.file); let modules: Vec<_> = @@ -898,7 +933,7 @@ impl<'db> ClassSite<'db> { .collect() } - /// The project's classes below this one, at any depth. + /// the project's classes below this one, at any depth fn descendants(&self, db: &'db dyn Db) -> Vec> { let mut found: Vec> = Vec::new(); let mut pending = self.subclasses(db); @@ -933,7 +968,7 @@ impl<'db> ClassSite<'db> { } } -/// Every class definition of a module, at any depth. +/// every class definition of a module, at any depth fn all_classes(parsed: &ParsedModuleRef) -> Vec<&ast::StmtClassDef> { #[derive(Default)] struct Classes<'a> { @@ -1040,7 +1075,7 @@ fn binds(statement: &Stmt, name: &str) -> bool { member_of(statement).is_some_and(|(bound, _)| bound.as_str() == name) } -/// The indices of the statements of `class`'s body that bind `name`. +/// the indices of the statements of `class`'s body that bind `name` fn member_statements(class: &ast::StmtClassDef, name: &str) -> Vec { class .body @@ -1230,8 +1265,8 @@ fn has_override(statement: &Stmt) -> bool { override_marker(statement).is_some() } -/// The range that takes `decorator` out: an `@override` line with its line break, or the -/// `override` modifier keyword with the space after it. +/// the range that takes `decorator` out: an `@override` line with its line break, or the +/// `override` modifier keyword with the space after it fn override_range(source: &str, decorator: &ast::Decorator) -> TextRange { if source[decorator.range()].starts_with('@') { TextRange::new( @@ -1264,7 +1299,7 @@ fn member_text(source: &str, body: &[Stmt], index: usize, strip_override: bool) cut.text } -/// The indentation the statements of `class`'s body are written with. +/// the indentation the statements of `class`'s body are written with fn body_indentation(source: &str, class: &ast::StmtClassDef) -> String { class.body.first().map_or_else( || " ".to_string(), @@ -1275,7 +1310,7 @@ fn body_indentation(source: &str, class: &ast::StmtClassDef) -> String { ) } -/// `text` with each line's leading `from` replaced by `to`. +/// `text` with each line's leading `from` replaced by `to` fn reindent(text: &str, from: &str, to: &str) -> String { if from == to { return text.to_string(); @@ -1288,7 +1323,7 @@ fn reindent(text: &str, from: &str, to: &str) -> String { .collect() } -/// Whether a body statement stands in for no body: `pass` or `...`. +/// whether a body statement stands in for no body: `pass` or `...` fn is_placeholder(statement: &Stmt) -> bool { match statement { Stmt::Pass(_) => true, @@ -1301,8 +1336,8 @@ fn is_docstring(statement: &Stmt) -> bool { matches!(statement, Stmt::Expr(expression) if expression.value.is_string_literal_expr()) } -/// The edits taking the statements `removed` out of `class`'s body, leaving `pass` when nothing -/// but a docstring would be left. +/// the edits taking the statements `removed` out of `class`'s body, leaving `pass` when nothing +/// but a docstring would be left fn remove_members( file: File, source: &str, @@ -1375,9 +1410,9 @@ fn remove_members( edits } -/// The edits adding `members` to `class`'s body: an attribute after its last attribute, before +/// the edits adding `members` to `class`'s body: an attribute after its last attribute, before /// its first method; a method after its last statement, a blank line away; in place of a body -/// that is only `pass` or `...`. +/// that is only `pass` or `...` fn insert_members( file: File, source: &str, @@ -1471,8 +1506,8 @@ fn insert_members( edits } -/// The edits importing `requests` into `class`'s module, each under the name the moved code -/// reads it by. +/// the edits importing `requests` into `class`'s module, each under the name the moved code +/// reads it by fn add_imports( db: &dyn Db, class: &ClassSite<'_>, @@ -1502,8 +1537,8 @@ fn add_imports( .collect()) } -/// The edits marking `statement`, a method of `class`, as an override: basedpython's `override` -/// modifier, or python's `@override` with its import. +/// the edits marking `statement`, a method of `class`, as an override: basedpython's `override` +/// modifier, or python's `@override` with its import fn add_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec { let Stmt::FunctionDef(function) = statement else { return Vec::new(); @@ -1559,7 +1594,7 @@ fn add_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec, statement: &Stmt) -> Vec { let Some(decorator) = override_marker(statement) else { return Vec::new(); @@ -1571,7 +1606,7 @@ fn remove_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec< }] } -/// The type of the receiver of the attribute whose name is at `range` in `file`: `x` of `x.name`. +/// the type of the receiver of the attribute whose name is at `range` in `file`: `x` of `x.name` fn attribute_receiver_type<'db>( db: &'db dyn Db, file: ProgramFile<'db>, @@ -1706,7 +1741,7 @@ class GrandChild(Child): let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); assert_snapshot!(test.movable(), @" cls.shapes.Child - members: LIMIT (Attribute), __init__(self, x) (Method), area(self) (Method), plain(self) (Method), helper(self) (Method) + members: LIMIT (Attribute), __init__(self, x) (Method), area(self) (Method), plain(self) (Method), helper(self) (Method), Base (Base) superclasses: cls.shapes.Base subclasses: cls.shapes.GrandChild "); @@ -1742,6 +1777,139 @@ class GrandChild(Child): "); } + fn mixins(source: &str) -> CursorTest { + CursorTestBuilder::default() + .source("main.py", source) + .build() + } + + /// A base the class writes is pulled up into the superclass's bases, after its own, where + /// every class below the superclass finds what it found before. + #[test] + fn a_base_is_pulled_up_into_the_superclass() { + let test = mixins( + r#"class Mixin: + def mix(self): + return 1 + + +class Base: + def keep(self): + return 1 + + +class Child(Base, Mixin, metaclass=type): + def own(self): + return self.mix() +"#, + ); + assert_snapshot!(test.movable(), @" + main.Child + members: own(self) (Method), Base (Base), Mixin (Base) + superclasses: main.Base, main.Mixin + subclasses: + "); + assert_snapshot!(test.pull_up(&["Mixin", "own"], "main.Base"), @" + --- /main.py + class Mixin: + def mix(self): + return 1 + + + class Base(Mixin): + def keep(self): + return 1 + + def own(self): + return self.mix() + + + class Child(Base, metaclass=type): + pass + "); + } + + /// Whatever a class below the superclass finds, it has to find in the same class afterwards. + #[test] + fn a_base_that_changes_what_a_class_finds_is_refused() { + let test = mixins( + r#"class Mixin: + def keep(self): + return 2 + + +class Base: + def keep(self): + return 1 + + +class Child(Mixin, Base): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: Child would find `keep` in another class than it does"); + + let test = mixins( + r#"class Mixin: + def shared(self): + return 2 + + +class Other: + def shared(self): + return 3 + + +class Base: + pass + + +class Child(Base, Mixin): + pass + + +class Sibling(Base, Other): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: Sibling would find `shared` in another class than it does"); + } + + /// The superclass names the base where it is defined, so the name has to be bound there, and + /// the class cannot lose the base it reaches the superclass through. + #[test] + fn a_base_the_superclass_cannot_take_is_refused() { + let test = mixins( + r#"class Base: + pass + + +class Mixin: + pass + + +class Child(Base, Mixin): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: `Mixin` is not bound above Base, where it would be written"); + + let test = mixins( + r#"class Base: + pass + + +class Middle(Base): + pass + + +class Child(Middle): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["Middle"], "main.Base"), @"refused: Child inherits Base through `Middle`, so it cannot lose it"); + } + #[test] fn a_method_that_needs_nothing_else_is_pulled_up() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); @@ -1779,7 +1947,7 @@ class GrandChild(Child): } /// `area` reads `self.x`, which only a `Child` has; `Other` would inherit an `area` that - /// fails. + /// fails #[test] fn a_method_that_reads_what_the_superclass_lacks_is_refused() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); @@ -1787,7 +1955,7 @@ class GrandChild(Child): assert_snapshot!(test.pull_up(&["helper"], "cls.shapes.Base"), @"refused: `helper` uses `self.plain` on line 19, which Base does not have; pull it up too"); } - /// Pulled up together, `helper` finds the `plain` it calls. + /// pulled up together, `helper` finds the `plain` it calls #[test] fn members_pulled_up_together_satisfy_each_other() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); @@ -1824,14 +1992,14 @@ class GrandChild(Child): "); } - /// `Base().keep()` would no longer find `keep`. + /// `Base().keep()` would no longer find `keep` #[test] fn a_member_read_off_the_class_itself_is_not_pushed_down() { let test = project(&SHAPES.replace("class Base:", "class Base:")); assert_snapshot!(test.push_down(&["keep"]), @"refused: cls/user.py:3 uses `keep` on what may be a Base rather than a subclass"); } - /// `helper` calls `self.plain()` on what may be a `Child` rather than a `GrandChild`. + /// `helper` calls `self.plain()` on what may be a `Child` rather than a `GrandChild` #[test] fn a_member_the_class_s_own_methods_read_is_not_pushed_down() { let test = project(&SHAPES.replace("class Child(Base)", "class Child(Base)")); @@ -1885,8 +2053,8 @@ def use(child: Child, other: Other): "); } - /// Pulled up where nothing above has the name, a method overrides nothing, and what has - /// the name below the superclass then overrides it: in basedpython it gains the modifier. + /// pulled up where nothing above has the name, a method overrides nothing, and what has + /// the name below the superclass then overrides it: in basedpython it gains the modifier #[test] fn override_follows_a_pull_up() { let test = CursorTestBuilder::default() @@ -1934,7 +2102,7 @@ class Leaf(Child): "#); } - /// In python it gains `@override`, imported. + /// in python it gains `@override`, imported #[test] fn override_follows_a_pull_up_in_python() { let test = CursorTestBuilder::default() @@ -1976,7 +2144,7 @@ class Other(Base): "#); } - /// `Leaf` would inherit `Mid`'s `describe` instead of the one it pulls up. + /// `Leaf` would inherit `Mid`'s `describe` instead of the one it pulls up #[test] fn a_class_between_with_the_name_is_refused() { let test = CursorTestBuilder::default() diff --git a/crates/ty_ide/src/refactor/class_members/bases.rs b/crates/ty_ide/src/refactor/class_members/bases.rs new file mode 100644 index 0000000000..93d2365276 --- /dev/null +++ b/crates/ty_ide/src/refactor/class_members/bases.rs @@ -0,0 +1,504 @@ +//! Pulling a class's bases up into a superclass, as python's own pull members up offers a mixin: +//! +//! ```py +//! class Shape: ... class Shape(Mixin): ... +//! class Square(Shape, Mixin): ... ──► class Square(Shape): ... +//! ``` +//! +//! Every class below the superclass then inherits the base, so the pull is refused unless every +//! class whose bases change — the superclass and every class of the project below it — finds each +//! name it found before in the class it found it in. A name only the base brings is gained, which +//! is what the pull is for. Python has to be able to order each class's bases afterwards, and the +//! superclass has to be able to name the base where it is defined. + +use ruff_db::files::File; +use ruff_db::parsed::parsed_module; +use ruff_diagnostics::Edit; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; +use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_project::Db; +use ty_python_core::ProgramFile; +use ty_python_semantic::types::Type; +use ty_python_semantic::{ + ProgramEnvironment, class_bases, instance_member_names, type_hierarchy_mro, + type_hierarchy_prepare, +}; + +use super::{ClassSite, all_classes}; +use crate::code_action::FileEdit; + +/// A base a class statement writes, by name, which a pull up can move. +pub(super) struct WrittenBase<'a> { + /// The base as written, which is how a client names it. + pub(super) text: &'a str, + pub(super) expression: &'a Expr, +} + +/// The bases `def` writes as a name or a dotted name, the kind a superclass can write too. +pub(super) fn written_bases<'a>( + def: &'a ast::StmtClassDef, + source: &'a str, +) -> Vec> { + def.bases() + .iter() + .filter(|base| root_name(base).is_some()) + .map(|expression| WrittenBase { + text: &source[expression.range()], + expression, + }) + .filter(|base| base.text != "object") + .collect() +} + +/// The name a base's name or dotted name starts with. +fn root_name(expression: &Expr) -> Option<&Name> { + match expression { + Expr::Name(name) => Some(&name.id), + Expr::Attribute(attribute) => root_name(&attribute.value), + _ => None, + } +} + +/// What pulling bases up does: the edits, and the names the superclass gains with them. +pub(super) struct PulledBases { + pub(super) edits: Vec, + pub(super) gained: FxHashSet, +} + +/// A class, as the file and name range its statement has. +type ClassKey = (File, TextRange); + +/// Pulls the bases `pulled`, as `class` writes them, up into `superclass`, after its own bases. +pub(super) fn pull_bases_up<'db>( + db: &'db dyn Db, + class: &ClassSite<'db>, + superclass: &ClassSite<'db>, + pulled: &[&str], + source: &str, +) -> Result { + let def = class + .def() + .ok_or_else(|| "there is no class here".to_string())?; + let super_def = superclass + .def() + .ok_or_else(|| format!("{} cannot be read", superclass.name))?; + let env = ProgramEnvironment::from_file(class.file); + let key_of = |ty: Type<'db>| { + type_hierarchy_prepare(db, &env, ty) + .map(|class| (class.file.file(db), class.selection_range)) + }; + let written = written_bases(def, source); + let resolved = class_bases(db, &env, class.ty); + if resolved.len() != def.bases().len() { + return Err(format!( + "a base of {} is not a class that can be told apart", + class.name + )); + } + + // the bases pulled, each as written and as the class it is + let mut bases: Vec<(&WrittenBase, Type<'db>, ClassKey)> = Vec::new(); + for name in pulled { + let base = written + .iter() + .find(|base| base.text == *name) + .ok_or_else(|| format!("{} has no base `{name}`", class.name))?; + let index = def + .bases() + .iter() + .position(|expression| expression.range() == base.expression.range()) + .ok_or_else(|| format!("{} has no base `{name}`", class.name))?; + let ty = resolved[index]; + let key = key_of(ty).ok_or_else(|| format!("`{name}` is not a class"))?; + bases.push((base, ty, key)); + } + + let super_key = (superclass.file_id, superclass.name_range); + for (base, ty, key) in &bases { + // the superclass is reached through the base, which the class would lose + let below_base = mro_keys(db, &env, *ty); + if *key == super_key || below_base.contains(&super_key) { + return Err(format!( + "{} inherits {} through `{}`, so it cannot lose it", + class.name, superclass.name, base.text + )); + } + if mro_keys(db, &env, superclass.ty).contains(key) { + return Err(format!( + "{} inherits `{}` already", + superclass.name, base.text + )); + } + } + + // the superclass names the base by the name the class does, so the name has to mean the + // same where the superclass is defined: both at the top of one module, with the name bound + // above the superclass and nowhere else + if superclass.file_id != class.file_id { + return Err(format!( + "{} is in another module, which would have to import what it inherits", + superclass.name + )); + } + let suite = class.parsed.suite(); + let top_level = |def: &ast::StmtClassDef| { + suite + .iter() + .position(|statement| statement.range() == def.range()) + }; + let (Some(super_index), Some(_)) = (top_level(super_def), top_level(def)) else { + return Err(format!( + "{} or {} is not defined at the top of its module", + superclass.name, class.name + )); + }; + for (base, _, _) in &bases { + let Some(root) = root_name(base.expression) else { + continue; + }; + let bindings: Vec = suite + .iter() + .enumerate() + .filter(|(_, statement)| binds_at_top(statement, root)) + .map(|(index, _)| index) + .collect(); + if bindings.is_empty() || bindings.iter().any(|&index| index >= super_index) { + return Err(format!( + "`{root}` is not bound above {}, where it would be written", + superclass.name + )); + } + } + + // every class whose bases change orders them as python would, and finds what it found + let descendants = superclass.descendants(db); + let affected: Vec<&ClassSite<'db>> = std::iter::once(superclass) + .chain(descendants.iter()) + .collect(); + let pulled_keys: Vec = bases.iter().map(|(_, _, key)| *key).collect(); + let class_key = (class.file_id, class.name_range); + let mut new_bases: FxHashMap> = FxHashMap::default(); + // the class each key is, for the classes above whose bases stay as they are + let mut types: FxHashMap> = + bases.iter().map(|(_, ty, key)| (*key, *ty)).collect(); + for site in &affected { + let Some(site_def) = site.def() else { + return Err(format!("{} cannot be read", site.name)); + }; + let site_env = ProgramEnvironment::from_file(site.file); + let site_bases = class_bases(db, &site_env, site.ty); + if site_bases.len() != site_def.bases().len() { + return Err(format!( + "a base of {} is not a class that can be told apart", + site.name + )); + } + let mut keys: Vec = Vec::new(); + for base in site_bases { + let key = key_of(base).ok_or_else(|| { + format!( + "a base of {} is not a class that can be told apart", + site.name + ) + })?; + types.insert(key, base); + keys.push(key); + } + let key = (site.file_id, site.name_range); + if key == super_key { + keys.extend(pulled_keys.iter().copied()); + } + if key == class_key { + keys.retain(|base| !pulled_keys.contains(base)); + } + new_bases.insert(key, keys); + } + let mut linearized: FxHashMap>> = FxHashMap::default(); + let mut names = OwnNames { + db, + program: class.file, + cache: FxHashMap::default(), + }; + for site in &affected { + let key = (site.file_id, site.name_range); + let before: Vec = std::iter::once(key) + .chain(mro_keys(db, &env, site.ty)) + .collect(); + let after = + linearize(db, &env, key, &new_bases, &types, &mut linearized).ok_or_else(|| { + format!( + "python could not order the bases of {} once it inherits {}", + site.name, + bases + .iter() + .map(|(base, _, _)| format!("`{}`", base.text)) + .collect::>() + .join(", ") + ) + })?; + let found_before = names.found_through(&before); + let found_after = names.found_through(&after); + let mut changed: Vec<&Name> = found_before + .iter() + .filter(|(name, owner)| found_after.get(*name) != Some(*owner)) + .map(|(name, _)| name) + .collect(); + changed.sort(); + if let Some(name) = changed.first() { + return Err(format!( + "{} would find `{name}` in another class than it does", + site.name + )); + } + } + + let mut edits = remove_bases(class.file_id, def, &bases); + edits.push(FileEdit { + file: superclass.file_id, + edit: add_bases( + super_def, + &bases + .iter() + .map(|(base, _, _)| base.text) + .collect::>() + .join(", "), + ), + }); + let gained = bases + .iter() + .flat_map(|(_, ty, _)| instance_member_names(db, &env, *ty)) + .collect(); + Ok(PulledBases { edits, gained }) +} + +/// The classes `ty` inherits from in its method resolution order, itself not among them. +fn mro_keys<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) -> Vec { + type_hierarchy_mro(db, env, ty) + .into_iter() + .map(|class| (class.file.file(db), class.selection_range)) + .collect() +} + +/// The method resolution order of `key` with the bases `new_bases` gives the classes whose bases +/// change, by python's C3 linearization; `None` where python could not order them. A class whose +/// bases stay as they are keeps the order python gives it, as `types` says which class it is. +fn linearize<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + key: ClassKey, + new_bases: &FxHashMap>, + types: &FxHashMap>, + linearized: &mut FxHashMap>>, +) -> Option> { + if let Some(done) = linearized.get(&key) { + return done.clone(); + } + let Some(bases) = new_bases.get(&key) else { + let order = types + .get(&key) + .map(|ty| std::iter::once(key).chain(mro_keys(db, env, *ty)).collect()); + linearized.insert(key, order.clone()); + return order; + }; + // a cycle of classes has no order; marking the class first ends one + linearized.insert(key, None); + let mut sequences: Vec> = Vec::new(); + for base in bases { + sequences.push(linearize(db, env, *base, new_bases, types, linearized)?); + } + sequences.push(bases.clone()); + let order = merge(sequences).map(|merged| std::iter::once(key).chain(merged).collect()); + linearized.insert(key, order.clone()); + order +} + +/// The C3 merge of `sequences`: python's order for a class's ancestors, given each base's order +/// and the bases' own. +fn merge(mut sequences: Vec>) -> Option> { + let mut merged = Vec::new(); + loop { + sequences.retain(|sequence| !sequence.is_empty()); + if sequences.is_empty() { + return Some(merged); + } + let head = sequences.iter().map(|sequence| sequence[0]).find(|head| { + !sequences + .iter() + .any(|sequence| sequence[1..].contains(head)) + })?; + merged.push(head); + for sequence in &mut sequences { + if sequence[0] == head { + sequence.remove(0); + } + } + } +} + +/// The names each class's own body binds, the ones a lookup through it finds there. +struct OwnNames<'db> { + db: &'db dyn Db, + program: ProgramFile<'db>, + cache: FxHashMap>, +} + +impl OwnNames<'_> { + /// Where a lookup through `order`, a method resolution order, finds each name it finds. + fn found_through(&mut self, order: &[ClassKey]) -> FxHashMap { + let mut found = FxHashMap::default(); + for owner in order { + for name in self.of(*owner) { + found.entry(name).or_insert(*owner); + } + } + found + } + + fn of(&mut self, key: ClassKey) -> Vec { + if let Some(names) = self.cache.get(&key) { + return names.clone(); + } + let program = ProgramFile::new(self.db, key.0, self.program.program(self.db)); + let parsed = parsed_module(self.db, program.python_file(self.db)).load(self.db); + let mut binder = Binder::default(); + if let Some(def) = all_classes(&parsed) + .into_iter() + .find(|def| def.name.range() == key.1) + { + binder.visit_body(&def.body); + } + let mut names: Vec = binder.names.into_iter().collect(); + names.sort(); + self.cache.insert(key, names.clone()); + names + } +} + +/// The names statements bind in the scope they are in, not in the functions and classes they +/// define. Every name a store writes counts, and every name an import or a definition binds, so +/// that no name a class could be found to have is missed. +#[derive(Default)] +struct Binder { + names: FxHashSet, +} + +impl<'a> SourceOrderVisitor<'a> for Binder { + fn visit_stmt(&mut self, statement: &'a Stmt) { + match statement { + Stmt::FunctionDef(function) => { + self.names.insert(function.name.id.clone()); + for decorator in &function.decorator_list { + self.visit_decorator(decorator); + } + } + Stmt::ClassDef(class) => { + self.names.insert(class.name.id.clone()); + } + Stmt::Import(import) => { + for alias in &import.names { + let bound = alias.asname.as_ref().map_or_else( + || alias.name.id.split('.').next().unwrap_or_default(), + |asname| asname.id.as_str(), + ); + self.names.insert(Name::new(bound)); + } + } + Stmt::ImportFrom(import) => { + for alias in &import.names { + let bound = alias.asname.as_ref().unwrap_or(&alias.name); + self.names.insert(bound.id.clone()); + } + } + _ => source_order::walk_stmt(self, statement), + } + } + + fn visit_expr(&mut self, expression: &'a Expr) { + match expression { + Expr::Name(name) if name.ctx.is_store() => { + self.names.insert(name.id.clone()); + } + // their own scopes + Expr::Lambda(_) + | Expr::ListComp(_) + | Expr::SetComp(_) + | Expr::DictComp(_) + | Expr::Generator(_) => {} + _ => source_order::walk_expr(self, expression), + } + } +} + +/// Whether a top-level statement binds `name`. +fn binds_at_top(statement: &Stmt, name: &Name) -> bool { + let mut binder = Binder::default(); + binder.visit_stmt(statement); + binder.names.contains(name) +} + +/// The edits that take `bases` out of the list `def` writes. A run of them is deleted up to the +/// argument after it, or, at the end of the list, from the argument before it. +fn remove_bases( + file: File, + def: &ast::StmtClassDef, + bases: &[(&WrittenBase, Type, ClassKey)], +) -> Vec { + let Some(arguments) = &def.arguments else { + return Vec::new(); + }; + let ranges: Vec<(TextRange, bool)> = arguments + .iter_source_order() + .map(|argument| { + let range = argument.range(); + let removed = bases + .iter() + .any(|(base, _, _)| base.expression.range() == range); + (range, removed) + }) + .collect(); + let mut edits = Vec::new(); + let mut index = 0; + while index < ranges.len() { + if !ranges[index].1 { + index += 1; + continue; + } + let first = index; + while index < ranges.len() && ranges[index].1 { + index += 1; + } + let deleted = match (ranges.get(index), first.checked_sub(1)) { + (Some((next, _)), _) => TextRange::new(ranges[first].0.start(), next.start()), + (None, Some(previous)) => { + TextRange::new(ranges[previous].0.end(), ranges[index - 1].0.end()) + } + (None, None) => arguments.range(), + }; + edits.push(FileEdit { + file, + edit: Edit::range_deletion(deleted), + }); + } + edits +} + +/// The edit that writes `bases` after the bases `def` writes, before its keywords. +fn add_bases(def: &ast::StmtClassDef, bases: &str) -> Edit { + let Some(arguments) = &def.arguments else { + let after = def + .type_params + .as_ref() + .map_or(def.name.end(), |params| params.end()); + return Edit::insertion(format!("({bases})"), after); + }; + if let Some(last) = arguments.args.last() { + Edit::insertion(format!(", {bases}"), last.end()) + } else if let Some(keyword) = arguments.keywords.first() { + Edit::insertion(format!("{bases}, "), keyword.start()) + } else { + Edit::insertion(bases.to_string(), arguments.start() + TextSize::new(1)) + } +} diff --git a/crates/ty_ide/src/refactor/move_symbol.rs b/crates/ty_ide/src/refactor/move_symbol.rs index c7d1f70c3d..c4b46398dc 100644 --- a/crates/ty_ide/src/refactor/move_symbol.rs +++ b/crates/ty_ide/src/refactor/move_symbol.rs @@ -970,7 +970,7 @@ pub(super) struct Cut { /// The definition with its decorators and the comments directly above it, ending in a line /// break. pub(super) text: String, - /// Where `text` starts in the module. + /// where `text` starts in the module pub(super) start: TextSize, /// Its lines and the blank lines that separated it from the statement after it, or, when it is /// the last statement, from the one before. @@ -1369,8 +1369,8 @@ fn rewrite_references( ); } -/// The range of the module a `from` import names, its leading dots included: from the token -/// after `from` to the end of the module's name, or of its dots when it has none. +/// the range of the module a `from` import names, its leading dots included: from the token +/// after `from` to the end of the module's name, or of its dots when it has none fn from_module_range(parsed: &ParsedModuleRef, from: &ast::StmtImportFrom) -> Option { let mut tokens = parsed.tokens().in_range(from.range()).iter(); let keyword = tokens.next()?; @@ -1461,7 +1461,7 @@ X = os.sep .build() } - /// Every file `moved` changes, as it is afterwards, and its warnings; or why it refused. + /// every file `moved` changes, as it is afterwards, and its warnings; or why it refused pub(in crate::refactor) fn rendered( db: &ty_project::TestDb, moved: Result, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 3eba1692bc..09bd52a7f9 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -2693,8 +2693,8 @@ fn direct_subtypes<'db>( subtypes } -/// The classes `class` names as its bases, as class literals, for a refactoring that asks what -/// a class inherits from above itself. Empty for a type that is no class. +/// the classes `class` names as its bases, as class literals, for a refactoring that asks what +/// a class inherits from above itself. empty for a type that is no class pub fn class_bases<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -2711,8 +2711,8 @@ pub fn class_bases<'db>( .collect() } -/// The names of the members an instance of `class` has, its own and inherited, attributes its -/// methods assign included. Empty for a type that is no class. +/// the names of the members an instance of `class` has, its own and inherited, attributes its +/// methods assign included. empty for a type that is no class pub fn instance_member_names<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -2729,9 +2729,9 @@ pub fn instance_member_names<'db>( .collect() } -/// Whether every value of `receiver` is one of `classes`, or below one of them: an instance of -/// one of them or of a subclass, or one of them or a subclass itself. A member read off such a -/// receiver is looked up from one of those classes down, never from above them. +/// whether every value of `receiver` is one of `classes`, or below one of them: an instance of +/// one of them or of a subclass, or one of them or a subclass itself. a member read off such a +/// receiver is looked up from one of those classes down, never from above them /// /// a gradual receiver may hold any value, so only a subtype of a class's instances, under any /// specialization of it, is within that class diff --git a/crates/ty_server/src/server/api/requests/move_members.rs b/crates/ty_server/src/server/api/requests/move_members.rs index 5d15b77003..cdac5ebe84 100644 --- a/crates/ty_server/src/server/api/requests/move_members.rs +++ b/crates/ty_server/src/server/api/requests/move_members.rs @@ -67,7 +67,8 @@ pub(crate) struct ClassItem { #[serde(rename_all = "camelCase")] pub(crate) struct MemberItem { name: String, - /// `method`, `classmethod`, `staticmethod`, `property` or `attribute` + /// `method`, `classmethod`, `staticmethod`, `property` or `attribute`, or `base` for a base + /// the class statement writes, which a pull up moves into the superclass's bases kind: String, /// how it reads in a list: a method or property as its name and parameters as written, /// `fetch(self, key: str)`, an attribute as its name @@ -135,6 +136,7 @@ impl BackgroundDocumentRequestHandler for PrepareMoveMembersRequestHandler { MemberKind::StaticMethod => "staticmethod", MemberKind::Property => "property", MemberKind::Attribute => "attribute", + MemberKind::Base => "base", } .to_string(), detail: member.detail, diff --git a/crates/ty_server/src/server/api/requests/move_symbol.rs b/crates/ty_server/src/server/api/requests/move_symbol.rs index f2f260ff8a..0ad32788a6 100644 --- a/crates/ty_server/src/server/api/requests/move_symbol.rs +++ b/crates/ty_server/src/server/api/requests/move_symbol.rs @@ -186,7 +186,7 @@ impl BackgroundDocumentRequestHandler for MoveSymbolRequestHandler { } impl MoveSymbolAnswer { - /// The answer to a move: its edits as a workspace edit and its warnings, or why it refused. + /// the answer to a move: its edits as a workspace edit and its warnings, or why it refused pub(super) fn of( db: &ProjectDatabase, snapshot: &DocumentSnapshot, diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index f0170a0db6..2868cdb16f 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -280,6 +280,12 @@ fn a_class_lists_its_members_and_where_they_could_go() -> Result<()> { "detail": "describe(self)", "range": { "start": { "line": 7, "character": 8 }, "end": { "line": 7, "character": 16 } }, }, + { + "name": "Shape", + "kind": "base", + "detail": "Shape", + "range": { "start": { "line": 4, "character": 13 }, "end": { "line": 4, "character": 18 } }, + }, ], "superclasses": [{ "name": "Shape", diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 621224ceba..b1f60ce808 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -765,6 +765,14 @@ moves with its decorators and the comments above it, reindented for the body it lands in: attributes before the methods, methods after them, in the order they were written +a base the class statement names — a mixin, say — is pulled up the same way, out +of the class's bases and into the end of the superclass's, as python's own pull +members up offers it. it is refused unless the superclass and every class of the +project below it still find every name they found in the class they found it in, +python can still order each one's bases, the class still reaches the superclass +without the base, and the base is bound at the top of the module above the +superclass, which is in the class's module + either is refused, with the reason, where the program would no longer mean what it meant: @@ -803,10 +811,14 @@ it once nothing above has the name and answers `{class, members, superclasses, subclasses}`: the class and the classes of the project above it, nearest first, and below it, each `{name, qualifiedName, uri, range}` with `range` its name; and its members, each `{name, kind, detail, range}`, `kind` one of `method`, `classmethod`, `staticmethod`, `property` and `attribute`, +or `base` for a base the class statement names, as it writes it, `detail` how the member reads in a list — a method or property as its name and its parameters as written, on one line, `fetch(self, key: str)`, an attribute as its name; `null` outside a class. `by/pullMembersUp` takes the same position, `members`, the -names to move, and `target`, a superclass's `qualifiedName`; `by/pushMembersDown` +names of the members to move, `bases`, the bases to move, each as listed with kind +`base`, and `target`, a superclass's `qualifiedName`; a request without `bases` +names its bases among `members`, each read by the kind `by/prepareMoveMembers` lists +it with, and a name listed as both is refused. `by/pushMembersDown` takes the position and `members`. both answer as `by/moveSymbol` does, `{edit, warnings}` or `{refused, warnings}` From 2b89e80a514d49cc20ae075e862ad5fb441ee310 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:35:16 +1000 Subject: [PATCH 069/110] say which document requests answer a closed document's textHash from disk, that by/documentUsageCounts always sends abstract and leaves className out for an implementation in no class, which fields of by/documentInheritedMembers and by/endpoints may be absent, and that buff/formatSettings names the project's configuration file and not one the editor's settings merge over it --- docs/basedpython/features/editor.md | 30 +++++++++++++++++------------ docs/basedpython/features/linter.md | 7 ++++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index b1f60ce808..8ae7ec59ad 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -455,8 +455,10 @@ class of its method resolution order declares, as that class's outline has it, attributes a method assigns on `self` among them, from the first class to declare the name, and none the class declares itself. `object`'s are there, last. each member is a document symbol of its superclass's file, with its -`name`, `kind`, `detail`, `range` and `selectionRange`, the `uri` of that file, -and the superclass as `containerName` and its module as `containerModule`: +`name`, `kind`, `detail` (absent where the symbol has none), `range` and +`selectionRange`, the `uri` of that file, and the superclass as `containerName` +and its module as `containerModule`, absent for a superclass in a file no module +is imported from: ```json [ @@ -1388,18 +1390,19 @@ its name, and the counts named in the request's `counts`: | `implementations` | of a class or a method, what `textDocument/implementation` finds from the name, less itself | so the count is the number of places the click on it goes to, when a client -sends that request from the name. a method with implementations is marked -`abstract` when it is an `@abstractmethod` or a protocol member with no body, -so that what overrides it implements it. a count not named is not worked out +sends that request from the name. every declaration has `abstract`, `true` for a +method with implementations that is an `@abstractmethod` or a protocol member +with no body, so that what overrides it implements it, and `false` otherwise. a count not named is not worked out with `implementations` comes `implementors`, the places it counts, in the order `textDocument/implementation` answers them, so that an editor can say "is overridden in Circle, Square" without asking. each is a subclass of the class, or a member overriding the method, with the `uri`, `range` and `selectionRange` it is declared at and the class it is in: `className`, the -subclass itself or the class whose member overrides, and `qualifiedName`, that -class's dotted name, `shapes.base.Circle`, absent for a class local to a -function +subclass itself or the class whose member overrides, absent for an +implementation in no class, and `qualifiedName`, that class's dotted name, +`shapes.base.Circle`, absent for a class local to a function or in a file no +module is imported from ```json { @@ -1548,7 +1551,8 @@ path, every prefix the route is mounted under included, with its parameters as the route writes them, `{item_id}` or ``; a `re_path()` pattern is joined to its prefixes with each part's `^` and `$` anchors dropped. `handler` is the function, or for a django view built by `as_view()` the class, with its -qualified name, its whole declaration as `range` and its name as +qualified name as `name` (absent where it has none, as `application`'s may +be), its whole declaration as `range` and its name as `selectionRange`; `declaration` is where the route is declared, the decorator without its `@`, the `add_api_route(...)` or the `path(...)`; `application` is the FastAPI application that serves it, or the urlconf module django starts at, and @@ -1834,9 +1838,11 @@ counted as one `\n`, written as sixteen lowercase hex digits. for a notebook cel that is the cell's own text. the request is then answered about that text and no other. when the server already holds it — the open buffer, or for `by/syntaxOutline`, `by/injections`, `by/superMembers`, -`by/documentSuperMembers`, `by/transpile`, `textDocument/semanticTokens/full` and -`textDocument/documentSymbol` the file on disk of a document the client has not -opened — it answers at once; otherwise the +`by/documentSuperMembers`, `by/documentInheritedMembers`, +`by/documentUsageCounts`, `by/declaration`, `by/prepareMoveSymbol`, +`by/moveSymbol`, `by/testSubjects`, `by/subjectTests`, `by/transpile`, +`textDocument/semanticTokens/full` and `textDocument/documentSymbol` the file on +disk of a document the client has not opened — it answers at once; otherwise the request waits for the `didOpen`, `didChange` or file write that brings the text, and is answered with `ServerCancelled` if nothing has after ten seconds. so a client need not know whether its `didOpen` has gone out before it asks, and an diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 6638cefc74..c845735e26 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -91,9 +91,10 @@ override in it, spelled as the configuration spells them: `indentStyle` is `space` or `tab`, `quoteStyle` is `double`, `single` or `preserve`, and `lineEnding` is `auto`, `lf`, `cr-lf` or `native`. -`configurationFile` is the file the settings were read from, and `null` when no -file configures the document. the answer is `null` for a document the editor has -not opened +`configurationFile` is the project's configuration file the settings were read +from, and `null` when none configures the document; a configuration file the +editor's own settings name is merged over it without being named here. the +answer is `null` for a document the editor has not opened ## what the linter does not check From 314cc49414181bf132ea91690999c041b2b8697c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:53:52 +1000 Subject: [PATCH 070/110] profile a program with by run --profile PATH, which writes the pstats snapshot cProfile writes with every function of a transpiled module named by its .by file and the .by line it starts at, and one of a carried-over .py by the project's file rather than the staged copy, so a profile viewer such as PyCharm's leads from a function to the file it was written in --- crates/ty/docs/cli.md | 2 + crates/ty/src/args.rs | 11 ++ crates/ty/src/by_commands.rs | 129 ++++++++++++++++++-- crates/ty/src/lib.rs | 2 + crates/ty/tests/by_e2e.rs | 194 ++++++++++++++++++++++++++++++ docs/basedpython/cli-reference.md | 29 +++++ 6 files changed, 357 insertions(+), 10 deletions(-) diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 209cac3f98..4630f1b913 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -264,6 +264,8 @@ by run [OPTIONS] [MODULE] [ARGS]...

The interpreter is chosen exactly as it is without this, and the program is started as LAUNCHER <interpreter> <runner> <module> <args>: the launcher is handed the interpreter by run chose rather than having to name one, and is run once, for the program — never for the version probe, which is made against the interpreter itself.

--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--profile path

Profile the program with cProfile and write the snapshot to PATH, as the pstats file cProfile writes (.pstat).

+

Every function of a transpiled module is named by its .by file and the .by line it starts at, and one of a .py the project carries over by the file in the project rather than the staged copy that ran, so a profile viewer's way to a function's source leads to the file it was written in. The profile starts at the program's first line and ends after its last atexit handler.

--python, --venv path

The interpreter to run on, or the environment holding it. A bare name that is neither, like python3.14, is found on PATH.

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 6501251569..236cf479e9 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -132,6 +132,17 @@ pub(crate) enum Command { /// the interpreter the program runs on. #[arg(long, value_name = "PATH", conflicts_with = "compiled")] coverage: Option, + /// profile the program with cProfile and write the snapshot to PATH, as + /// the `pstats` file `cProfile` writes (`.pstat`) + /// + /// every function of a transpiled module is named by its `.by` file and + /// the `.by` line it starts at, and one of a `.py` the project carries over + /// by the file in the project rather than the staged copy that ran, so a + /// profile viewer's way to a function's source leads to the file it was + /// written in. The profile starts at the program's first line and ends + /// after its last `atexit` handler. + #[arg(long, value_name = "PATH", conflicts_with = "coverage")] + profile: Option, }, /// Start a new project. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index d92c44e2eb..2a0be58d9a 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -144,6 +144,8 @@ pub(crate) struct RunFlags<'a> { pub(crate) launcher: Option<&'a Path>, /// `--coverage`: where to write an LCOV report of what ran. pub(crate) coverage: Option<&'a Path>, + /// `--profile`: where to write a `pstats` snapshot of where the time went + pub(crate) profile: Option<&'a Path>, } #[allow(clippy::exit, clippy::print_stderr)] @@ -160,6 +162,7 @@ pub(crate) fn cmd_run( python: python_flag, launcher, coverage, + profile, } = flags; let cwd = std::env::current_dir().context("failed to get current directory")?; // one resolution of the project, for the environment and the target version @@ -398,6 +401,29 @@ pub(crate) fn cmd_run( } None => None, }; + // the runner writes the snapshot itself, every function already named by its + // project file: the program can run somewhere else, so the path is settled here + let snapshot = match profile { + Some(snapshot) => { + let snapshot = cwd.join(snapshot); + if let Some(parent) = snapshot.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("could not create {}", parent.display()))?; + } + // a snapshot left from an earlier run would read as this run's when + // this one leaves none + match fs::remove_file(&snapshot) { + Err(error) if error.kind() != io::ErrorKind::NotFound => { + return Err(error) + .with_context(|| format!("could not replace {}", snapshot.display())); + } + _ => {} + } + command.arg("--profile").arg(&snapshot); + Some(snapshot) + } + None => None, + }; let status = command .arg(&module) .args(args) @@ -436,6 +462,15 @@ pub(crate) fn cmd_run( ); } } + if let Some(snapshot) = &snapshot + && !snapshot.is_file() + { + eprintln!( + "note: no profile was written: the program ended without returning to `by run`, \ + as one that calls `os._exit`, replaces itself with `os.exec*` or is killed by a \ + signal does" + ); + } drop(measured); // drop the temp dir explicitly: `process::exit` skips destructors, so // exiting while it's still in scope would leak the directory @@ -2307,6 +2342,7 @@ import importlib.machinery import importlib.util import json import linecache +import marshal import os import sys import traceback @@ -2904,8 +2940,9 @@ def _entry_spec(module): return module, spec -def _run_as_main(module): - """Run `module` as `__main__`, with `.by` paths for `__file__` and `argv[0]`. +def _run_as_main(module, profiler=None): + """Run `module` as `__main__`, with `.by` paths for `__file__` and `argv[0]`, + measured by `profiler` from its first line when there is one. `runpy.run_module` fetches the entry module's code straight from its loader rather than executing it through one, so the loader wrapper that fixes every @@ -2931,6 +2968,10 @@ def _run_as_main(module): # leaving this unset broke every spawned child main_module.__spec__ = spec sys.modules["__main__"] = main_module + if profiler is not None: + # here rather than before the module was found, so the profile is the + # program's and not the runner's search for it + profiler.enable() exec(code, main_module.__dict__) @@ -3065,24 +3106,92 @@ def _write_coverage(measurement, report): json.dump({"files": files}, handle) +def _start_profile(snapshot): + """A profiler for the program, which writes what it measured to `snapshot` once + the program has ended: after its other threads and its own `atexit` handlers, + which python runs before the ones registered ahead of them.""" + import cProfile + + profiler = cProfile.Profile() + atexit.register(_write_profile, profiler, snapshot) + return profiler + + +def _summed(left, right): + """Two measurements of one function added up, as `pstats` adds two profiles: the + call counts and times, and for a function's own row the callers after them, each + caller's added up the same way.""" + counts = tuple(a + b for a, b in zip(left[:4], right[:4])) + if len(left) == 4: + return counts + callers = dict(left[4]) + for caller, timing in right[4].items(): + callers[caller] = _summed(callers[caller], timing) if caller in callers else timing + return (*counts, callers) + + +def _write_profile(profiler, snapshot): + """What `profiler` measured, as the `pstats` file `cProfile` writes, with every + function of a staged module named by the project file and line it was written at. + + A function is the file, first line and name of its code object. One `_by_code` + compiled is the `.by` already; one another loader compiled — a test module's own + top level, which pytest compiles — is the staged `.py`, at the generated python's + lines, and moves to its `.by` here, as a staged copy of a hand-written `.py` moves + to the project file. The two can then name one function, and are added up as one. + A line the transpiler wrote on its own has no `.by` line, and is line 0. + """ + profiler.disable() + profiler.create_stats() + located = {} + + def project_function(function): + filename, line, name = function + if function not in located: + real = os.path.realpath(filename) + if real in _BY_MAP or real in _CARRIED: + path, line = project_location(filename, line) + located[function] = (path, line or 0, name) + else: + located[function] = function + return located[function] + + stats = {} + for function, (*counts, callers) in profiler.stats.items(): + by_caller = {} + for caller, timing in callers.items(): + caller = project_function(caller) + by_caller[caller] = ( + _summed(by_caller[caller], timing) if caller in by_caller else timing + ) + row = (*counts, by_caller) + key = project_function(function) + stats[key] = _summed(stats[key], row) if key in stats else row + with open(snapshot, "wb") as handle: + marshal.dump(stats, handle) + + def main(): # the plugins `by run` puts in the tree reach this module by name; it is running # as `__main__`, which the program's own module takes over below sys.modules["_by_runner"] = sys.modules[__name__] sys.excepthook = _excepthook sys.meta_path.insert(0, _ByFinder()) - # `by run --coverage` names where to write what ran in front of the module, - # where no module name can be - report = None - if sys.argv[1] == "--coverage": - report = sys.argv[2] + # `by run --coverage` and `--profile` name where to write what was measured in + # front of the module, where no module name can be + written = {} + while sys.argv[1] in ("--coverage", "--profile"): + written[sys.argv[1]] = sys.argv[2] del sys.argv[1:3] module = sys.argv[1] sys.argv = sys.argv[1:] - if report is not None: - _start_coverage(report) + if "--coverage" in written: + _start_coverage(written["--coverage"]) + profiler = None + if "--profile" in written: + profiler = _start_profile(written["--profile"]) try: - _run_as_main(module) + _run_as_main(module, profiler) except SystemExit: raise except BaseException: diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index 1f0317d562..74b7e320d0 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -127,6 +127,7 @@ fn run_command(command: Command) -> anyhow::Result { compiled, in_build, coverage, + profile, } => by_commands::cmd_run( module.as_deref(), &args, @@ -138,6 +139,7 @@ fn run_command(command: Command) -> anyhow::Result { python: python.as_deref(), launcher: launcher.as_deref(), coverage: coverage.as_deref(), + profile: profile.as_deref(), }, ), Command::Init { diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 91995889cd..0dbb19f7a5 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1611,6 +1611,200 @@ fn run_coverage_maps_a_module_a_test_runner_compiled() { ); } +/// one function of a `pstats` snapshot: where it is, how often it was called (the +/// calls that were not recursive first), and where it was called from +#[derive(Debug)] +struct Profiled { + file: String, + line: u32, + name: String, + primitive_calls: u64, + calls: u64, + callers: Vec, +} + +/// a function as a `pstats` snapshot names it: its file, first line and name +type ProfiledFunction = (String, u32, String); + +/// the functions of the `pstats` snapshot at `snapshot`, as python's `pstats` reads +/// them +fn profiled_functions(python: &str, snapshot: &Path) -> Vec { + let output = Command::new(python) + .args([ + "-c", + "import json, pstats, sys\n\ + stats = pstats.Stats(sys.argv[1]).stats\n\ + print(json.dumps([[function, cc, nc, list(callers)] \ + for function, (cc, nc, _, _, callers) in stats.items()]))", + ]) + .arg(snapshot) + .output() + .expect("python runs"); + assert!( + output.status.success(), + "pstats could not read {}:\n{}", + snapshot.display(), + String::from_utf8_lossy(&output.stderr) + ); + let rows: Vec<(ProfiledFunction, u64, u64, Vec)> = + serde_json::from_slice(&output.stdout).expect("json"); + rows.into_iter() + .map( + |((file, line, name), primitive_calls, calls, callers)| Profiled { + file, + line, + name, + primitive_calls, + calls, + callers, + }, + ) + .collect() +} + +/// a profile viewer finds a function's source by the file and line the snapshot +/// names it by, so a function of a transpiled module is named by its `.by` and the +/// line it starts at there, and one of a carried-over `.py` by the project's file +/// rather than the staged copy that ran, which is gone once the run ends +#[test] +fn run_profile_names_functions_by_their_project_files() { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("pkg"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.py"), "").unwrap(); + fs::write( + package.join("calc.by"), + "def fib(n: int) -> int:\n if n < 2:\n return n\n return fib(n - 1) + fib(n - 2)\n", + ) + .unwrap(); + fs::write( + package.join("plain.py"), + "def helper() -> int:\n return 3\n", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "from pkg.calc import fib\nfrom pkg.plain import helper\n\n\nclass Worker:\n def work(self, n: int) -> int:\n return fib(n) + helper()\n\n\nprint(Worker().work(10))\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + let file = |relative: &str| project.join(relative).display().to_string(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args([ + "run", + "--python", + &python, + "--profile", + "out/run.pstat", + "main", + ]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "by run --profile failed:\n{stderr}" + ); + // python writes its output in text mode, which ends a line with `\r\n` on windows + assert_eq!( + String::from_utf8_lossy(&output.stdout).replace("\r\n", "\n"), + "58\n", + "{stderr}" + ); + let functions = profiled_functions(&python, &dir.path().join("out/run.pstat")); + let find = |file: &str, name: &str| { + functions + .iter() + .find(|function| function.file == file && function.name == name) + .unwrap_or_else(|| panic!("no {name} in {file}: {functions:#?}")) + }; + + let fib = find(&file("pkg/calc.by"), "fib"); + assert_eq!(fib.line, 1); + assert_eq!((fib.primitive_calls, fib.calls), (1, 177)); + let work = find(&file("main.by"), "work"); + assert_eq!(work.line, 6); + assert!( + fib.callers + .contains(&(file("main.by"), 6, "work".to_owned())) + && fib + .callers + .contains(&(file("pkg/calc.by"), 1, "fib".to_owned())), + "fib is called by work and by itself: {:?}", + fib.callers + ); + let helper = find(&file("pkg/plain.py"), "helper"); + assert_eq!(helper.line, 1); + assert_eq!( + helper.callers, + [(file("main.by"), 6, "work".to_owned())], + "the caller is named the same way" + ); + let staged: Vec<&String> = functions + .iter() + .flat_map(|function| { + std::iter::once(&function.file).chain(function.callers.iter().map(|(file, ..)| file)) + }) + .filter(|named| { + ["main.py", "calc.py", "plain.py"] + .iter() + .any(|copy| named.ends_with(copy)) + && **named != file("pkg/plain.py") + }) + .collect(); + assert!(staged.is_empty(), "a staged copy is named: {staged:?}"); +} + +/// a test runner compiles a test module itself, so the module's own top level runs +/// as the staged `.py`, at the generated python's lines, and is still named by the +/// `.by` +#[test] +fn run_profile_maps_a_module_a_test_runner_compiled() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + let project = dir.path().canonicalize().unwrap(); + let snapshot = project.join("tests.pstat"); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("run") + .args(["--python", &python, "--in-build", "--profile"]) + .arg(&snapshot) + .args(["pytest", "-q", "-p", "no:cacheprovider"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + // the project's tests fail on purpose; the profile is written all the same + assert_eq!( + output.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let functions = profiled_functions(&python, &snapshot); + let test_module = project.join("tests/test_calc.by").display().to_string(); + for name in ["", "test_sum"] { + assert!( + functions + .iter() + .any(|function| function.file == test_module && function.name == name), + "no {name} in {test_module}: {functions:#?}" + ); + } + assert!( + !functions + .iter() + .any(|function| function.file.ends_with("test_calc.py")), + "the staged test module is named: {functions:#?}" + ); +} + /// A project of `.by` tests for pytest to run, with a module they test, checked /// against the environment the tests run in: the `conftest` imports pytest, which /// `--python` does not configure. `None`, having said why, when `$PYTHON` (or diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index a7bbbe343c..4b450a6d8e 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -125,6 +125,35 @@ leaves without returning to `by run` — `os._exit` — leaves no report, and `--compiled` cannot be combined with it, since a native module runs no python lines to measure +`--profile PATH` profiles the program with cProfile and writes the snapshot to +`PATH`, in the `pstats` format `cProfile` writes — the `.pstat` file python's +`pstats` module and PyCharm's profiler viewer open. a profiler names each +function by the file and first line of its code, so the snapshot is mapped back +before it is written, as a coverage report is: a function of a transpiled module +is named by its `.by` file and the `.by` line it starts at, and one of a `.py` +the build carries over by the file in the project rather than the staged copy +that ran. a viewer's way from a function to its source then leads to the file it +was written in. that holds for a module a test runner compiled itself too, and +two functions that turn out to be one — the same name at the same `.by` line — +are added up as one, as `pstats` adds two profiles. a function the transpiler +wrote on its own, such as a helper pasted into a module, has no `.by` line, and +is named by its module's `.by` at line 0 + +```console +$ by run --profile run.pstat main +$ python -m pstats run.pstat +``` + +the profile starts at the program's first line and ends after its last `atexit` +handler, so it holds the program and not `by run`'s search for it. what `by run` +does to load a module — finding it, and compiling it under its `.by` lines — is +time the program spends, and is named by the runner's file in the staged tree. +cProfile measures every thread on python 3.12 and later, and only the thread the +program starts on before that, as `python -m cProfile` does. a program that leaves +without returning to `by run` — on `os._exit`, an `os.exec*` or a signal — leaves no snapshot, and `--coverage` +cannot be combined with it, since a profile taken under coverage.py's tracer +measures the tracer + the emitted code targets that interpreter's version by default. an explicit `--min-version` wins, but must not exceed the interpreter — `by run` refuses rather than emit code the interpreter cannot parse. an interpreter *older* than From ce74558c2debcf9c806e5ea85b9808bf84604497 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:01:31 +1000 Subject: [PATCH 071/110] design by repl, a basedpython session checked and transpiled as one module whose source is its inputs in order, run in one python process, with by/inputStatus saying whether an input is complete and --session-file giving an editor's console the module to complete against --- docs/basedpython/features/index.md | 2 + docs/basedpython/features/repl.md | 131 +++++++++++++++++++++++++++++ zensical.toml | 1 + 3 files changed, 134 insertions(+) create mode 100644 docs/basedpython/features/repl.md diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 7dab9b8871..e2371a90f5 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -49,6 +49,8 @@ features that apply to a project rather than to a file into scope - [the project server](project-server.md) — `by check` answered out of the language server's warm state +- [the repl](repl.md) — an interactive session in basedpython, checked as one + module
diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md new file mode 100644 index 0000000000..9bd50803d6 --- /dev/null +++ b/docs/basedpython/features/repl.md @@ -0,0 +1,131 @@ +# the repl + +> **status: design.** what this page describes is being built; `by repl` and `by/inputStatus` +> land in the commits that follow it + +`by repl` is an interactive session in which you type basedpython. each input is checked and +transpiled with everything the session has defined so far, and runs in one python process +that lives as long as the session + +```console +$ by repl +>>> let rate = 0.2 +>>> def taxed(amount: float) -> float: +... return amount * (1 + rate) +... +>>> taxed(10) +12.0 +>>> rate = 0.3 + --> :1:1 +error[invalid-assignment]: Reassignment of `Final` symbol `rate` is not allowed +``` + +it starts where `by run` would: in the project around the working directory, on the +project's interpreter (`--python` names another), with the project's modules staged as `by run` stages them. so `import app.pricing` imports the project's `.by` module, transpiled, and an +error raised inside it is reported at its `.by` line. like `by run`, it does not start while +the project has check errors + +## a session is a module + +the inputs that ran, in order, are the source of one module, and each new input is that +module's continuation. it is checked as such: a name defined two inputs ago is defined, an +import is in scope, a `let` stays final, an extension declared earlier applies, and the +checker's answer for the input is the one it would give for the same lines at the end of a +file. what this means in practice: + +- an input with check errors does not run, and is not part of the session: it is as if it had + not been typed. its errors are reported against the input, as `` and the + input's own lines +- warnings are not reported. an import that nothing uses yet is not unused, it is unused *so + far*, and an expression whose value is discarded is the value the repl prints +- an input that raised is part of the session: the statements before the one that raised + have run, and what they bound is bound, as in python's repl +- a name declared with a type cannot be declared again with another, since one module cannot + do that. a name assigned without a declaration can be assigned anything + +what runs is the python written for the input: the module made of the session and the input +is transpiled as a whole, and the statements transpiled from the input's lines run, after +any runtime helper they need that has not run yet. the earlier inputs' statements ran when +they were entered. an input that would change the python an earlier input was transpiled to — +which has already run, and cannot run again differently — is refused, and says so + +two things differ from `by run`, because a session is never run as a whole. an `import` runs +when it is entered rather than lazily, so a module that cannot be imported says so then, and a +`def main` is a function like any other: nothing calls it + +## what is printed + +an input that ends in an expression prints its value, through `sys.displayhook`, as python's +repl prints it. an exception prints a traceback whose frames name the +input they ran in, at the input's line, with the line quoted, and the project's modules at +their `.by` lines + +```console +>>> def ratio(a: int, b: int) -> float: +... return a / b +... +>>> ratio(1, 0) +Traceback (most recent call last): + File "", line 1, in + ratio(1, 0) + File "", line 2, in ratio + return a / b +ZeroDivisionError: division by zero +``` + +## input + +on a terminal, prompts are `>>>` and `...`, and lines are read the way python's repl reads them: +a line that completes a statement runs it, a compound statement runs at the blank line after +it, and a line that cannot be the start of anything runs at once, to report what is wrong with +it. whether an input is complete is a question about basedpython's grammar, so `by` answers it +— the same answer the [language server](#in-an-editor) gives + +the end of input (ctrl-d) ends the session, and its exit status is the program's: 0, or what +`sys.exit` was given. ctrl-c abandons the input being typed, or interrupts the one running with +a `KeyboardInterrupt`, which is printed, and the session carries on + +an input can also be sent whole, framed as a terminal frames a paste: `ESC [200~` before it and +`ESC [201~` after it. the text between the two is one input whatever it holds, blank lines +inside a function body included, and runs when the closing marker arrives. this is how an +editor sends one + +while an input runs, the lines that arrive are the program's: `input()` reads them, as it reads +python's + +## in an editor + +an editor's console runs `by repl` as a process and talks to it through its standard streams: + +- **sending.** each input is written to `by repl`'s standard input framed, as above, followed by + a newline. while the program reads `input()`, a line is written as it is + +- **output.** the program's output, the printed values and the tracebacks are the process's + standard output and standard error, as they are python's. prompts are written only when + standard input is a terminal, so a console that reads a pipe shows none + +- **interrupting.** `by repl` itself ignores `SIGINT`, so a console interrupts the input that is + running by sending `SIGINT` to the process tree, as a terminal sends it to the process group + +- **enter.** whether enter runs the input or starts a new line is the language server's + `by/inputStatus` request, given the input's text: + + ```json + { "text": "def taxed(amount: float) -> float:\n" } + ``` + + ```json + { "status": "incomplete" } + ``` + + `complete` — it can run; `incomplete` — it needs more lines: a compound statement whose body + could still go on, an open bracket, an unterminated triple-quoted string, a line ending in + `\`; `invalid` — no more lines can make it valid, and running it reports why. blank input, + and input of comments only, is `complete` + +- **completion.** `--session-file PATH` keeps `PATH` holding the session's source — the inputs + that ran, in order, each ending in a newline — from the moment the session starts. completion + in the console is the language server's ordinary completion, asked of a document whose text + is that file's followed by the input being typed, at the caret's offset past the file's + length: the same module the session checks the input as, so what is offered is what the + session has defined diff --git a/zensical.toml b/zensical.toml index d852eaee25..d5ce10a1d7 100644 --- a/zensical.toml +++ b/zensical.toml @@ -95,6 +95,7 @@ features = [ { "linting" = "features/linter.md" }, { "unused imports" = "features/unused-imports.md" }, { "the project server" = "features/project-server.md" }, + { "the repl" = "features/repl.md" }, ] }, { "standard library" = [ { "typeshed improvements" = "features/typeshed.md" }, From ff2f65ee7dd181c8dc95015d8e4a6784cb0f70e3 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:08:42 +1000 Subject: [PATCH 072/110] answer by/inputStatus with whether the text typed into a by repl console is complete, incomplete or invalid, as python's repl decides it for python and in basedpython's grammar: a compound statement goes on until a blank line, an open bracket, a triple-quoted string or a continued line until it closes, and an error no further line can fix runs at once --- Cargo.lock | 3 + crates/by_stage/Cargo.toml | 3 + crates/by_stage/src/lib.rs | 3 + crates/by_stage/src/session.rs | 326 ++++++++++++++++++ crates/ruff_python_parser/src/lib.rs | 18 + crates/ruff_python_parser/src/parser/mod.rs | 22 ++ crates/ty_server/src/server/api.rs | 4 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/input_status.rs | 58 ++++ crates/ty_server/tests/e2e/input_status.rs | 59 ++++ crates/ty_server/tests/e2e/main.rs | 1 + docs/basedpython/features/editor.md | 10 + docs/basedpython/features/repl.md | 4 +- 13 files changed, 511 insertions(+), 2 deletions(-) create mode 100644 crates/by_stage/src/session.rs create mode 100644 crates/ty_server/src/server/api/requests/input_status.rs create mode 100644 crates/ty_server/tests/e2e/input_status.rs diff --git a/Cargo.lock b/Cargo.lock index 7f044f1ac7..705adbba5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,6 +441,9 @@ dependencies = [ "dunce", "rayon", "ruff_db", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "salsa", "serde", "serde_json", diff --git a/crates/by_stage/Cargo.toml b/crates/by_stage/Cargo.toml index 7074d09240..26b5510b53 100644 --- a/crates/by_stage/Cargo.toml +++ b/crates/by_stage/Cargo.toml @@ -16,6 +16,9 @@ doctest = false [dependencies] by_transforms = { workspace = true } ruff_db = { workspace = true, features = ["os"] } +ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_text_size = { workspace = true } ty_module_resolver = { workspace = true } ty_project = { workspace = true } ty_python_semantic = { workspace = true } diff --git a/crates/by_stage/src/lib.rs b/crates/by_stage/src/lib.rs index 8a86c944b5..72b3134851 100644 --- a/crates/by_stage/src/lib.rs +++ b/crates/by_stage/src/lib.rs @@ -49,6 +49,8 @@ //! which source, for an editor to ask. //! - [`run_module`] — which file `by run ` executes, and the module a //! file runs as. +//! - [`session`] — a `by repl` session, and whether what has been typed into it +//! is a whole input yet pub mod emit; pub mod layout; @@ -57,6 +59,7 @@ pub mod record; pub mod restage; pub mod run_module; pub mod runtime; +pub mod session; pub mod sourcemap; pub mod staging; pub mod verbatim; diff --git a/crates/by_stage/src/session.rs b/crates/by_stage/src/session.rs new file mode 100644 index 0000000000..8f45e255bb --- /dev/null +++ b/crates/by_stage/src/session.rs @@ -0,0 +1,326 @@ +//! a `by repl` session: basedpython typed an input at a time, run in one python process +//! +//! a session is a module whose source is the inputs that ran, in order, and each new input is +//! checked and transpiled as that module's continuation (see `docs/basedpython/features/repl.md`). +//! What is here is the part of that which is a question about source text, so the command and +//! the language server give the same answers: whether what has been typed is a whole input yet. + +use ruff_python_ast::{PySourceType, Stmt}; +use ruff_python_parser::{ + InterpolatedStringErrorType, LexicalErrorType, ParseError, ParseErrorType, + parse_unchecked_source, +}; + +/// whether the text typed so far is an input that can run +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum InputStatus { + /// it can run as it is + Complete, + /// it needs more lines: a compound statement whose body could still go on, an open + /// bracket, an unterminated triple-quoted string, a line continued with `\` + Incomplete, + /// no line that follows can make it valid. running it reports why + Invalid, +} + +/// whether `text`, the lines typed so far, is a whole input, as python's repl decides it for +/// python: a statement runs once a line completes it, a compound statement once a blank line +/// follows it, since until then its body could go on, and input that no further line can make +/// valid runs at once, to report what is wrong with it +/// +/// blank input, and input of comments only, is complete: there is nothing to wait for +pub fn input_status(text: &str) -> InputStatus { + let parsed = parse_unchecked_source(text, PySourceType::BasedPython); + // the parser goes on past an error and can report others that follow from it, so the + // first one is the one that says what went wrong + let Some(first) = parsed + .errors() + .iter() + .min_by_key(|error| error.location.start()) + else { + return match parsed.suite().last() { + Some(last) if has_suite(last) && !ends_with_blank_line(text) => InputStatus::Incomplete, + _ => InputStatus::Complete, + }; + }; + if ran_out_inside_a_token(text, first) { + return InputStatus::Incomplete; + } + // the parser met the end of the input where the grammar needs more: a block's body, a + // decorator's definition, a `try`'s handler. a blank line says the input has ended, as + // it does after a compound statement + if parsed.ran_out_of_input() && !ends_with_blank_line(text) { + InputStatus::Incomplete + } else { + InputStatus::Invalid + } +} + +/// whether the input ended while the tokenizer was inside something only a later line can +/// close: an open bracket or a line continued with `\`, which it reports as running into the +/// end of the input, or a triple-quoted string, which it reports where the string starts +fn ran_out_inside_a_token(text: &str, error: &ParseError) -> bool { + match &error.error { + ParseErrorType::Lexical( + LexicalErrorType::Eof + | LexicalErrorType::FStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ) + | LexicalErrorType::TStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ), + ) => true, + ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError) => { + opens_triple_quoted_string(&text[usize::from(error.location.start())..]) + } + _ => false, + } +} + +/// whether `text` begins with a string literal whose quotes are tripled, after any prefix +fn opens_triple_quoted_string(text: &str) -> bool { + let quotes = text.trim_start_matches(|c: char| c.is_ascii_alphabetic()); + quotes.starts_with("\"\"\"") || quotes.starts_with("'''") +} + +/// whether `stmt` ends in an indented suite, which a line typed after it could still extend. +/// a bodiless `def`, basedpython's overload declaration, is one line +fn has_suite(stmt: &Stmt) -> bool { + match stmt { + Stmt::FunctionDef(def) => !def.body.is_empty(), + Stmt::ClassDef(class) => !class.body.is_empty(), + Stmt::For(_) + | Stmt::While(_) + | Stmt::If(_) + | Stmt::With(_) + | Stmt::Match(_) + | Stmt::Try(_) => true, + // `let [first, *rest] := words else:` and its block + Stmt::Let(binding) => !binding.orelse.is_empty(), + // `type Element[T] = match T:` and its `case` blocks + Stmt::TypeAlias(alias) => !alias.cases.is_empty(), + _ => false, + } +} + +/// whether the last line of `text` is blank: a line ending the input that holds nothing but +/// whitespace, which is what ends a compound statement typed a line at a time +fn ends_with_blank_line(text: &str) -> bool { + let Some(before_last_newline) = text.strip_suffix('\n') else { + return false; + }; + let before_last_newline = before_last_newline + .strip_suffix('\r') + .unwrap_or(before_last_newline); + let last_line = before_last_newline + .rsplit_once('\n') + .map_or(before_last_newline, |(_, line)| line); + last_line.trim().is_empty() && before_last_newline.contains('\n') +} + +#[cfg(test)] +mod tests { + use super::{InputStatus, input_status}; + + #[track_caller] + fn assert_status(text: &str, expected: InputStatus) { + assert_eq!(input_status(text), expected, "{text:?}"); + } + + #[test] + fn a_simple_statement_is_complete_once_typed() { + assert_status("x = 1\n", InputStatus::Complete); + assert_status("x = 1", InputStatus::Complete); + assert_status("print(x)\n", InputStatus::Complete); + assert_status("let rate = 0.2\n", InputStatus::Complete); + assert_status("x ?? 1\n", InputStatus::Complete); + } + + #[test] + fn nothing_typed_is_complete() { + assert_status("", InputStatus::Complete); + assert_status("\n", InputStatus::Complete); + assert_status(" \n", InputStatus::Complete); + assert_status("# a comment\n", InputStatus::Complete); + } + + /// as in python's repl, a compound statement runs at the blank line after it, since until + /// then another line of its body, or an `else`, could follow + #[test] + fn a_compound_statement_waits_for_a_blank_line() { + assert_status("if x:\n", InputStatus::Incomplete); + assert_status("if x:\n y = 1\n", InputStatus::Incomplete); + assert_status("if x:\n y = 1\n\n", InputStatus::Complete); + assert_status("if x:\n y = 1\n \n", InputStatus::Complete); + assert_status("def f():\n return 1\n", InputStatus::Incomplete); + assert_status("def f():\n return 1\n\n", InputStatus::Complete); + assert_status("def f(): return 1\n", InputStatus::Incomplete); + assert_status("class C:\n x: int\n", InputStatus::Incomplete); + assert_status("for i in xs:\n pass\n", InputStatus::Incomplete); + assert_status("while x:\n pass\n", InputStatus::Incomplete); + assert_status("with f() as g:\n pass\n", InputStatus::Incomplete); + assert_status( + "try:\n pass\nexcept E:\n pass\n", + InputStatus::Incomplete, + ); + assert_status( + "match x:\n case 1:\n pass\n", + InputStatus::Incomplete, + ); + assert_status("@decorator\n", InputStatus::Incomplete); + assert_status("@decorator\ndef f():\n pass\n", InputStatus::Incomplete); + } + + /// a header waits for the block that follows it, whatever the statement: a line of its own + /// could still start that block + #[test] + fn every_compound_header_waits_for_its_body() { + for header in [ + "if x:", + "while x:", + "for i in xs:", + "async for i in xs:", + "with f() as g:", + "async with f() as g:", + "def f():", + "def f() -> int:", + "async def f():", + "class C:", + "class C(Base):", + "try:", + "match x:", + "if let x := y:", + "let [a, b] := xs else:", + "type Element[T] = match T:", + ] { + assert_status(&format!("{header}\n"), InputStatus::Incomplete); + assert_status(header, InputStatus::Incomplete); + } + } + + #[test] + fn a_continuation_clause_waits_for_its_body() { + assert_status("if x:\n pass\nelif y:\n", InputStatus::Incomplete); + assert_status("if x:\n pass\nelse:\n", InputStatus::Incomplete); + assert_status("for i in xs:\n pass\nelse:\n", InputStatus::Incomplete); + assert_status("while x:\n pass\nelse:\n", InputStatus::Incomplete); + assert_status("try:\n pass\nexcept E:\n", InputStatus::Incomplete); + assert_status("try:\n pass\nexcept* E:\n", InputStatus::Incomplete); + assert_status( + "try:\n pass\nexcept E:\n pass\nelse:\n", + InputStatus::Incomplete, + ); + assert_status("try:\n pass\nfinally:\n", InputStatus::Incomplete); + assert_status("match x:\n case 1:\n", InputStatus::Incomplete); + } + + /// a `try` is not over until a handler or `finally` follows its block, so the next line + /// could still be one + #[test] + fn a_try_block_waits_for_its_handler() { + assert_status("try:\n pass\n", InputStatus::Incomplete); + assert_status("try:\n x = 1\n y = 2\n", InputStatus::Incomplete); + assert_status("try:\n pass\n\n", InputStatus::Invalid); + } + + #[test] + fn a_decorator_waits_for_what_it_decorates() { + assert_status("@decorator\n", InputStatus::Incomplete); + assert_status("@decorator\n@other\n", InputStatus::Incomplete); + assert_status("class C:\n @property\n", InputStatus::Incomplete); + assert_status( + "def f():\n class C:\n @property\n", + InputStatus::Incomplete, + ); + assert_status("@decorator\n\n", InputStatus::Invalid); + } + + /// as in python's repl, a blank line ends the input, so a header with no body after it + /// cannot be finished any more + #[test] + fn a_blank_line_after_a_header_is_invalid() { + assert_status("if x:\n\n", InputStatus::Invalid); + assert_status("try:\n\n", InputStatus::Invalid); + assert_status("match x:\n\n", InputStatus::Invalid); + assert_status("if x:\n pass\nelse:\n\n", InputStatus::Invalid); + } + + #[test] + fn a_comment_line_does_not_end_a_compound_statement() { + assert_status("if x:\n y = 1\n# more\n", InputStatus::Incomplete); + } + + /// a statement that only spans several lines, because a bracket was open, is complete + /// once the bracket closes, as python's repl runs it + #[test] + fn a_simple_statement_over_several_lines_is_complete_when_it_closes() { + assert_status("x = (1,\n", InputStatus::Incomplete); + assert_status("x = (1,\n 2)\n", InputStatus::Complete); + assert_status("call(\n", InputStatus::Incomplete); + assert_status("xs = [\n 1,\n", InputStatus::Incomplete); + assert_status("x = 1 + \\\n", InputStatus::Incomplete); + assert_status("x = 1 + \\\n 2\n", InputStatus::Complete); + } + + #[test] + fn an_open_triple_quoted_string_is_incomplete() { + assert_status("x = \"\"\"one\n", InputStatus::Incomplete); + assert_status("x = '''one\ntwo\n", InputStatus::Incomplete); + assert_status("x = f\"\"\"one {y}\n", InputStatus::Incomplete); + assert_status("x = \"\"\"one\n\"\"\"\n", InputStatus::Complete); + } + + #[test] + fn an_error_no_line_can_fix_is_invalid() { + assert_status("x = = 1\n", InputStatus::Invalid); + assert_status("x = \"one\n", InputStatus::Invalid); + assert_status(")\n", InputStatus::Invalid); + assert_status(" x = 1\n", InputStatus::Invalid); + assert_status("else:\n", InputStatus::Invalid); + assert_status("if x:\ny = 1\n", InputStatus::Invalid); + assert_status("x = 1 +\n", InputStatus::Invalid); + assert_status("lambda:\n", InputStatus::Invalid); + assert_status("def f(:\n", InputStatus::Invalid); + assert_status("x = (1, = 2\n", InputStatus::Invalid); + assert_status("if x:\n pass\nelse\n", InputStatus::Invalid); + assert_status("x = f\"a{\n", InputStatus::Invalid); + assert_status("x = 1 +", InputStatus::Invalid); + assert_status("x = = (1,\n", InputStatus::Invalid); + assert_status("if x:\n @decorator\ny = 1\n", InputStatus::Invalid); + assert_status("try:\n pass\nx = 1\n", InputStatus::Invalid); + } + + #[test] + fn a_nested_block_header_waits_for_its_body() { + assert_status("def f():\n if x:\n", InputStatus::Incomplete); + assert_status("class C:\n @property\n", InputStatus::Incomplete); + assert_status("def f():\n\tif x:\n", InputStatus::Incomplete); + assert_status("if x:\n pass\nelif y:\n", InputStatus::Incomplete); + assert_status("@dec(1)\n# why\n", InputStatus::Incomplete); + } + + /// a framed input is sent whole, so a compound statement followed by a simple one is one + /// input that ends in a simple statement + #[test] + fn statements_after_a_compound_one_complete_the_input() { + assert_status("if x:\n y = 1\nz = 2\n", InputStatus::Complete); + assert_status( + "def f():\n a = 1\n\n return a\nf()\n", + InputStatus::Complete, + ); + } + + #[test] + fn basedpython_compound_statements_wait_for_a_blank_line() { + assert_status("if let x := y:\n pass\n", InputStatus::Incomplete); + assert_status( + "let [a, b] := xs else:\n raise E\n", + InputStatus::Incomplete, + ); + assert_status( + "let [a, b] := xs else:\n raise E\n\n", + InputStatus::Complete, + ); + } +} diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index fb775b3be7..9996049123 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -331,6 +331,7 @@ pub fn parse_cells_unchecked( let mut body = Suite::new(); let mut tokens = Vec::new(); let mut errors = Vec::new(); + let mut ran_out_of_input = false; let mut unsupported_syntax_errors = Vec::new(); let mut generated_names = GeneratedNames::default(); let mut previous_end: Option = None; @@ -347,6 +348,7 @@ pub fn parse_cells_unchecked( syntax, tokens: cell_tokens, errors: cell_errors, + ran_out_of_input: cell_ran_out_of_input, unsupported_syntax_errors: cell_unsupported_syntax_errors, generated_names: cell_generated_names, } = Parser::new_starts_at(cell_source, range.start(), options.clone()) @@ -356,6 +358,9 @@ pub fn parse_cells_unchecked( body.extend(syntax.body); tokens.extend(cell_tokens); + if errors.is_empty() { + ran_out_of_input = cell_ran_out_of_input; + } errors.extend(cell_errors); unsupported_syntax_errors.extend(cell_unsupported_syntax_errors); generated_names.extend(cell_generated_names); @@ -392,6 +397,7 @@ pub fn parse_cells_unchecked( tokens: Tokens::new(tokens), generated_names, errors, + ran_out_of_input, unsupported_syntax_errors, } } @@ -402,6 +408,7 @@ pub struct Parsed { syntax: T, tokens: Tokens, errors: Vec, + ran_out_of_input: bool, unsupported_syntax_errors: Vec, generated_names: GeneratedNames, } @@ -422,6 +429,15 @@ impl Parsed { &self.errors } + /// basedpython: whether the first syntax error the parser met is the input ending where the + /// grammar needs more of it — a block whose body has not begun, a decorator with nothing + /// after it, a `try` with no handler yet — which is what an interactive console reads as + /// an input still being typed. an open bracket or an unterminated string is the lexer's + /// error instead, in [`Parsed::errors`] + pub fn ran_out_of_input(&self) -> bool { + self.ran_out_of_input + } + /// Returns a list of version-related syntax errors found during parsing. pub fn unsupported_syntax_errors(&self) -> &[UnsupportedSyntaxError] { &self.unsupported_syntax_errors @@ -517,6 +533,7 @@ impl Parsed { syntax: module, tokens: self.tokens, errors: self.errors, + ran_out_of_input: self.ran_out_of_input, unsupported_syntax_errors: self.unsupported_syntax_errors, generated_names: self.generated_names, }), @@ -538,6 +555,7 @@ impl Parsed { syntax: expression, tokens: self.tokens, errors: self.errors, + ran_out_of_input: self.ran_out_of_input, unsupported_syntax_errors: self.unsupported_syntax_errors, generated_names: self.generated_names, }), diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 3579e3ec94..0e6181e16e 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -74,6 +74,10 @@ pub(crate) struct Parser<'src> { /// Stores all the syntax errors found during the parsing. errors: Vec, + /// basedpython: whether the first of `errors` was met at the end of the input, see + /// [`Parsed::ran_out_of_input`]. it says nothing while `errors` is empty + first_error_at_end: bool, + /// Stores non-fatal syntax errors found during parsing, such as version-related errors. unsupported_syntax_errors: Vec, @@ -180,6 +184,7 @@ impl<'src> Parser<'src> { options, source, errors: Vec::new(), + first_error_at_end: false, unsupported_syntax_errors: Vec::new(), tokens, name_interner: NameInterner::default(), @@ -284,6 +289,7 @@ impl<'src> Parser<'src> { ); // TODO consider re-integrating lexical error handling into the parser? let parse_errors = self.errors; + let ran_out_of_input = !parse_errors.is_empty() && self.first_error_at_end; let (tokens, lex_errors) = self.tokens.finish(); // Fast path for when there are no lex errors. @@ -294,6 +300,7 @@ impl<'src> Parser<'src> { syntax, tokens: Tokens::new(tokens), errors: parse_errors, + ran_out_of_input, unsupported_syntax_errors: self.unsupported_syntax_errors, generated_names: self.generated_names, }; @@ -327,6 +334,7 @@ impl<'src> Parser<'src> { syntax, tokens: Tokens::new(tokens), errors: merged, + ran_out_of_input, unsupported_syntax_errors: self.unsupported_syntax_errors, generated_names: self.generated_names, } @@ -721,9 +729,23 @@ impl<'src> Parser<'src> { } } + if self.errors.is_empty() { + self.first_error_at_end = self.at_end_of_input(); + } inner(&mut self.errors, error, ranged.range()); } + /// basedpython: whether every token is consumed but the dedents that close the input's + /// open blocks. an implicit newline ending the last line does not count: the line it ends + /// is over + fn at_end_of_input(&self) -> bool { + match self.current_token_kind() { + TokenKind::EndOfFile => true, + TokenKind::Dedent => self.current_token_range().start() == TextSize::of(self.source), + _ => false, + } + } + /// Add an [`UnsupportedSyntaxError`] with the given [`UnsupportedSyntaxErrorKind`] and /// [`TextRange`] if its minimum version is less than [`Parser::target_version`]. fn add_unsupported_syntax_error(&mut self, kind: UnsupportedSyntaxErrorKind, range: TextRange) { diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 5310e9fdf8..38440bb5e8 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -147,6 +147,10 @@ pub(super) fn request(req: server::Request) -> Task { requests::RulesHandler::METHOD => { background_request_task::(req, BackgroundSchedule::Worker) } + // a question about the text a `by repl` console holds, with no document behind it + requests::InputStatusHandler::METHOD => { + background_request_task::(req, BackgroundSchedule::Worker) + } // The program model a run configuration is built from. Asked by URI rather than by open // document, because the module a configuration names is usually not open, and asked off // an editor's own schedule, so none of the three is latency sensitive. diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 9f66cd7c96..f60b3c4e26 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -50,6 +50,7 @@ mod goto_type_definition; mod hover; mod injections; mod inlay_hints; +mod input_status; mod merge_conflicts; mod module_name; mod move_members; @@ -126,6 +127,7 @@ pub(super) use goto_type_definition::GotoTypeDefinitionRequestHandler; pub(super) use hover::HoverRequestHandler; pub(super) use injections::InjectionsRequestHandler; pub(super) use inlay_hints::InlayHintRequestHandler; +pub(super) use input_status::InputStatusHandler; pub(super) use merge_conflicts::{DiffIgnoredRangesHandler, ResolveMergeConflictsHandler}; pub(super) use module_name::ModuleNameHandler; pub(super) use move_members::{ diff --git a/crates/ty_server/src/server/api/requests/input_status.rs b/crates/ty_server/src/server/api/requests/input_status.rs new file mode 100644 index 0000000000..795d064f9f --- /dev/null +++ b/crates/ty_server/src/server/api/requests/input_status.rs @@ -0,0 +1,58 @@ +//! `by/inputStatus` — whether the text typed into a `by repl` console is a whole input yet +//! +//! an editor's console decides at each enter whether to run what has been typed or to start a +//! new line of it, which is a question about basedpython's grammar: a compound statement goes on +//! until a blank line, an open bracket until it closes. `by repl` answers it the same way for +//! the lines it reads itself, from [`by_stage::session::input_status`], so the two agree. the text +//! is all there is to it: no document, and no project + +use by_stage::session::{InputStatus, input_status}; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; + +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::SessionSnapshot; +use crate::session::client::Client; + +pub(crate) enum InputStatusRequest {} + +impl Request for InputStatusRequest { + type Params = InputStatusParams; + type Result = InputStatusResult; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/inputStatus"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct InputStatusParams { + /// every line typed so far, as the console holds it + text: String, +} + +#[derive(Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct InputStatusResult { + status: InputStatus, +} + +pub(crate) struct InputStatusHandler; + +impl RequestHandler for InputStatusHandler { + type RequestType = InputStatusRequest; +} + +impl BackgroundRequestHandler for InputStatusHandler { + fn run( + _snapshot: &SessionSnapshot, + _client: &Client, + params: InputStatusParams, + ) -> crate::server::Result { + Ok(InputStatusResult { + status: input_status(¶ms.text), + }) + } +} + +impl RetriableRequestHandler for InputStatusHandler {} diff --git a/crates/ty_server/tests/e2e/input_status.rs b/crates/ty_server/tests/e2e/input_status.rs new file mode 100644 index 0000000000..60f55b78ef --- /dev/null +++ b/crates/ty_server/tests/e2e/input_status.rs @@ -0,0 +1,59 @@ +use crate::TestServerBuilder; + +/// the request as a client sends it, in json throughout +enum InputStatus {} + +impl lsp_types::Request for InputStatus { + type Params = serde_json::Value; + type Result = serde_json::Value; + const METHOD: lsp_types::LspRequestMethod<'static> = + lsp_types::LspRequestMethod::Custom("by/inputStatus"); + const MESSAGE_DIRECTION: lsp_types::MessageDirection = + lsp_types::MessageDirection::ClientToServer; +} + +/// `by/inputStatus` says whether a `by repl` console's text runs at enter, with no document +/// open: a block runs at the blank line after it, a header, a decorator, an open bracket, a +/// continued line or an open triple-quoted string waits for the next line, and input no line +/// can fix runs at once +#[test] +fn input_status_says_whether_enter_runs_the_input() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .build() + .wait_until_workspaces_are_initialized(); + + for (text, status) in [ + ("let rate = 0.2\n", "complete"), + ("def taxed(amount: float) -> float:\n", "incomplete"), + ( + "def taxed(amount: float) -> float:\n return amount * (1 + rate)\n", + "incomplete", + ), + ( + "def taxed(amount: float) -> float:\n return amount * (1 + rate)\n\n", + "complete", + ), + ("total = (1 +\n", "incomplete"), + ("x = = 1\n", "invalid"), + ("try:\n", "incomplete"), + ("try:\n risky()\n", "incomplete"), + ("try:\n risky()\nexcept ValueError:\n", "incomplete"), + ("try:\n risky()\nfinally:\n", "incomplete"), + ("match command:\n", "incomplete"), + ("match command:\n case \"go\":\n", "incomplete"), + ("with open(path) as f:\n", "incomplete"), + ("class Shape:\n", "incomplete"), + ("async def fetch():\n", "incomplete"), + ("@cache\n", "incomplete"), + ("if ready:\n go()\nelif waiting:\n", "incomplete"), + ("if ready:\n go()\nelse:\n", "incomplete"), + ("total = 1 + \\\n", "incomplete"), + ("doc = \"\"\"one\n", "incomplete"), + ("try:\n\n", "invalid"), + ("lambda:\n", "invalid"), + ] { + let answer = server.send_request_await::(serde_json::json!({ "text": text })); + assert_eq!(answer, serde_json::json!({ "status": status }), "{text:?}"); + } + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index ab19a94708..d09fc2a172 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -57,6 +57,7 @@ mod inherited_members; mod initialize; mod injections; mod inlay_hints; +mod input_status; mod merge_conflicts; mod move_symbol; mod named_text; diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 8ae7ec59ad..69bad9b246 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1138,6 +1138,16 @@ a removed rule is left out. `by/explainRule` gives a rule's whole documentation, and the linter's rules, `F401` among them, are `buff`'s. rules have no categories +## a console's input + +a console running [`by repl`](repl.md) asks `by/inputStatus`, with the text +typed so far as `text`, whether enter runs it or starts another line of it. the +answer's `status` is `complete`, `incomplete` — a compound statement whose body +could still go on, an open bracket, an unterminated triple-quoted string, a line +continued with `\` — or `invalid`, input no further line can make valid, which +runs to report why. it is the answer `by repl` gives itself for the lines it +reads, and needs no document open + ## suppressing a diagnostic every diagnostic has two quick fixes that silence it rather than fix it. diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index 9bd50803d6..716f2178e1 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -1,7 +1,7 @@ # the repl -> **status: design.** what this page describes is being built; `by repl` and `by/inputStatus` -> land in the commits that follow it +> **status: design.** `by/inputStatus` is built; `by repl` itself is being built, and lands in +> the commits that follow `by repl` is an interactive session in which you type basedpython. each input is checked and transpiled with everything the session has defined so far, and runs in one python process From ff4b6e7c2a0af8c2ee4f751b87b3fdccf2bebf82 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:12:11 +1000 Subject: [PATCH 073/110] transpile a by repl session with Config::session, under which a top-level main gets no __main__ guard, since a session never runs as a script, and an import the emitted code needs stays on a line of its own rather than joining an earlier input's import, which has already run --- crates/by_transforms/src/config.rs | 9 ++++ .../src/transforms/ast_driver.rs | 7 ++-- .../src/transforms/main_function.rs | 30 +++++++++++++- .../by_transforms/src/transforms/modifiers.rs | 41 +++++++++++++++++++ 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index ce9d5142a8..62f492726d 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -86,6 +86,14 @@ pub struct Config { /// writes into its tree; a transpile with nowhere to write one leaves it /// `None` pub runtime_module: Option, + /// when true, the module is a `by repl` session: the inputs typed so far, transpiled again + /// with each new one, of which only the statements written for the new input run. off by + /// default. two things follow. the module never runs as a script, so a top-level `main` + /// gets no `__main__` guard: it is a function like any other, and nothing calls it. and an + /// earlier input's statements have already run, so an import the emitted code needs is + /// written on a line of its own rather than joined to one of the module's own imports, + /// which would change a statement that cannot run again + pub session: bool, } /// What a float or complex literal type becomes in the emitted python. @@ -127,6 +135,7 @@ impl Default for Config { float_literals: FloatLiteralLowering::Nominal, runtime_module: None, stamps: BTreeMap::new(), + session: false, } } } diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index b8fc6362bc..176ca47e37 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1496,7 +1496,7 @@ pub(crate) fn run_against_source<'a>( let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref, written); let modifiers_pass = modifiers::ModifiersPass::new(source_ref, written, config); - let main_function_pass = main_function::MainFunction::new(source_ref, written); + let main_function_pass = main_function::MainFunction::new(source_ref, written, config.session); let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); let overload_pass = overload::Overload::new(source_ref, written, config.is_stub); @@ -2275,8 +2275,9 @@ pub(crate) fn run_against_source<'a>( ctx.required_imports.splice(0..0, builtins); // a name the module already imports from the same place goes on its line rather than on // one of ours. done here and not against `required_imports` alone, because the statement - // it joins is the module's own - if !ctx.required_imports.is_empty() { + // it joins is the module's own. a session's own imports have run already, so none of them + // is rewritten + if !config.session && !ctx.required_imports.is_empty() { ctx.required_imports = merge_into_own_imports(&mut out, std::mem::take(&mut ctx.required_imports)); } diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index 4ef8da408c..60d438dba4 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -27,11 +27,17 @@ use super::source_util::{is_synthetic_decorator, python_string_literal}; pub(crate) struct MainFunction<'src> { source: &'src str, written: WrittenNames<'src>, + /// a `by repl` session never runs as a script, so its `main` is called by nothing + session: bool, } impl<'src> MainFunction<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>) -> Self { - Self { source, written } + pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, session: bool) -> Self { + Self { + source, + written, + session, + } } } @@ -43,6 +49,9 @@ impl AstPass for MainFunction<'_> { } fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + if self.session { + return; + } let Some(entry) = entry_point(&module.body, self.source, self.written) else { return; }; @@ -641,6 +650,23 @@ mod tests { ); } + /// a session never runs as a script, so its `main` is a function like any other + #[test] + fn a_session_s_main_gets_no_guard() { + let config = Config { + session: true, + ..Config::test_default() + }; + let source = indoc! {" + def main(): + print(\"hi\") + "}; + assert_eq!( + transpile(source, &config).unwrap(), + crate::python_passthrough::lazify_expected(source) + ); + } + #[test] fn bodyless_main_gets_guard() { check( diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index 79efa8428d..c393f76421 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -1368,6 +1368,47 @@ mod tests { ); } + /// the `Final` a `let` needs joins the module's own `typing` import, which is a line + /// already there + #[test] + fn let_decl_joins_the_module_s_typing_import() { + check( + indoc! {" + from typing import Protocol + let MAX = 100 + "}, + indoc! {" + from typing import Protocol, Final + MAX: Final = 100 + "}, + ); + } + + /// in a session the module's import is an earlier input, which has run, so the `Final` + /// is imported on a line of its own + #[test] + fn let_decl_in_a_session_imports_final_apart() { + let config = Config { + session: true, + ..Config::test_default() + }; + assert_eq!( + transpile( + indoc! {" + from typing import Protocol + let MAX = 100 + "}, + &config + ) + .unwrap(), + crate::python_passthrough::lazify_expected(indoc! {" + from typing import Final + from typing import Protocol + MAX: Final = 100 + "}) + ); + } + #[test] fn class_var_annotated_decl() { check( From 2d03dc99171713c6646e38cacccf66f586230937 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:37:11 +1000 Subject: [PATCH 074/110] start an interactive basedpython session with by repl, in which each input is checked and transpiled as the continuation of the module made of the inputs that ran, its errors reported against and its own lines, and only the python written for it runs, in one interpreter staged as by run stages the project, printing an expression's value and tracebacks at the inputs' lines, taking a block framed as a terminal frames a paste whole, and keeping --session-file holding the session's source for an editor's console --- Cargo.lock | 2 + crates/by_stage/src/project.rs | 9 +- crates/by_stage/src/session.rs | 298 +++++++++++++++++- crates/ty/Cargo.toml | 6 +- crates/ty/docs/cli.md | 26 ++ crates/ty/src/args.rs | 24 ++ crates/ty/src/by_commands.rs | 359 +++++++++++++++++---- crates/ty/src/by_repl.rs | 508 ++++++++++++++++++++++++++++++ crates/ty/src/lib.rs | 12 + crates/ty/tests/by_e2e.rs | 249 +++++++++++++++ docs/basedpython/cli-reference.md | 14 + docs/basedpython/features/repl.md | 31 +- 12 files changed, 1464 insertions(+), 74 deletions(-) create mode 100644 crates/ty/src/by_repl.rs diff --git a/Cargo.lock b/Cargo.lock index 705adbba5c..c1339a5b38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4892,10 +4892,12 @@ dependencies = [ "ruff_command_line", "ruff_db", "ruff_diagnostics", + "ruff_notebook", "ruff_python_ast", "ruff_python_parser", "ruff_python_trivia", "ruff_ranged_value", + "ruff_source_file", "ruff_text_size", "salsa", "serde", diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 52ecfbdeea..aa97997e60 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -98,11 +98,18 @@ impl Rebuilder { } pub fn rebuild(&self) -> Box { + Box::new(self.project_database()) + } + + /// The project's db built again, and not frozen: what a caller holds that goes on + /// changing an input after the build, as a `by repl` session changes its module with + /// every input. + pub fn project_database(&self) -> ProjectDatabase { let mut db = ProjectDatabase::use_defaults(self.metadata.clone(), OsSystem::new(&self.root)); db.project() .set_included_paths(&mut db, self.included.clone()); - Box::new(db) + db } } diff --git a/crates/by_stage/src/session.rs b/crates/by_stage/src/session.rs index 8f45e255bb..553fb7141e 100644 --- a/crates/by_stage/src/session.rs +++ b/crates/by_stage/src/session.rs @@ -2,14 +2,18 @@ //! //! a session is a module whose source is the inputs that ran, in order, and each new input is //! checked and transpiled as that module's continuation (see `docs/basedpython/features/repl.md`). -//! What is here is the part of that which is a question about source text, so the command and -//! the language server give the same answers: whether what has been typed is a whole input yet. +//! What is here is the part of that which is a question about source text: whether what has been +//! typed is a whole input yet, which the command and the language server both answer, and which +//! of the python the module comes to is the new input's to run. + +use std::collections::HashSet; use ruff_python_ast::{PySourceType, Stmt}; use ruff_python_parser::{ InterpolatedStringErrorType, LexicalErrorType, ParseError, ParseErrorType, parse_unchecked_source, }; +use ruff_text_size::{Ranged, TextSize}; /// whether the text typed so far is an input that can run #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -119,6 +123,209 @@ fn ends_with_blank_line(text: &str) -> bool { last_line.trim().is_empty() && before_last_newline.contains('\n') } +/// The inputs of a session that ran, as the source of the one module they make. +/// +/// Each new input is checked and transpiled as this module's continuation, by +/// [`Session::with_input`]; [`Session::select`] then picks out of the python the module came +/// to the statements that are the input's to run, and [`Session::commit`] makes the input part +/// of the session once it is handed over to run. +#[derive(Debug, Default)] +pub struct Session { + source: String, + /// each input that ran, in order, with the line of `source` it starts at + inputs: Vec, + /// the python each statement that has run was transpiled to. a statement is matched by + /// its text, which is what it means to python: one transpiled the same way again is the + /// one that ran + ran: HashSet, +} + +#[derive(Debug)] +struct RanInput { + /// the `N` the input is named by, `` + number: u32, + first_line: u32, + line_count: u32, +} + +/// Where an input sits in the module made of a session and that input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InputLines { + /// the module line, 0-based, the input's first line is + pub first: u32, + /// how many lines the input is + pub count: u32, +} + +impl InputLines { + fn contains(self, line: u32) -> bool { + line >= self.first && line - self.first < self.count + } +} + +/// What of the python a session and its new input came to runs for the input. +#[derive(Debug, PartialEq, Eq)] +pub struct Selection { + /// the top-level statements of the python that run, by their index among them, in order + pub statements: Vec, + /// for each line of the python, 0-based, the line of the input it was written for, 0-based, + /// or `None` for one written for no line of the input + pub lines: Vec>, + /// the python of each statement that runs, which has run once the input is committed + texts: Vec, +} + +/// Why the python written for an input cannot run. +#[derive(Debug, PartialEq, Eq)] +pub enum Refusal { + /// the python written for an earlier input, which has already run, came out differently + /// with this one after it. `input` and `line` (1-based) say where + ChangesAnEarlierInput { input: u32, line: u32 }, + /// one statement of the python was written for this input and an earlier one together + JoinsAnEarlierInput { input: u32 }, + /// the python is not python: the transpiler wrote something no interpreter reads + NotPython(String), +} + +impl Session { + /// The module made of the session's inputs: the session's source. + pub fn source(&self) -> &str { + &self.source + } + + /// The source of the module made of the session and `input` after it, and where in it + /// the input is. An input is a whole number of lines, so one that does not end in a line + /// ending is given one. + pub fn with_input(&self, input: &str) -> (String, InputLines) { + let mut source = self.source.clone(); + let first = line_count(&source); + source.push_str(input); + if !input.ends_with('\n') { + source.push('\n'); + } + let count = line_count(&source) - first; + (source, InputLines { first, count }) + } + + /// The input of the session that holds `line` of its source, 0-based, as the number it is + /// named by and the line of it, 0-based. `None` for a line past the session's, which is + /// the new input's. + pub fn input_at(&self, line: u32) -> Option<(u32, u32)> { + self.inputs + .iter() + .find(|input| line >= input.first_line && line - input.first_line < input.line_count) + .map(|input| (input.number, line - input.first_line)) + } + + /// Which of `python`, what the module made of the session and the input at `input` was + /// transpiled to, runs for the input. `line_map` is the transpile's: for each line of + /// `python`, the module line it was written for. + /// + /// A top-level statement written for the input's lines runs. One written for an earlier + /// input's has run already, and is skipped — unless it came out differently this time, + /// which means the input changed what an earlier input meant after it ran, and refuses + /// the input. One written for no line at all is the transpiler's own, a runtime helper + /// or an import it needs, and runs the first time it appears. + pub fn select( + &self, + python: &str, + line_map: &[Option], + input: InputLines, + ) -> Result { + let parsed = ruff_python_parser::parse_module(python) + .map_err(|error| Refusal::NotPython(error.to_string()))?; + let line_starts = line_starts(python); + let line_of = |offset: TextSize| { + let offset = usize::from(offset); + u32::try_from(line_starts.partition_point(|&start| start <= offset) - 1) + .unwrap_or(u32::MAX) + }; + let mut statements = Vec::new(); + let mut texts = Vec::new(); + for (index, stmt) in parsed.suite().iter().enumerate() { + let first = line_of(stmt.start()); + // the line of its last character, which a statement ending in a newline is not on + let last = + line_of(stmt.end().max(stmt.start() + TextSize::from(1)) - TextSize::from(1)); + let written_for: Vec = (first..=last) + .filter_map(|line| line_map.get(line as usize).copied().flatten()) + .collect(); + let earlier = written_for.iter().find(|line| !input.contains(**line)); + let this = written_for.iter().any(|line| input.contains(*line)); + let text = &python[stmt.range()]; + match (this, earlier) { + (true, Some(&line)) => { + return Err(Refusal::JoinsAnEarlierInput { + input: self.input_at(line).map_or(0, |(number, _)| number), + }); + } + (false, Some(&line)) => { + if !self.ran.contains(text) { + let (input, line) = self.input_at(line).unwrap_or((0, line)); + return Err(Refusal::ChangesAnEarlierInput { + input, + line: line + 1, + }); + } + } + (true, None) => { + statements.push(index); + texts.push(text.to_owned()); + } + (false, None) => { + if !self.ran.contains(text) { + statements.push(index); + texts.push(text.to_owned()); + } + } + } + } + let lines = (0..line_starts.len()) + .map(|line| { + line_map + .get(line) + .copied() + .flatten() + .filter(|line| input.contains(*line)) + .map(|line| line - input.first) + }) + .collect(); + Ok(Selection { + statements, + lines, + texts, + }) + } + + /// Make `input`, named ``, part of the session, with the python + /// `selection` picked for it as what has run. + pub fn commit(&mut self, number: u32, input: &str, selection: Selection) { + let (source, lines) = self.with_input(input); + self.source = source; + self.inputs.push(RanInput { + number, + first_line: lines.first, + line_count: lines.count, + }); + self.ran.extend(selection.texts); + } +} + +/// How many lines `text` is, counting a last line with no line ending. +fn line_count(text: &str) -> u32 { + let endings = text.bytes().filter(|byte| *byte == b'\n').count(); + let unended = usize::from(!text.is_empty() && !text.ends_with('\n')); + u32::try_from(endings + unended).unwrap_or(u32::MAX) +} + +/// The offset each line of `text` starts at. +fn line_starts(text: &str) -> Vec { + std::iter::once(0) + .chain(text.match_indices('\n').map(|(at, _)| at + 1)) + .filter(|&start| start < text.len() || start == 0) + .collect() +} + #[cfg(test)] mod tests { use super::{InputStatus, input_status}; @@ -324,3 +531,90 @@ mod tests { ); } } + +#[cfg(test)] +mod selection_tests { + use super::{InputLines, Refusal, Session}; + + /// a session with `inputs` committed, each transpiled to itself line for line + fn session_of(inputs: &[&str]) -> Session { + let mut session = Session::default(); + for (number, input) in (1..).zip(inputs) { + let (_, lines) = session.with_input(input); + let (module, _) = session.with_input(input); + let line_map: Vec> = (0..lines.first + lines.count).map(Some).collect(); + let selection = session + .select(&module, &line_map, lines) + .expect("each input transpiled to itself"); + session.commit(number, input, selection); + } + session + } + + #[test] + fn an_input_runs_the_statements_written_for_its_lines() { + let session = session_of(&["x = 1\n"]); + let (module, lines) = session.with_input("y = x\nprint(y)"); + assert_eq!(module, "x = 1\ny = x\nprint(y)\n"); + assert_eq!(lines, InputLines { first: 1, count: 2 }); + let selection = session + .select(&module, &[Some(0), Some(1), Some(2)], lines) + .expect("runs"); + assert_eq!(selection.statements, [1, 2]); + assert_eq!(selection.lines, [None, Some(0), Some(1)]); + } + + /// a runtime helper or an import the transpiler wrote is written for no line, and runs the + /// first time it appears + #[test] + fn what_the_transpiler_wrote_runs_once() { + let mut session = Session::default(); + let (_, lines) = session.with_input("let MAX = 1\n"); + let python = "from typing import Final\nMAX: Final = 1\n"; + let selection = session + .select(python, &[None, Some(0)], lines) + .expect("runs"); + assert_eq!(selection.statements, [0, 1]); + session.commit(1, "let MAX = 1\n", selection); + + let (_, lines) = session.with_input("let LIMIT = MAX\n"); + let python = "from typing import Final\nMAX: Final = 1\nLIMIT: Final = MAX\n"; + let selection = session + .select(python, &[None, Some(0), Some(1)], lines) + .expect("runs"); + assert_eq!(selection.statements, [2]); + assert_eq!(selection.lines, [None, None, Some(0)]); + } + + #[test] + fn an_input_that_changes_an_earlier_one_is_refused() { + let session = session_of(&["x = 1\n", "y = 2\n"]); + let (_, lines) = session.with_input("z = 3\n"); + assert_eq!( + session.select( + "x = 1\ny = 2.0\nz = 3\n", + &[Some(0), Some(1), Some(2)], + lines + ), + Err(Refusal::ChangesAnEarlierInput { input: 2, line: 1 }) + ); + } + + #[test] + fn a_statement_written_for_an_earlier_input_and_this_one_is_refused() { + let session = session_of(&["x = 1\n"]); + let (_, lines) = session.with_input("y = 2\n"); + assert_eq!( + session.select("x = (1,\n 2)\n", &[Some(0), Some(1)], lines), + Err(Refusal::JoinsAnEarlierInput { input: 1 }) + ); + } + + #[test] + fn a_line_of_the_session_is_found_in_its_input() { + let session = session_of(&["x = 1\n", "def f():\n return x\n\n"]); + assert_eq!(session.input_at(0), Some((1, 0))); + assert_eq!(session.input_at(2), Some((2, 1))); + assert_eq!(session.input_at(4), None); + } +} diff --git a/crates/ty/Cargo.toml b/crates/ty/Cargo.toml index f33fbafc80..a065371eee 100644 --- a/crates/ty/Cargo.toml +++ b/crates/ty/Cargo.toml @@ -19,17 +19,15 @@ default-run = "by" name = "by" path = "src/main.rs" -[package.metadata.cargo-shear] -# Used only to enable parser stack growth. -ignored = ["ruff_python_parser"] - [lib] doctest = false [dependencies] ruff_command_line = { workspace = true } ruff_db = { workspace = true, features = ["os", "cache", "junit"] } +ruff_notebook = { workspace = true } ruff_python_ast = { workspace = true } +ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } ruff_diagnostics = { workspace = true } ruff_python_parser = { workspace = true, features = ["stacker"] } diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 4630f1b913..b8c06e1388 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -19,6 +19,7 @@ by
by version

Display ty's version

by explain

Explain rules and other parts of ty

by run

Transpile and run a module with python -m <module>

+
by repl

start an interactive session in basedpython

by init

Start a new project

by build

Build the project as python

by restage

Recompute the slots of a set of edited files in a build tree that already exists

@@ -273,6 +274,31 @@ by run [OPTIONS] [MODULE] [ARGS]...

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+## by repl + +start an interactive session in basedpython + +each input is checked and transpiled as the continuation of the module made of the inputs that ran before it, and runs in one python process that lives as long as the session. the project is staged as `by run` stages it, so its modules import as they do in a program + +

Usage

+ +``` +by repl [OPTIONS] +``` + +

Options

+ +
--help, -h

Print help (see a summary with '-h')

+
--min-version version

minimum Python version the session's python must run on [default: the version of the interpreter that will run it]

+
--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--python, --venv path

the interpreter to run on, or the environment holding it, as by run takes it

+
--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

+
--session-file path

keep PATH holding the session's source: the inputs that ran, in order

+

for an editor's console, whose completion is asked of the module this file holds followed by the input being typed

+
--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

+

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+
+ ## by init Start a new project. diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 236cf479e9..36cb2628f3 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -145,6 +145,30 @@ pub(crate) enum Command { profile: Option, }, + /// Start an interactive session in basedpython. + /// + /// Each input is checked and transpiled as the continuation of the module made of the + /// inputs that ran before it, and runs in one python process that lives as long as the + /// session. The project is staged as `by run` stages it, so its modules import as they do + /// in a program. + Repl { + /// minimum Python version the session's python must run on + /// [default: the version of the interpreter that will run it] + #[arg(long, value_name = "VERSION")] + min_version: Option, + /// The interpreter to run on, or the environment holding it, as `by run` takes it. + #[arg(long, value_name = "PATH", alias = "venv")] + python: Option, + #[command(flatten)] + lowering: LoweringArgs, + /// Keep PATH holding the session's source: the inputs that ran, in order. + /// + /// For an editor's console, whose completion is asked of the module this file holds + /// followed by the input being typed. + #[arg(long, value_name = "PATH")] + session_file: Option, + }, + /// Start a new project. /// /// Writes a `pyproject.toml` that names the basedpython build backend, a diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 2a0be58d9a..09d07c7fc8 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -148,22 +148,46 @@ pub(crate) struct RunFlags<'a> { pub(crate) profile: Option<&'a Path>, } -#[allow(clippy::exit, clippy::print_stderr)] -pub(crate) fn cmd_run( - module: Option<&str>, - args: &[String], +/// What `by run` and `by repl` both start from: the project checked, transpiled and staged +/// into a temporary tree, and the interpreter that runs out of it. +pub(crate) struct StagedRun { + /// where the command was invoked, which is where the program runs + pub(crate) cwd: PathBuf, + project: ResolvedProject, + pub(crate) interpreter: Interpreter, + /// what the project was transpiled for: the interpreter's version unless told otherwise + pub(crate) config: Config, + /// the staged tree, deleted when this is dropped + pub(crate) tree: tempfile::TempDir, + /// the project file behind each copy in the tree + pub(crate) carried: BTreeMap, + traceback_entries: Vec, + /// the means to build the project's db again, unfrozen + pub(crate) rebuilder: Rebuilder, + /// the module `by run` runs, which a session has none of + module: Option, +} + +/// What a staged tree is for. +#[derive(Clone, Copy)] +pub(crate) enum RunTarget<'a> { + /// `by run`'s program: the module named, or the project's entry point + Module(Option<&'a str>), + /// a `by repl` session, which runs no module of the project's + Session, +} + +/// Check, transpile and stage the project around the working directory for `target`, on the +/// interpreter `python_flag` names or discovery finds. `Err` is the exit status of a project +/// that cannot run: one that does not check, having said why. +#[allow(clippy::print_stderr)] +pub(crate) fn stage_run( min_version: Option<&str>, lowering: &LoweringArgs, - flags: RunFlags<'_>, -) -> anyhow::Result { - let RunFlags { - compiled, - in_build, - python: python_flag, - launcher, - coverage, - profile, - } = flags; + python_flag: Option<&Path>, + compiled: bool, + run_target: RunTarget<'_>, +) -> anyhow::Result> { let cwd = std::env::current_dir().context("failed to get current directory")?; // one resolution of the project, for the environment and the target version // both: they are two readings of the same configuration and must not be able @@ -176,7 +200,8 @@ pub(crate) fn cmd_run( // match what that python actually supports. an explicit `--min-version` // wins, but it cannot exceed the interpreter — that would emit code the // interpreter cannot parse - let probed = detect_python_version(&python); + let probe = probe_interpreter(&python); + let probed = probe.as_ref().map(|probe| probe.version); let mut config = match (min_version, probed) { (Some(flag), probed) => { let config = parse_version(flag)?; @@ -220,30 +245,32 @@ pub(crate) fn cmd_run( let tmp = tempfile::TempDir::new().context("failed to create temp directory")?; // the program is checked against the packages of the python it is about to run on - let checked_against = interpreter - .named_for_run - .then(|| environment_of(&python)) - .flatten(); + let checked_against = probe + .filter(|_| interpreter.named_for_run) + .map(|probe| probe.prefix); let (db, handles, rebuilder, root) = build_project_db_for(&cwd, BY_SOURCES, None, checked_against.as_deref())?; - if handles.is_empty() { + // a session can start in a project with nothing to stage, as python's repl starts + // anywhere; a program has to be somewhere + if handles.is_empty() && matches!(run_target, RunTarget::Module(_)) { eprintln!("no .by files found"); - return Ok(ExitStatus::Failure); + return Ok(Err(ExitStatus::Failure)); } let roots = module_roots(&db, &root); // an explicit module always wins; otherwise the project's configured entry // point stands in for it. resolving before the (much slower) check means a // project with neither fails immediately rather than after transpiling - let module = match module { - Some(module) => module.to_owned(), - None => match by_stage::run_module::configured_main(&db) { - Some(main) => main, + let module = match run_target { + RunTarget::Module(Some(module)) => Some(module.to_owned()), + RunTarget::Module(None) => match by_stage::run_module::configured_main(&db) { + Some(main) => Some(main), None => anyhow::bail!( "no module given and no entry point configured — \ name a module (`by run main`) or set `run.main` in the project configuration" ), }, + RunTarget::Session => None, }; // each generated `.py` paired with its source `.by` and the line table that @@ -271,7 +298,7 @@ pub(crate) fn cmd_run( }, )?; if !matches!(status, ExitStatus::Success) { - return Ok(status); + return Ok(Err(status)); } // a program is its data as much as its modules: a `.py` module it imports, a // json file it opens, a template it renders. running out of a directory @@ -288,7 +315,7 @@ pub(crate) fn cmd_run( // configuration would emit different code in exactly the case that matters stage_build_record( &mut staging, - &BuildRecord::new(&root, &roots, Some(module.clone()), compiled, &config), + &BuildRecord::new(&root, &roots, module.clone(), compiled, &config), )?; // the project file behind each copy in the tree, for the runner to run it as // and a coverage report to name: the copy is deleted with the tree when the @@ -302,6 +329,58 @@ pub(crate) fn cmd_run( .collect(); write_project_module(&mut staging, &root, &carried)?; staging.finish()?; + Ok(Ok(StagedRun { + cwd, + project, + interpreter, + config, + tree: tmp, + carried, + traceback_entries, + rebuilder, + module, + })) +} + +#[allow(clippy::exit, clippy::print_stderr)] +pub(crate) fn cmd_run( + module: Option<&str>, + args: &[String], + min_version: Option<&str>, + lowering: &LoweringArgs, + flags: RunFlags<'_>, +) -> anyhow::Result { + let RunFlags { + compiled, + in_build, + python: python_flag, + launcher, + coverage, + profile, + } = flags; + let staged = match stage_run( + min_version, + lowering, + python_flag, + compiled, + RunTarget::Module(module), + )? { + Ok(staged) => staged, + Err(status) => return Ok(status), + }; + let StagedRun { + cwd, + project, + interpreter, + config, + tree: tmp, + carried, + traceback_entries, + rebuilder: _, + module, + } = staged; + let python = interpreter.path.clone(); + let module = module.context("a program is staged for the module it runs")?; if compiled { // the extension lands beside the generated `.py`, and python's finder @@ -705,7 +784,8 @@ pub(crate) fn default_project_python_version() -> PythonVersion { if interpreter.is_from_path { return latest_python_version(); } - detect_python_version(&interpreter.path).unwrap_or_else(|| Config::default().min_version) + probe_interpreter(&interpreter.path) + .map_or_else(|| Config::default().min_version, |probe| probe.version) } /// The newest python this release can emit for. @@ -718,9 +798,9 @@ fn latest_python_version() -> PythonVersion { /// The interpreter `by run` executes on, and how it was chosen. #[derive(Clone)] -struct Interpreter { - path: String, - origin: String, +pub(crate) struct Interpreter { + pub(crate) path: String, + pub(crate) origin: String, /// whether the command was told which python to run on, by `--python` or `$PYTHON`, rather /// than finding the project's environment: the checker finds that one itself, and an /// interpreter named only for the run has to be handed to it @@ -971,38 +1051,34 @@ fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Optio .map(|interpreter| interpreter.to_string()) } -/// Probe `python`'s `major.minor` version (e.g. `3.9`) so `run` can target the -/// interpreter it will execute on. Returns `None` if the interpreter can't be -/// run or its output can't be parsed. -/// The environment `python` runs in, as the checker takes one: the interpreter's own path when -/// it names a file, and otherwise — a bare name such as `python3.14`, found on `PATH` — the -/// `sys.prefix` the interpreter reports, since the checker does not search `PATH`. -fn environment_of(python: &str) -> Option { - if Path::new(python).is_file() { - return SystemPathBuf::from_path_buf(std::path::absolute(python).ok()?).ok(); - } - let output = Command::new(python) - .arg("-c") - .arg("import sys; print(sys.prefix)") - .output() - .ok()?; - if !output.status.success() { - return None; - } - let prefix = String::from_utf8(output.stdout).ok()?; - Some(SystemPathBuf::from(prefix.trim())) +/// what an interpreter says of itself, asked once +struct Probed { + /// its `major.minor`, e.g. `3.9`, which `run` targets + version: PythonVersion, + /// its `sys.prefix`: the environment it runs in, as the checker takes one + prefix: SystemPathBuf, } -fn detect_python_version(python: &str) -> Option { +/// `python`'s version and the environment it runs in, or `None` when it cannot be run or what +/// it says cannot be read +/// +/// the environment is the one the interpreter reports rather than one derived from where its +/// file is, so an interpreter named by path and the same one found on `PATH` are checked +/// against one environment, a launcher script that runs another included +fn probe_interpreter(python: &str) -> Option { let output = Command::new(python) .arg("-c") - .arg("import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')") + .arg("import json, sys; print(json.dumps([sys.version_info[0], sys.version_info[1], sys.prefix]))") .output() .ok()?; if !output.status.success() { return None; } - String::from_utf8(output.stdout).ok()?.trim().parse().ok() + let (major, minor, prefix): (u8, u8, String) = serde_json::from_slice(&output.stdout).ok()?; + Some(Probed { + version: PythonVersion::from((major, minor)), + prefix: SystemPathBuf::from(prefix), + }) } // ── build ──────────────────────────────────────────────────────────────────── @@ -1913,6 +1989,7 @@ fn write_traceback_runtime( None, BY_PYTEST_PLUGIN_SRC, )?; + staging.write(Path::new(BY_REPL_FILENAME), None, BY_REPL_SRC)?; for (name, text) in BY_PYTEST_DISTRIBUTION { staging.write( &Path::new(BY_PYTEST_DISTRIBUTION_DIR).join(name), @@ -1923,6 +2000,174 @@ fn write_traceback_runtime( Ok(()) } +/// the python half of a `by repl` session, which `by repl` runs in the tree it staged +pub(crate) const BY_REPL_FILENAME: &str = "_by_repl.py"; + +const BY_REPL_SRC: &str = r#"# generated by `by repl` — the python half of a session, see `by_repl.rs` +import ast +import json +import linecache +import os +import signal +import socket +import sys +import types + +import _by_runner + +# what a terminal sends around a paste, and an editor around an input it sends whole +_PASTE_START = "\x1b[200~" +_PASTE_END = "\x1b[201~" + +_INTERACTIVE = sys.stdin.isatty() + + +class _Session: + """The connection to `by repl`, which says whether an input is whole and what it runs as.""" + + def __init__(self): + port = int(os.environ.pop("BY_REPL_PORT")) + token = os.environ.pop("BY_REPL_TOKEN") + self._socket = socket.create_connection(("127.0.0.1", port)) + self._reader = self._socket.makefile("r", encoding="utf-8", newline="\n") + self.tell({"token": token}) + + def tell(self, message): + self._socket.sendall((json.dumps(message) + "\n").encode("utf-8")) + + def ask(self, message): + # a ctrl-c that arrives while the answer is on its way is held until it has arrived, + # so the next question is not answered with this one's answer + interrupted = [] + previous = signal.signal(signal.SIGINT, lambda *_: interrupted.append(True)) + try: + self.tell(message) + line = self._reader.readline() + finally: + signal.signal(signal.SIGINT, previous) + if not line: + raise EOFError + if interrupted: + raise KeyboardInterrupt + return json.loads(line) + + +def _read(prompt): + # what the last input printed is out before anything else is read, as it is on a terminal, + # wherever the streams go + sys.stdout.flush() + sys.stderr.flush() + if _INTERACTIVE: + return input(prompt) + "\n" + line = sys.stdin.readline() + if not line: + raise EOFError + return line + + +def _read_input(session): + """The next input: lines until `by repl` says they are whole, or a framed block.""" + line = _read(">>> ") + if line.startswith(_PASTE_START): + text = line[len(_PASTE_START):] + while _PASTE_END not in text: + text += _read("") + return text[: text.index(_PASTE_END)] + text = line + while session.ask({"status": text})["status"] == "incomplete": + text += _read("... ") + return text + + +def _run(session, namespace, text): + reply = session.ask({"run": text}) + if "python" not in reply: + # refused: `by repl` has said why + return + number = reply["input"] + filename = f"" + lines = text.splitlines(True) + # what a traceback quotes for a line of the input, for as long as the session lasts + linecache.cache[filename] = (len(text), None, lines, filename) + generated = reply["lines"] + widths = [len(line.rstrip("\r\n")) for line in lines] + + def by_line(line): + mapped = generated[line - 1] if line is not None and 0 < line <= len(generated) else None + return 0 if mapped is None else mapped + 1 + + def width(line): + return widths[line - 1] if 0 < line <= len(widths) else 0 + + module = ast.parse(reply["python"]) + body = [module.body[index] for index in reply["statements"]] + shown = None + if reply["display"] and body and isinstance(body[-1], ast.Expr): + shown = body.pop() + try: + codes = [] + if body: + codes.append( + compile(ast.Module(body=body, type_ignores=[]), filename, "exec", dont_inherit=True) + ) + if shown is not None: + # as python's repl compiles an input, which prints an expression's value + codes.append( + compile(ast.Interactive(body=[shown]), filename, "single", dont_inherit=True) + ) + codes = [_by_runner._relined(code, filename, by_line, width) for code in codes] + except Exception: + sys.stderr.write("error: the python this input was transpiled to does not compile:\n") + sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) + return + session.tell({"ran": number}) + try: + for code in codes: + exec(code, namespace) + except SystemExit: + raise + except BaseException: + sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) + + +def main(): + session = _Session() + sys.meta_path.insert(0, _by_runner._ByFinder()) + sys.excepthook = _by_runner._excepthook + sys.argv = [""] + main_module = types.ModuleType("__main__") + sys.modules["__main__"] = main_module + namespace = main_module.__dict__ + if _INTERACTIVE: + try: + import readline # noqa: F401 — line editing for `input` + except ImportError: + pass + while True: + try: + text = _read_input(session) + except EOFError: + if _INTERACTIVE: + sys.stdout.write("\n") + return + except KeyboardInterrupt: + sys.stderr.write("\nKeyboardInterrupt\n") + continue + if not text.strip(): + continue + try: + _run(session, namespace, text) + except KeyboardInterrupt: + sys.stderr.write("\nKeyboardInterrupt\n") + except EOFError: + # `by repl` has gone, and with it the session + return + + +if __name__ == "__main__": + main() +"#; + /// the pytest plugin that moves a test module's functions to the `.by` it was /// transpiled from, so that a failure is reported at the `.by` line that failed const BY_PYTEST_PLUGIN_FILENAME: &str = "_by_pytest.py"; @@ -2362,6 +2607,8 @@ _BY_SOURCES = {os.path.realpath(info[0]): py for py, info in SOURCEMAP.items()} # this file, whose frames are never the program's _RUNNER = os.path.realpath(__file__) +# the start of the name `by repl` compiles an input under, `` +_SESSION_INPUT = ", + lowering: &LoweringArgs, + python_flag: Option<&Path>, + session_file: Option<&Path>, +) -> anyhow::Result { + let staged = match stage_run( + min_version, + lowering, + python_flag, + false, + RunTarget::Session, + )? { + Ok(staged) => staged, + Err(status) => return Ok(status), + }; + let StagedRun { + cwd, + interpreter, + config, + tree, + rebuilder, + .. + } = staged; + let session_file = session_file.map(|path| cwd.join(path)); + if let Some(path) = &session_file { + fs::write(path, "").with_context(|| format!("could not write {}", path.display()))?; + } + let mut repl = Repl::new(&rebuilder, config, session_file)?; + + let listener = + TcpListener::bind(("127.0.0.1", 0)).context("could not listen for the session")?; + let port = listener.local_addr()?.port(); + let token = token(); + let mut child = Command::new(&interpreter.path) + .arg(tree.path().join(BY_REPL_FILENAME)) + .env("BY_REPL_PORT", port.to_string()) + .env("BY_REPL_TOKEN", &token) + .current_dir(&cwd) + .spawn() + .with_context(|| { + format!( + "could not run `{}`, the interpreter from {}", + interpreter.path, interpreter.origin + ) + })?; + // ctrl-c is the interpreter's to answer, with a `KeyboardInterrupt` in the input it is + // running; this process lives as long as the session does + ctrlc::set_handler(|| {}).context("could not hand ctrl-c to the session")?; + + let served = accept(&listener, &token, &mut child).and_then(|reader| match reader { + Some(reader) => repl.serve(reader), + None => Ok(()), + }); + if let Err(error) = &served { + eprintln!("error: the session ended: {error:#}"); + // the interpreter is left with no one to answer it + let _ = child.kill(); + } + let status = child + .wait() + .context("the session's interpreter could not be waited for")?; + drop(repl); + // `process::exit` skips destructors, and the tree is a temporary directory + drop(tree); + std::process::exit(status.code().unwrap_or(1)); +} + +/// A token nothing else can guess, from the process's own random hasher keys. +fn token() -> String { + let state = RandomState::new(); + format!( + "{:016x}{:016x}", + state.hash_one(std::process::id()), + state.hash_one(Instant::now()) + ) +} + +/// The interpreter's connection, read through the one buffer every message after the token +/// is read through, once it has sent `token`. `None` when the interpreter ended without +/// making one. +fn accept( + listener: &TcpListener, + token: &str, + child: &mut std::process::Child, +) -> anyhow::Result>> { + listener.set_nonblocking(true)?; + let started = Instant::now(); + loop { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false)?; + stream.set_read_timeout(Some(CONNECT_TIMEOUT))?; + let mut reader = BufReader::new(stream); + let mut first = String::new(); + // a connection that is not the interpreter's is dropped, and the wait goes on + if reader.read_line(&mut first).is_ok() + && serde_json::from_str::(&first) + .is_ok_and(|message| message["token"] == token) + { + reader.get_ref().set_read_timeout(None)?; + return Ok(Some(reader)); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if child.try_wait()?.is_some() { + return Ok(None); + } + anyhow::ensure!( + started.elapsed() < CONNECT_TIMEOUT, + "the interpreter did not connect to the session" + ); + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(error).context("could not accept the session's interpreter"), + } + } +} + +/// The session as this process holds it: the module, the db that checks it, and what each +/// input that ran was. +struct Repl<'a> { + db: ProjectDatabase, + rebuilder: &'a Rebuilder, + /// what the module is transpiled for: the project's run configuration, as a session + config: Config, + session: Session, + /// the file the module is checked as, outside the project + module: File, + module_path: PathBuf, + /// keeps the module's directory as long as the session + _module_dir: tempfile::TempDir, + /// each input that ran, for a diagnostic of a later one to point into + inputs: Vec, + /// the input handed over to run, which is part of the session once the interpreter says it + /// has started running it + pending: Option, + /// the number the next input is named by + next: u32, + /// `--session-file`: where an editor reads the session's source + session_file: Option, +} + +struct RanInput { + /// where in the session's source the input starts + start: TextSize, + file: SourceFile, +} + +struct Pending { + number: u32, + text: String, + selection: by_stage::session::Selection, +} + +impl<'a> Repl<'a> { + fn new( + rebuilder: &'a Rebuilder, + config: Config, + session_file: Option, + ) -> anyhow::Result { + let mut db = rebuilder.project_database(); + let module_dir = tempfile::TempDir::new().context("failed to create temp directory")?; + // named for what it is in a message that cannot help naming it + let module_path = module_dir.path().join("session.by"); + fs::write(&module_path, "")?; + let system_path = SystemPath::from_std_path(&module_path) + .with_context(|| format!("non-utf8 path: {}", module_path.display()))?; + let module = system_path_to_file(&db, system_path) + .with_context(|| format!("could not read {}", module_path.display()))?; + // outside the project, so checked only as a file the project has open + db.project().open_file(&mut db, module); + Ok(Self { + db, + rebuilder, + config: Config { + session: true, + // a `from` import binds its names when it is entered, so a name that cannot be + // imported says so then, as it does in python's repl. deferred, it would be + // bound through a holder that each later input's transpile writes afresh + defer_from_imports: false, + runtime_module: None, + ..config + }, + session: Session::default(), + module, + module_path, + _module_dir: module_dir, + inputs: Vec::new(), + pending: None, + next: 1, + session_file, + }) + } + + /// Answer the interpreter until it hangs up. + fn serve(&mut self, mut reader: BufReader) -> anyhow::Result<()> { + let mut writer = reader.get_ref().try_clone()?; + let mut line = String::new(); + loop { + line.clear(); + if reader.read_line(&mut line)? == 0 { + return Ok(()); + } + let message: Value = serde_json::from_str(&line).context("an unreadable message")?; + let reply = if let Some(text) = message["status"].as_str() { + Some(json!({ "status": input_status(text) })) + } else if let Some(text) = message["run"].as_str() { + Some(self.run(text)?) + } else if let Some(number) = message["ran"].as_u64() { + self.ran(number)?; + None + } else { + anyhow::bail!("an unknown message: {line}"); + }; + if let Some(reply) = reply { + writeln!(writer, "{reply}")?; + writer.flush()?; + } + } + } + + /// What `text` runs as, or what is in the way of it running, said on standard error. + fn run(&mut self, text: &str) -> anyhow::Result { + let number = self.next; + self.next += 1; + self.pending = None; + let refused = json!({ "input": number }); + let input_file = SourceFileBuilder::new(format!(""), text).finish(); + + // the input's own grammar first: a line that continues an earlier input, `else:` after + // an `if`, would otherwise parse as part of it + let parsed = parse_unchecked_source(text, PySourceType::BasedPython); + if !parsed.errors().is_empty() { + let diagnostics: Vec = parsed + .errors() + .iter() + .map(|error| { + Diagnostic::invalid_syntax(input_file.clone(), &error.error, error.location) + }) + .collect(); + self.render(&diagnostics); + return Ok(refused); + } + let display = matches!(parsed.suite().last(), Some(Stmt::Expr(_))); + + let (source, lines) = self.session.with_input(text); + fs::write(&self.module_path, &source) + .with_context(|| format!("could not write {}", self.module_path.display()))?; + File::sync_path( + &mut self.db, + SystemPath::from_std_path(&self.module_path).context("non-utf8 path")?, + ); + let start = TextSize::try_from(self.session.source().len())?; + let errors: Vec = + ty_project::check_file_ignoring_check_mode(&self.db, self.module) + .into_iter() + .filter(|diagnostic| diagnostic.severity() >= Severity::Error) + .map(|diagnostic| self.in_inputs(diagnostic, start, &input_file)) + .collect(); + if !errors.is_empty() { + self.render(&errors); + return Ok(refused); + } + + let rebuild = || Some(self.rebuilder.rebuild()); + let transpiled = ty_project::catch_panic( + &self.db, + self.module, + PanicActivity::Transpiling, + AssertUnwindSafe(|| { + by_transforms::transpile_typed_with_report( + &self.db, + self.module, + &self.config, + Some(&rebuild), + ) + }), + ); + let (python, line_map) = match transpiled { + Ok(Ok((python, line_map, _))) => (python, line_map), + Ok(Err(error)) => { + let diagnostic = by_stage::emit::transpile_failure_diagnostic(self.module, &error); + let diagnostic = self.in_inputs(diagnostic, start, &input_file); + self.render(&[diagnostic]); + return Ok(refused); + } + Err(panic) => { + self.render(&[panic]); + return Ok(refused); + } + }; + let selection = match self.session.select(&python, &line_map, lines) { + Ok(selection) => selection, + Err(refusal) => { + report_refusal(&refusal); + return Ok(refused); + } + }; + let reply = json!({ + "input": number, + "python": python, + "statements": selection.statements, + "lines": selection.lines, + "display": display, + }); + self.pending = Some(Pending { + number, + text: text.to_owned(), + selection, + }); + Ok(reply) + } + + /// The interpreter has started running input `number`, which is part of the session from + /// now on, whatever it raises. + fn ran(&mut self, number: u64) -> anyhow::Result<()> { + let Some(pending) = self.pending.take() else { + return Ok(()); + }; + if u64::from(pending.number) != number { + return Ok(()); + } + let start = TextSize::try_from(self.session.source().len())?; + let file = SourceFileBuilder::new(format!("", pending.number), &*pending.text) + .finish(); + self.inputs.push(RanInput { start, file }); + self.session + .commit(pending.number, &pending.text, pending.selection); + if let Some(path) = &self.session_file { + fs::write(path, self.session.source()) + .with_context(|| format!("could not write {}", path.display()))?; + } + Ok(()) + } + + /// `diagnostic` with every place it points to in the session's module moved to the input + /// that place is in, `input` being the one at `start` that has not run yet. + fn in_inputs( + &self, + mut diagnostic: Diagnostic, + start: TextSize, + input: &SourceFile, + ) -> Diagnostic { + let module = self.module; + let place = |range: TextRange| -> (SourceFile, TextRange) { + let (file, at) = if range.start() >= start { + (input.clone(), start) + } else { + let ran = self + .inputs + .iter() + .rev() + .find(|ran| ran.start <= range.start()) + .or_else(|| self.inputs.first()); + match ran { + Some(ran) => (ran.file.clone(), ran.start), + None => (input.clone(), start), + } + }; + let length = TextSize::try_from(file.source_text().len()).unwrap_or_default(); + let moved = TextRange::new( + (range.start() - at.min(range.start())).min(length), + (range.end().max(at) - at).min(length), + ); + (file, moved) + }; + let move_span = |span: &Span| -> Option { + match span.file() { + UnifiedFile::Ty(file) if *file == module => { + let range = span.range().unwrap_or_default(); + let (file, range) = place(range); + Some(Span::from(file).with_range(range)) + } + _ => None, + } + }; + for annotation in diagnostic.annotations_mut() { + if let Some(span) = move_span(annotation.get_span()) { + annotation.set_span(span); + } + } + for sub in diagnostic.sub_diagnostics_mut() { + for annotation in sub.annotations_mut() { + if let Some(span) = move_span(annotation.get_span()) { + annotation.set_span(span); + } + } + } + diagnostic + } + + /// Print `diagnostics` on standard error, as `by check` prints them. + #[allow(clippy::print_stderr)] + fn render(&self, diagnostics: &[Diagnostic]) { + let config = DisplayDiagnosticConfig::new("ty") + .color(colored::control::SHOULD_COLORIZE.should_colorize()) + .context(0); + let resolver = Inputs(&self.db); + eprint!( + "{}", + FullRenderer::new(&resolver, &config).display(diagnostics) + ); + } +} + +/// Say why an input's python cannot run. +#[allow(clippy::print_stderr)] +fn report_refusal(refusal: &Refusal) { + match refusal { + Refusal::ChangesAnEarlierInput { input, line } => eprintln!( + "error: this input changes the python that , line {line} was \ + transpiled to, which has run already and cannot run again differently" + ), + Refusal::JoinsAnEarlierInput { input } => eprintln!( + "error: this input is transpiled into one statement with , \ + which has run already" + ), + Refusal::NotPython(error) => { + eprintln!("error: the python this input was transpiled to does not parse: {error}"); + } + } +} + +/// The project's db as the diagnostics of a session read their files: the project's files +/// through it, and the inputs, which are no file of its, as the text each diagnostic carries. +struct Inputs<'db>(&'db ProjectDatabase); + +impl FileResolver for Inputs<'_> { + fn path(&self, file: ruff_db::files::File) -> &str { + FileResolver::path(self.0, file) + } + + fn input(&self, file: ruff_db::files::File) -> Input { + FileResolver::input(self.0, file) + } + + fn notebook_index(&self, file: &UnifiedFile) -> Option { + match file { + UnifiedFile::Ty(_) => FileResolver::notebook_index(self.0, file), + UnifiedFile::Ruff(_) => None, + } + } + + fn is_notebook(&self, file: &UnifiedFile) -> bool { + match file { + UnifiedFile::Ty(_) => FileResolver::is_notebook(self.0, file), + UnifiedFile::Ruff(_) => false, + } + } + + fn current_directory(&self) -> &Path { + FileResolver::current_directory(self.0) + } +} diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index 74b7e320d0..d460d5c5b4 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -4,6 +4,7 @@ mod by_coverage; mod by_init; mod by_lowering; mod by_project_server; +mod by_repl; mod by_source_encoding; mod by_stamps; mod by_wheels; @@ -142,6 +143,17 @@ fn run_command(command: Command) -> anyhow::Result { profile: profile.as_deref(), }, ), + Command::Repl { + min_version, + python, + lowering, + session_file, + } => by_repl::cmd_repl( + min_version.as_deref(), + &lowering, + python.as_deref(), + session_file.as_deref(), + ), Command::Init { path, name, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 0dbb19f7a5..cade18415c 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -979,6 +979,97 @@ fn run_checks_against_the_python_it_runs_on() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); } +/// the environment a program is checked against is the one the interpreter `--python` names +/// says it runs in, not one guessed from where the file is: a launcher script that runs a +/// virtual environment's python is checked against that environment +#[cfg(unix)] +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run_checks_against_the_environment_a_launcher_runs_in() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().expect("tempdir"); + let environment = dir.path().join("env"); + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let made = Command::new(&python) + .args(["-m", "venv", "--without-pip"]) + .arg(&environment) + .status() + .is_ok_and(|status| status.success()); + if !made { + eprintln!("skipping: {python} cannot make a virtual environment"); + return; + } + // where the environment keeps its packages, which is `lib/pythonX.Y/site-packages` on unix + // and `Lib/site-packages` on windows + let purelib = Command::new(&python) + .args([ + "-c", + "import sys, sysconfig\n\ + print(sysconfig.get_path('purelib', 'venv', vars={'base': sys.argv[1], 'platbase': sys.argv[1]}))", + ]) + .arg(&environment) + .output() + .expect("failed to spawn python"); + if !purelib.status.success() { + eprintln!( + "skipping: {python} cannot say where a virtual environment keeps its packages:\n{}", + String::from_utf8_lossy(&purelib.stderr) + ); + return; + } + let site_packages = PathBuf::from(String::from_utf8_lossy(&purelib.stdout).trim()); + fs::create_dir_all(site_packages.join("fancypkg")).unwrap(); + fs::write(site_packages.join("fancypkg/__init__.py"), "VALUE = 42\n").unwrap(); + let launcher = dir.path().join("launcher").join("python3"); + fs::create_dir_all(launcher.parent().unwrap()).unwrap(); + fs::write( + &launcher, + format!( + "#!/bin/sh\nexec '{}' \"$@\"\n", + environment.join("bin").join("python").display() + ), + ) + .unwrap(); + fs::set_permissions(&launcher, fs::Permissions::from_mode(0o755)).unwrap(); + + let project = dir.path().join("project"); + fs::create_dir_all(&project).unwrap(); + fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"project\"\nversion = \"0\"\n[tool.basedpython]\n", + ) + .unwrap(); + fs::write( + project.join("main.by"), + "import fancypkg\n\nprint(fancypkg.VALUE)\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env_remove("VIRTUAL_ENV") + .env_remove("CONDA_PREFIX") + .env_remove("PYTHON") + .arg("run") + .arg("--python") + .arg(&launcher) + .arg("main") + .current_dir(&project) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + /// the program runs out of a directory of transpiled copies that is deleted when /// the run ends, so every path python derives from a module's origin named a /// temporary file — leaving a tool started anywhere but the project root with @@ -1805,6 +1896,164 @@ fn run_profile_maps_a_module_a_test_runner_compiled() { ); } +/// What a `by repl` session in `dir` printed, and how it ended, given `input` on its standard +/// input, which is not a terminal: no prompts are written. +fn repl_session(dir: &Path, input: &str, extra: &[&str]) -> (Option, String, String) { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let mut child = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env("NO_COLOR", "1") + .args(["repl", "--python", &python]) + .args(extra) + .current_dir(dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn by"); + child + .stdin + .take() + .expect("stdin") + .write_all(input.as_bytes()) + .expect("the session reads its input"); + let output = child.wait_with_output().expect("the session ends"); + // python writes its output in text mode, which ends a line with `\r\n` on windows + ( + output.status.code(), + String::from_utf8_lossy(&output.stdout).replace("\r\n", "\n"), + String::from_utf8_lossy(&output.stderr).replace("\r\n", "\n"), + ) +} + +/// A project with a `.by` module for a session to import. +fn project_with_a_module(dir: &Path) { + let package = dir.join("pkg"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.py"), "").unwrap(); + fs::write( + package.join("calc.by"), + "def fib(n: int) -> int:\n if n < 2:\n return n\n return fib(n - 1) + fib(n - 2)\n\n\ndef ratio(a: int, b: int) -> float:\n return a / b\n", + ) + .unwrap(); +} + +/// each input is checked as the continuation of the inputs before it: a name defined earlier +/// is defined, a `let` stays read-only, and an expression's value is printed +#[test] +fn repl_checks_each_input_as_the_session_s_continuation() { + let dir = tempfile::tempdir().expect("tempdir"); + project_with_a_module(dir.path()); + let (status, stdout, stderr) = repl_session( + dir.path(), + "let rate = 0.2\n\ + def taxed(amount: float) -> float:\n return amount * (1 + rate)\n\n\ + taxed(10.0)\n\ + rate = 0.3\n\ + rate\n\ + from pkg.calc import fib\n\ + fib(10)\n\ + def main():\n print(\"main ran\")\n\n\ + print('done')\n", + &[], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!(stdout, "12.0\n0.2\n55\ndone\n", "{stderr}"); + // the refused input is reported against itself, and the declaration it breaks against the + // input that made it + assert!( + stderr.contains("error[invalid-assignment]") + && stderr.contains("--> :1:1") + && stderr.contains("::: :1:1"), + "{stderr}" + ); +} + +/// an input that raises prints a traceback whose frames are the inputs and the project's +/// modules, at their own lines, and the session goes on +#[test] +fn repl_reports_an_exception_at_the_inputs_lines() { + let dir = tempfile::tempdir().expect("tempdir"); + project_with_a_module(dir.path()); + let (status, stdout, stderr) = repl_session( + dir.path(), + "from pkg.calc import ratio\n\ + def share(total: int, parts: int) -> float:\n return ratio(total, parts)\n\n\ + share(1, 0)\n\ + 'still here'\n", + &[], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!(stdout, "'still here'\n", "{stderr}"); + let calc = dir + .path() + .canonicalize() + .unwrap() + .join("pkg/calc.by") + .display() + .to_string(); + assert!( + stderr.contains(" File \"\", line 1, in \n share(1, 0)\n"), + "{stderr}" + ); + assert!( + stderr.contains( + " File \"\", line 2, in share\n return ratio(total, parts)\n" + ), + "{stderr}" + ); + assert!( + stderr.contains(&format!( + " File \"{calc}\", line 8, in ratio\n return a / b\n" + )), + "{stderr}" + ); + assert!( + stderr.contains("ZeroDivisionError: division by zero"), + "{stderr}" + ); + assert!(!stderr.contains("_by_repl.py"), "{stderr}"); +} + +/// an input sent framed, as a terminal frames a paste, is one input whatever blank lines it +/// holds; the lines after it that the program reads with `input()` are the program's; and +/// the session's exit status is what `sys.exit` was given +#[test] +fn repl_takes_a_framed_input_whole_and_hands_input_lines_to_the_program() { + let dir = tempfile::tempdir().expect("tempdir"); + let (status, stdout, stderr) = repl_session( + dir.path(), + "\x1b[200~def total() -> int:\n first = 1\n\n return first + 1\n\x1b[201~\n\ + total()\n\ + name = input()\n\ + basedpython\n\ + name\n\ + import sys\n\ + sys.exit(3)\n\ + 'never'\n", + &[], + ); + assert_eq!(status, Some(3), "{stderr}"); + assert_eq!(stdout, "2\n'basedpython'\n", "{stderr}"); +} + +/// `--session-file` holds the inputs that ran, and not one the checker refused, for an +/// editor's console to complete against +#[test] +fn repl_session_file_holds_the_inputs_that_ran() { + let dir = tempfile::tempdir().expect("tempdir"); + let (status, _, stderr) = repl_session( + dir.path(), + "let rate = 0.2\nrate = 0.3\nx = rate * 2\n", + &["--session-file", "session.by"], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!( + fs::read_to_string(dir.path().join("session.by")).expect("the session file"), + "let rate = 0.2\nx = rate * 2\n" + ); +} + /// A project of `.by` tests for pytest to run, with a module they test, checked /// against the environment the tests run in: the `conftest` imports pytest, which /// `--python` does not configure. `None`, having said why, when `$PYTHON` (or diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 4b450a6d8e..e63e7cd501 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -14,6 +14,7 @@ in addition to the cli provided by `ty`, `by` includes: | command | what it does | | ------------------- | ------------------------------------------------------------------ | | `run` | transpile and run a module with `python -m ` | +| `repl` | an interactive session in basedpython | | `build` | transpile every `.by`/`.byi` file and write to `build/` | | `compile` | compile `.by` files to native CPython extension modules | | `generate-api-file` | write a public-api lockfile (see [api-lock](features/api-lock.md)) | @@ -172,6 +173,19 @@ hidden directories (`.claude`, `.git`, `.venv`, …) and build outputs are never treated as project source: they are neither checked nor transpiled, by `run` and `build` alike +## `by repl` + +```sh +by repl # an interactive session, on the project's interpreter +by repl --python python3.14 # on another interpreter +``` + +an interactive session in which you type basedpython, each input checked and transpiled as the +continuation of the inputs before it and run in one python process. it starts where `by run` +would, with the project staged as `by run` stages it, so the project's modules import as they +do in a program. `--session-file PATH` keeps `PATH` holding the inputs that ran, for an editor's +console to complete against. see [the repl](features/repl.md) + ## `by build` ```sh diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index 716f2178e1..8f59d9feec 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -1,8 +1,5 @@ # the repl -> **status: design.** `by/inputStatus` is built; `by repl` itself is being built, and lands in -> the commits that follow - `by repl` is an interactive session in which you type basedpython. each input is checked and transpiled with everything the session has defined so far, and runs in one python process that lives as long as the session @@ -13,11 +10,19 @@ $ by repl >>> def taxed(amount: float) -> float: ... return amount * (1 + rate) ... ->>> taxed(10) +>>> taxed(10.0) 12.0 >>> rate = 0.3 - --> :1:1 -error[invalid-assignment]: Reassignment of `Final` symbol `rate` is not allowed +error[invalid-assignment]: read-only symbol `rate` cannot be reassigned + --> :1:1 + | +1 | rate = 0.3 + | ^^^^^^^^^^ reassigned here + | + ::: :1:1 + | +1 | let rate = 0.2 + | ---- declared read-only here ``` it starts where `by run` would: in the project around the working directory, on the @@ -49,14 +54,15 @@ any runtime helper they need that has not run yet. the earlier inputs' statement they were entered. an input that would change the python an earlier input was transpiled to — which has already run, and cannot run again differently — is refused, and says so -two things differ from `by run`, because a session is never run as a whole. an `import` runs -when it is entered rather than lazily, so a module that cannot be imported says so then, and a -`def main` is a function like any other: nothing calls it +two things differ from `by run`, because a session is never run as a whole. a `from` import +binds its names when it is entered rather than at their first use, so a name that cannot be +imported says so then, and a `def main` is a function like any other: nothing calls it ## what is printed -an input that ends in an expression prints its value, through `sys.displayhook`, as python's -repl prints it. an exception prints a traceback whose frames name the +each expression statement of an input prints its value, through `sys.displayhook`, as python's +repl prints it: both of `1; 2`, and one in a loop's body, but not one in a function's. an input is numbered whether it runs or not, as python numbers its own. an +exception prints a traceback whose frames name the input they ran in, at the input's line, with the line quoted, and the project's modules at their `.by` lines @@ -68,6 +74,7 @@ their `.by` lines Traceback (most recent call last): File "", line 1, in ratio(1, 0) + ~~~~~^^^^^^ File "", line 2, in ratio return a / b ZeroDivisionError: division by zero @@ -82,7 +89,7 @@ it. whether an input is complete is a question about basedpython's grammar, so ` — the same answer the [language server](#in-an-editor) gives the end of input (ctrl-d) ends the session, and its exit status is the program's: 0, or what -`sys.exit` was given. ctrl-c abandons the input being typed, or interrupts the one running with +`sys.exit` was given, or, when a signal ends the interpreter, 128 and the signal's number. ctrl-c abandons the input being typed, or interrupts the one running with a `KeyboardInterrupt`, which is printed, and the session carries on an input can also be sent whole, framed as a terminal frames a paste: `ESC [200~` before it and From e09ef2be43cc6ddd7b1c4e71deacf11e5249579f Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:38:32 +1000 Subject: [PATCH 075/110] check that a console completes against a document holding a by repl session's source followed by the input being typed, the contract docs/basedpython/features/repl.md gives an editor for completion in a console --- crates/ty_server/tests/e2e/input_status.rs | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/ty_server/tests/e2e/input_status.rs b/crates/ty_server/tests/e2e/input_status.rs index 60f55b78ef..58424bbf6a 100644 --- a/crates/ty_server/tests/e2e/input_status.rs +++ b/crates/ty_server/tests/e2e/input_status.rs @@ -57,3 +57,34 @@ fn input_status_says_whether_enter_runs_the_input() -> anyhow::Result<()> { } Ok(()) } + +/// a console completes against a document holding the session's source, as `by repl +/// --session-file` keeps it, followed by the input being typed: what the session defined is +/// offered, `let` and all, though no file holds it +#[test] +fn a_console_completes_against_the_session_s_source() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .build() + .wait_until_workspaces_are_initialized(); + let session = + "let rate = 0.2\ndef taxed(amount: float) -> float:\n return amount * (1 + rate)\n"; + let typed = "tax"; + let uri = lsp_types::Uri::parse("untitled:console-1.by")?; + server.send_notification::( + lsp_types::DidOpenTextDocumentParams { + text_document: lsp_types::TextDocumentItem { + uri: uri.clone(), + language_id: lsp_types::LanguageKind::Python, + version: 1, + text: format!("{session}{typed}"), + }, + }, + ); + + let completions = server.completion_request(&uri, lsp_types::Position::new(3, 3)); + assert!( + completions.iter().any(|item| item.label == "taxed"), + "{completions:#?}" + ); + Ok(()) +} From dd648193b00bd80dc454dc89dff52fb8c8d7f2d7 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:43:43 +1000 Subject: [PATCH 076/110] check a by repl session's module as the project's own first-party code, so a sealed class declared in one input can be subclassed in a later one, and count what the transpiler writes past an input's last line, a sealed class's subclass list among it, as the input's --- crates/by_stage/src/project.rs | 32 +++++++++++++++++++++++++++++++ crates/by_stage/src/session.rs | 5 ++++- crates/ty/src/by_repl.rs | 19 +++++++++++------- crates/ty/tests/by_e2e.rs | 17 ++++++++++++++++ crates/ty_project/src/metadata.rs | 11 +++++++++++ docs/basedpython/features/repl.md | 6 ++++-- 6 files changed, 80 insertions(+), 10 deletions(-) diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index aa97997e60..e15f1b39fe 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -101,6 +101,38 @@ impl Rebuilder { Box::new(self.project_database()) } + /// the same project with `extra` one of its first-party roots too, searched after its own + /// + /// a `by repl` session's module lives outside the project, and the checker reads a module + /// as the project's own code only from a first-party root: a `sealed` class declared in a + /// session can be subclassed in the session, as in any module of the project, only once the + /// session's module is first-party + #[must_use] + pub fn with_first_party_root(&self, extra: &SystemPath) -> Self { + let db = self.project_database(); + let mut roots: Vec = db + .project() + .program(&db) + .search_paths(&db) + .first_party_roots() + .map(RelativePathBuf::cli) + .collect(); + roots.push(RelativePathBuf::cli(extra)); + let mut metadata = self.metadata.clone(); + metadata.apply_override_options(Options { + environment: Some(EnvironmentOptions { + root: Some(roots), + ..EnvironmentOptions::default() + }), + ..Options::default() + }); + Self { + metadata, + root: self.root.clone(), + included: self.included.clone(), + } + } + /// The project's db built again, and not frozen: what a caller holds that goes on /// changing an input after the build, as a `by repl` session changes its module with /// every input. diff --git a/crates/by_stage/src/session.rs b/crates/by_stage/src/session.rs index 553fb7141e..5cf483fac5 100644 --- a/crates/by_stage/src/session.rs +++ b/crates/by_stage/src/session.rs @@ -158,8 +158,11 @@ pub struct InputLines { } impl InputLines { + /// whether `line` of the module is the input's. the input is the end of the module, so a + /// line past its last is the input's too: the transpiler writes what follows a statement, + /// such as a sealed class's list of its subclasses, against the line after it fn contains(self, line: u32) -> bool { - line >= self.first && line - self.first < self.count + line >= self.first } } diff --git a/crates/ty/src/by_repl.rs b/crates/ty/src/by_repl.rs index d9611ecc81..e83a932140 100644 --- a/crates/ty/src/by_repl.rs +++ b/crates/ty/src/by_repl.rs @@ -172,9 +172,10 @@ fn accept( /// The session as this process holds it: the module, the db that checks it, and what each /// input that ran was. -struct Repl<'a> { +struct Repl { db: ProjectDatabase, - rebuilder: &'a Rebuilder, + /// the project, with the module's directory one of its first-party roots + rebuilder: Rebuilder, /// what the module is transpiled for: the project's run configuration, as a session config: Config, session: Session, @@ -206,17 +207,21 @@ struct Pending { selection: by_stage::session::Selection, } -impl<'a> Repl<'a> { +impl Repl { fn new( - rebuilder: &'a Rebuilder, + rebuilder: &Rebuilder, config: Config, session_file: Option, ) -> anyhow::Result { - let mut db = rebuilder.project_database(); let module_dir = tempfile::TempDir::new().context("failed to create temp directory")?; - // named for what it is in a message that cannot help naming it - let module_path = module_dir.path().join("session.by"); + // a module name nothing a project imports is called, since the directory is searched + // for first-party modules after the project's own roots + let module_path = module_dir.path().join("__by_session__.by"); fs::write(&module_path, "")?; + let directory = SystemPath::from_std_path(module_dir.path()) + .with_context(|| format!("non-utf8 path: {}", module_dir.path().display()))?; + let rebuilder = rebuilder.with_first_party_root(directory); + let mut db = rebuilder.project_database(); let system_path = SystemPath::from_std_path(&module_path) .with_context(|| format!("non-utf8 path: {}", module_path.display()))?; let module = system_path_to_file(&db, system_path) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index cade18415c..5de5c73532 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2037,6 +2037,23 @@ fn repl_takes_a_framed_input_whole_and_hands_input_lines_to_the_program() { assert_eq!(stdout, "2\n'basedpython'\n", "{stderr}"); } +/// the session's module is the project's own code, so a `sealed` class declared in it can be +/// subclassed in a later input, as in any module of the project +#[test] +fn repl_s_module_is_first_party() { + let dir = tempfile::tempdir().expect("tempdir"); + project_with_a_module(dir.path()); + let (status, stdout, stderr) = repl_session( + dir.path(), + "\x1b[200~sealed class Shape:\n pass\n\x1b[201~\n\ + \x1b[200~class Circle(Shape):\n pass\n\x1b[201~\n\ + type(Circle()).__name__\n", + &[], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!(stdout, "'Circle'\n", "{stderr}"); +} + /// `--session-file` holds the inputs that ran, and not one the checker refused, for an /// editor's console to complete against #[test] diff --git a/crates/ty_project/src/metadata.rs b/crates/ty_project/src/metadata.rs index fe1d45a897..d4229aa738 100644 --- a/crates/ty_project/src/metadata.rs +++ b/crates/ty_project/src/metadata.rs @@ -456,6 +456,17 @@ impl ProjectMetadata { self.override_options = Some(Box::new(options)); } + /// adds highest-precedence options for this project over any previous overrides, which + /// still apply where `options` leave a setting unset + pub fn apply_override_options(&mut self, options: Options) { + if let Some(existing) = self.override_options.as_mut() { + let previous = std::mem::replace(existing.as_mut(), options); + existing.combine_with(previous); + } else { + self.override_options = Some(Box::new(options)); + } + } + pub(crate) fn uv_workspace(&self) -> &UvWorkspace { &self.uv_workspace } diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index 8f59d9feec..1b2e50c4e3 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -45,8 +45,10 @@ file. what this means in practice: far*, and an expression whose value is discarded is the value the repl prints - an input that raised is part of the session: the statements before the one that raised have run, and what they bound is bound, as in python's repl -- a name declared with a type cannot be declared again with another, since one module cannot - do that. a name assigned without a declaration can be assigned anything +- a name can be assigned again, declared again with another type, and a function or a class + defined again, as they can further down a module. a `let` cannot, since it is final +- the session's module is the project's own code, so a `sealed` class declared in one input + can be subclassed in a later one what runs is the python written for the input: the module made of the session and the input is transpiled as a whole, and the statements transpiled from the input's lines run, after From 42d22efe638dcaa85840cdc30fc80bd2fa10defb Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:54:29 +1000 Subject: [PATCH 077/110] resolve the call that ends an optional chain against the chain's present receiver where the call is a statement of its own or the value assigned to an undeclared name, so s?.upper() on a str? no longer reports call-non-callable: a standalone callee region now hands its chain links' present types back to the region that extends them, and the assignment path that infers a call without infer_call_expression treats the call as a link of the chain --- .../mdtest/basedpython_optional_chain.md | 14 ++++++++++ crates/ty_python_semantic/src/types/infer.rs | 27 +++++++++++++++++++ .../src/types/infer/builder.rs | 19 ++++++++++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_chain.md b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_chain.md index 8d1fd7efc8..ed31c9865d 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_optional_chain.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_optional_chain.md @@ -50,6 +50,20 @@ def f(c: C | None) -> None: reveal_type(c?.greet()) # revealed: str | None ``` +## a chain that ends in a call, as a statement or an assignment's value + +the call is still a link of the chain where nothing gives it a type context: a statement on its own +line, or the value assigned to a name nothing declares. + +```by +def f(s: str | None) -> None: + s?.upper() # no diagnostic + upper = s?.upper() + reveal_type(upper) # revealed: str | None + lower = s?.upper().lower() + reveal_type(lower) # revealed: str | None +``` + ## chain continues through a subscript ```by diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index e1d58ac598..c3ec7f8af3 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -2305,6 +2305,15 @@ struct ExpressionInferenceExtra<'db> { /// parameters to — see [`CallSolution`]. call_solutions: FxHashMap>, + /// basedpython `?.`: the type each optional-chain link in this region has when every + /// `?.` receiver in its chain is present + /// + /// a region can be rooted at a chain link that the region asking for it goes on to + /// extend — the callee `s?.upper` of a statement-level call `s?.upper()` is a standalone + /// expression — and that next link must be looked up against the present receiver, not + /// against the short-circuit `None` the link's own type carries + basedpython_chain_present: FrozenMap>, + /// The creation-time type of a fluid specialization candidate whose assigned value /// is this region, with literal types retained. fluid_creation: Option>, @@ -2406,6 +2415,14 @@ impl<'db> ExpressionInference<'db> { None => fluid_creation.recursive_type_normalized(db, env, cycle), }; } + for (link, present) in &mut extra.basedpython_chain_present { + *present = match previous.basedpython_chain_present(*link) { + Some(previous_present) => { + present.cycle_normalized(db, env, previous_present, cycle) + } + None => present.recursive_type_normalized(db, env, cycle), + }; + } if let Some(fluid_timeline) = extra.fluid_timeline.take() { extra.fluid_timeline = Some(fluid_timeline.cycle_normalized( db, @@ -2485,6 +2502,16 @@ impl<'db> ExpressionInference<'db> { .unwrap_or_else(Type::unknown) } + /// the present-receiver type of a basedpython optional-chain link in this region — see + /// [`ExpressionInferenceExtra::basedpython_chain_present`] + fn basedpython_chain_present(&self, link: ExpressionNodeKey) -> Option> { + self.extra + .as_deref()? + .basedpython_chain_present + .get(&link) + .copied() + } + /// Whether this call expression's type is `Never` only because the call left a type variable /// unsolved, rather than because the callee does not return. pub(crate) fn is_unsolved_typevar_call( diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 0aabaf9af8..0dfbc61bc7 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -977,6 +977,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.fluid_adoptions.extend(extra.fluid_adoptions.iter()); self.call_solutions.extend(extra.call_solutions.iter()); + + self.basedpython_chain_present + .extend(extra.basedpython_chain_present.iter().copied()); } } @@ -1027,6 +1030,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.fluid_adoptions .extend(inference.fluid_adoptions.iter()); self.call_solutions.extend(inference.call_solutions.iter()); + self.basedpython_chain_present + .extend(inference.basedpython_chain_present.iter()); if !matches!(self.region, InferenceRegion::Scope(..)) { self.bindings.extend( @@ -4614,6 +4619,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // (single target, no unpackings). That means it's a valid syntactic form // for a legacy TypeVar creation; check for that. let callable_type = self.infer_callee(&call_expr.func, TypeContext::default()); + // basedpython `x = a?.f()`: as in `infer_call_expression`, the call is a link + // of the chain, so it calls the present-receiver callable and the chain's + // `None` rides out to the call's own type + let (callable_type, in_chain) = + self.basedpython_chain_receiver(&call_expr.func, callable_type); let ty = if let Some(namedtuple_kind) = NamedTupleKind::from_type(self.db(), callable_type) @@ -4721,6 +4731,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // be kept symbolic here too — otherwise `a = x.foo()` would name a // different value than the `x.foo()` a `return` names let ty = self.basedpython_symbolic_call(call_expr, callable_type, ty); + let ty = self.basedpython_chain_result(call_expr, ty, in_chain); self.store_expression_type(value, ty); ty @@ -17467,9 +17478,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { deferred_decorator_calls: _, discards_dict_key_assignments: _, + basedpython_chain_present, + // builder only state expression_cache: _, - basedpython_chain_present: _, basedpython_statement_expression_values: _, reachability_cache: _, typevar_binding_context: _, @@ -17492,6 +17504,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { collection_use_constraints, fluid_adoptions, call_solutions, + basedpython_chain_present, fluid_creation, fluid_timeline, string_annotations, @@ -18266,6 +18279,7 @@ struct FullExpressionCacheEntry<'db> { collection_use_constraints: CollectionUseConstraints<'db>, fluid_adoptions: FxHashMap>, call_solutions: FxHashMap>, + basedpython_chain_present: FxHashMap>, fluid_creation: Option>, fluid_timeline: Option>, string_annotations: FxHashSet, @@ -18299,6 +18313,7 @@ impl<'db> FullExpressionCacheEntry<'db> { && self.collection_use_constraints.is_empty() && self.fluid_adoptions.is_empty() && self.call_solutions.is_empty() + && self.basedpython_chain_present.is_empty() && self.fluid_creation.is_none() && self.fluid_timeline.is_none() && self.string_annotations.is_empty() @@ -18324,6 +18339,7 @@ impl<'db> FullExpressionCacheEntry<'db> { || !self.collection_use_constraints.is_empty() || !self.fluid_adoptions.is_empty() || !self.call_solutions.is_empty() + || !self.basedpython_chain_present.is_empty() || self.fluid_creation.is_some() || self.fluid_timeline.is_some() || !self.expected_types.is_empty() @@ -18353,6 +18369,7 @@ impl<'db> FullExpressionCacheEntry<'db> { unsolved_typevar_calls: FrozenSet::from(self.unsolved_typevar_calls), fluid_adoptions: self.fluid_adoptions, call_solutions: self.call_solutions, + basedpython_chain_present: FrozenMap::from(self.basedpython_chain_present), fluid_creation: self.fluid_creation, fluid_timeline: self.fluid_timeline, comparison_truthiness: FrozenMap::from(self.comparison_truthiness), From ebb77fd0b74c5ab69935493c87007155ca63d047 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:59:59 +1000 Subject: [PATCH 078/110] insert a sealed class's __sealed_members__ assignment as a statement-anchored insert, so a statement directly after the last subclass with no blank line between, whose expression a soundness check wraps, no longer takes the assignment into the check's call and transpiles to invalid python --- .../by_transforms/src/transforms/modifiers.rs | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index c393f76421..e8593e1ed7 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -38,7 +38,7 @@ use ruff_python_ast::{ use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{AstPass, PassContext}; +use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; use crate::{Config, PythonVersion}; @@ -932,10 +932,14 @@ impl AstPass for ModifiersPass<'_> { let assignment = format!("{sealed}.__sealed_members__ = {tuple}"); // Insert at the start of the line after the anchor statement so the // edit never collides with `empty_declarations`' mid-line `: ...`. + // Without a blank line in between, that is where the next statement + // starts, so the insert is anchored to the statement boundary: a + // rewrite of that statement's first expression — a soundness check + // wrapping it — would otherwise absorb the assignment into itself. if let Some(rel) = self.source[anchor_end..].find('\n') { let pos = TextSize::try_from(anchor_end + rel + 1).expect("offset fits u32"); - ctx.text_edits - .push((TextRange::empty(pos), format!("{assignment}\n"))); + ctx.statement_inserts + .push((pos, vec![Fragment::Lit(format!("{assignment}\n"))])); } else { let pos = TextSize::try_from(self.source.len()).expect("offset fits u32"); ctx.text_edits @@ -2395,6 +2399,34 @@ mod tests { ); } + /// with no blank line after the last subclass, the members tuple is assigned + /// where the next statement starts, and a soundness check wrapping that + /// statement's expression keeps it out of the call it opens there + #[test] + fn sealed_members_ahead_of_a_checked_statement() { + let out = transpile( + indoc! {" + xs: list[str] = [] + sealed class A + class B(A) + xs[0] + "}, + &Config { + soundness: crate::SoundnessPositions::defaults(), + ..Config::test_default() + }, + ) + .unwrap(); + assert!( + out.contains(indoc! {" + class B(A): ... + A.__sealed_members__ = (B,) + _soundness_check(xs[0], str) + "}), + "got:\n{out}" + ); + } + #[test] fn sealed_class_no_members() { check( From 579576292cd6d098d4f9686418fe531efde6432a Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:05:32 +1000 Subject: [PATCH 079/110] make the by init test name the interpreter by run runs the new project on and target the project at that interpreter's version, rather than reading python3 off PATH while by run takes PYTHON, and say truly in PYTHON's documentation that the project's own environment wins over it and that it wins over python3 on PATH, for by init's default version as for by run --- crates/ty/docs/environment.md | 12 ++++++++---- crates/ty/tests/by_e2e.rs | 30 ++++++++++-------------------- crates/ty_static/src/env_vars.rs | 12 ++++++++---- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 331937ee31..4d072e8391 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -81,10 +81,14 @@ If both `VIRTUAL_ENV` and `CONDA_PREFIX` are present, `VIRTUAL_ENV` will be pref ### `PYTHON` -The interpreter `by run` executes the transpiled program on. - -Overrides the project environment, and is itself overridden by -`by run --python`. +the interpreter `by run` executes the transpiled program on, and the +version `by init` targets a new project at, where the project has no +environment of its own + +the project's environment — its `environment.python`, an activated +virtual or Conda environment, or a `.venv` beside its `pyproject.toml` — +wins over it, and it wins over a bare `python3` on `PATH`. the +`--python` of `by run` overrides both ### `PYTHONPATH` diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 5de5c73532..04e1ccf930 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -7555,25 +7555,6 @@ fn interpreter_version(python: &Path) -> (u8, u8) { (major, minor) } -/// the version of the interpreter `by run` would pick, so a test can name one -/// that is definitely newer. outside an environment that is `$PYTHON`, and -/// `python3` when nothing pins one -fn running_python_version() -> (u8, u8) { - let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); - let output = Command::new(python) - .args([ - "-c", - "import sys; print(f'{sys.version_info[0]} {sys.version_info[1]}')", - ]) - .output() - .expect("an interpreter is needed to run this test"); - let rendered = String::from_utf8_lossy(&output.stdout); - let mut parts = rendered.split_whitespace(); - let major = parts.next().unwrap().parse().unwrap(); - let minor = parts.next().unwrap().parse().unwrap(); - (major, minor) -} - /// the project environment is the environment the project *is* — `by check` /// resolved this project's imports against it, so running against a different /// python answers a question nobody asked @@ -8006,10 +7987,16 @@ fn build_does_not_carry_a_compilers_output_directory() { /// what `by init` writes has to be a project the rest of the toolchain accepts, /// or it is a template for a thing that does not work +/// +/// the new project has no environment of its own, so `by run` runs it on the +/// python `PYTHON` names. the test names that interpreter itself, and targets +/// the project at its version, rather than leaving an activated environment or +/// whichever `python3` comes first on `PATH` to decide either #[test] fn init_writes_a_project_that_builds_and_runs() { let dir = tempfile::tempdir().expect("tempdir"); - let (major, minor) = running_python_version(); + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let (major, minor) = interpreter_version(Path::new(&python)); let output = Command::new(env!("CARGO_BIN_EXE_by")) .env(EnvVars::BY_NO_PROJECT_SERVER, "1") @@ -8032,6 +8019,9 @@ fn init_writes_a_project_that_builds_and_runs() { let output = Command::new(env!("CARGO_BIN_EXE_by")) .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env_remove("VIRTUAL_ENV") + .env_remove("CONDA_PREFIX") + .env("PYTHON", &python) .arg("run") .current_dir(&project) .output() diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 668f7caf20..306e6cf7eb 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -98,10 +98,14 @@ impl EnvVars { /// Used to detect an activated virtual environment. pub const VIRTUAL_ENV: &'static str = "VIRTUAL_ENV"; - /// The interpreter `by run` executes the transpiled program on. - /// - /// Overrides the project environment, and is itself overridden by - /// `by run --python`. + /// the interpreter `by run` executes the transpiled program on, and the + /// version `by init` targets a new project at, where the project has no + /// environment of its own + /// + /// the project's environment — its `environment.python`, an activated + /// virtual or Conda environment, or a `.venv` beside its `pyproject.toml` — + /// wins over it, and it wins over a bare `python3` on `PATH`. the + /// `--python` of `by run` overrides both pub const PYTHON: &'static str = "PYTHON"; /// Adds additional directories to ty's search paths. From a3f76a42e3226a3d0fed8b470309293639afef8d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:21:02 +1000 Subject: [PATCH 080/110] run the stand-in uv once from the test before the server runs it in the tests of a check held for a script's environment, since macOS holds the first run of a just-written executable while it assesses the file, for over ten seconds while the ty suite runs new executables beside it, which is what wait_until_started timed out on --- crates/ty_server/tests/e2e/check_workspace.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/ty_server/tests/e2e/check_workspace.rs b/crates/ty_server/tests/e2e/check_workspace.rs index 390969c7a8..fe5cddd2e0 100644 --- a/crates/ty_server/tests/e2e/check_workspace.rs +++ b/crates/ty_server/tests/e2e/check_workspace.rs @@ -189,6 +189,7 @@ fn the_report_comes_whole_in_the_response() -> Result<()> { mod a_script_environment_being_set_up { use std::fs; use std::os::unix::fs::PermissionsExt; + use std::process::Command; use std::time::{Duration, Instant}; use anyhow::{Result, bail}; @@ -205,8 +206,10 @@ mod a_script_environment_being_set_up { use crate::{AwaitResponseError, TestServer, TestServerBuilder}; /// run beside the script, it says it has started, waits to be released, and fails, which - /// ends the setup with an environment that has an error rather than one that is pending + /// ends the setup with an environment that has an error rather than one that is pending. + /// asked for its version, it answers at once, which is how the test runs it first const UV: &str = "#!/bin/sh +[ \"$1\" = --version ] && exit 0 touch started i=0 while [ ! -e release ] && [ \"$i\" -lt 3000 ]; do @@ -262,6 +265,15 @@ exit 1 .with_file("bin/uv", UV)?; let uv = builder.file_path("bin/uv"); fs::set_permissions(uv.as_std_path(), fs::Permissions::from_mode(0o755))?; + // the first run of an executable that was just written can be held before it starts + // while the operating system assesses the file. macOS does, and holds it for longer than + // `wait_until_started` waits while other new executables are being run, as the rest of + // the test suite runs them. so the test runs it once itself, for as long as that takes, + // and what `wait_until_started` waits for is the server alone + let warmed = Command::new(uv.as_std_path()).arg("--version").status()?; + if !warmed.success() { + bail!("the stand-in `uv` did not run: {warmed}"); + } let setup = Setup { started: builder.file_path("src/started"), release: builder.file_path("src/release"), From de509b847ee55aa59565af88702bd55c9cd9dba3 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:36:54 +1000 Subject: [PATCH 081/110] report a \N{...} name in a regex that re would refuse on the python checked against as invalid-regex, undefined character name, looking it up as unicodedata.lookup does through unicode_names2, with the python each code point is first named in generated from every python's own unicodedata by scripts/generate_regex_character_names.py, and leaving to python the names derived from a code point, which pythons derive differently, and every name past python 3.14, whose unicode the table is newer than --- Cargo.lock | 1 + crates/basedpython/Cargo.lock | 1 + crates/ty_python_semantic/Cargo.toml | 1 + .../resources/mdtest/basedpython_regex.md | 68 ++++- crates/ty_python_semantic/src/types/regex.rs | 1 + .../src/types/regex/character_names.rs | 288 ++++++++++++++++++ .../src/types/regex/parse.rs | 218 ++++++++++++- docs/basedpython/features/regex-groups.md | 19 +- scripts/generate_regex_character_names.py | 244 +++++++++++++++ .../generate_regex_character_names.py.lock | 15 + 10 files changed, 839 insertions(+), 17 deletions(-) create mode 100644 crates/ty_python_semantic/src/types/regex/character_names.rs create mode 100644 scripts/generate_regex_character_names.py create mode 100644 scripts/generate_regex_character_names.py.lock diff --git a/Cargo.lock b/Cargo.lock index c1339a5b38..899bb64dd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5169,6 +5169,7 @@ dependencies = [ "ruff_python_trivia", "ruff_source_file", "ruff_text_size", + "ruff_unicode_names2", "rustc-hash", "salsa", "schemars", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index ab8884fef2..b658bfdafd 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -3886,6 +3886,7 @@ dependencies = [ "ruff_python_trivia", "ruff_source_file", "ruff_text_size", + "ruff_unicode_names2", "rustc-hash", "salsa", "serde", diff --git a/crates/ty_python_semantic/Cargo.toml b/crates/ty_python_semantic/Cargo.toml index e0912c5f88..aa4fc3309e 100644 --- a/crates/ty_python_semantic/Cargo.toml +++ b/crates/ty_python_semantic/Cargo.toml @@ -28,6 +28,7 @@ ruff_python_stdlib = { workspace = true } ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } +ruff_unicode_names2 = { workspace = true } ty_combine = { workspace = true } ty_module_resolver = { workspace = true } ty_site_packages = { workspace = true } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md index c6435425fd..1bc659dbfb 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_regex.md @@ -341,10 +341,14 @@ re.compile(r"\x4") # error: [invalid-regex] "incomplete escape \x4 at position re.compile(rb"\u0041") # error: [invalid-regex] "bad escape \u at position 0" re.compile(r"(a)\2") # error: [invalid-regex] "invalid group reference 2 at position 4" re.compile(r"(a\1)") # error: [invalid-regex] "cannot refer to an open group at position 2" +# error: [invalid-regex] "cannot refer to an open group at position 10" +re.compile(r"(?P(?P=a))") re.compile(r"\A\d\w+[\b\s]\u0041\N{EM DASH}\x41\0(a)\1\Z") # no diagnostic ``` +## an escape a newer python knows + `\z` is an anchor from python 3.14, and a bad escape before it. ```toml @@ -358,6 +362,62 @@ import re re.compile(r"a\z") # error: [invalid-regex] "bad escape \z at position 1" ``` +## character names + +`\N{...}` names a character, looked up the way `unicodedata.lookup` looks it up in the python +version checked against: case-insensitively, among the characters' names and their aliases. A name +that python's unicode database does not have is an error, and so is a named sequence, which stands +for several characters rather than one. + +```toml +[environment] +python-version = "3.13" +``` + +```py +import re + +re.compile(r"\N{em dash}\N{BYTE ORDER MARK}\N{LINE FEED}") # no diagnostic + +re.compile(r"a\N{EM DASH}") # error: [invalid-regex] "undefined character name 'EM DASH' at position 1" +# error: [invalid-regex] "undefined character name 'LATIN CAPITAL LETTER A WITH MACRON AND GRAVE' at position 0" +re.compile(r"\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}") +``` + +A character is named from the python whose unicode version assigns it: `ALIEN MONSTER OPEN JAWS` is +new in unicode 16.0, which python 3.14 ships, so python 3.13 does not know the name. + +```py +import re + +# error: [invalid-regex] "undefined character name 'ALIEN MONSTER OPEN JAWS' at position 0" +re.compile(r"\N{ALIEN MONSTER OPEN JAWS}") +``` + +An alias is known from the python whose unicode version adds it, which can be long after the +character it is for was named: `BAMUM LETTER PHASE-B NGGOM` is an alias from unicode 17.0, which +python 3.15 ships. + +```py +import re + +# error: [invalid-regex] "undefined character name 'BAMUM LETTER PHASE-B NGGOM' at position 0" +re.compile(r"\N{BAMUM LETTER PHASE-B NGGOM}") +``` + +## a character name a newer python knows + +```toml +[environment] +python-version = "3.14" +``` + +```py +import re + +re.compile(r"\N{ALIEN MONSTER OPEN JAWS}") # no diagnostic +``` + ## an invalid pattern's error points at the character it is about python names the position in the pattern where it gave up, which is the character the error is @@ -380,20 +440,20 @@ error[invalid-regex]: missing ), unterminated subpattern at position 2 the position is a character of the pattern the program holds, which is not always one character of the literal: an escape writes one character in several, and the error is about all of them. here -`\x28` writes the parenthesis left open, after a tab written as `\t`. +`\N{LEFT PARENTHESIS}` writes the parenthesis left open, after a tab written as `\t`. ```py import re -re.compile("\t\x28") # snapshot +re.compile("\t\N{LEFT PARENTHESIS}") # snapshot ``` ```snapshot error[invalid-regex]: missing ), unterminated subpattern at position 1 --> src/mdtest_snippet.py:6:15 | -6 | re.compile("\t/x28") # snapshot - | ^^^^ +6 | re.compile("\t\N{LEFT PARENTHESIS}") # snapshot + | ^^^^^^^^^^^^^^^^^^^^ ``` literals written next to each other are one pattern, so the error is in whichever of them wrote the diff --git a/crates/ty_python_semantic/src/types/regex.rs b/crates/ty_python_semantic/src/types/regex.rs index 9ea3cd0103..edd2b2ddf6 100644 --- a/crates/ty_python_semantic/src/types/regex.rs +++ b/crates/ty_python_semantic/src/types/regex.rs @@ -10,6 +10,7 @@ //! (see [`Type::regex_instance`]), so they survive assignment, narrowing and a //! round trip through `re.compile` +mod character_names; mod parse; mod source; diff --git a/crates/ty_python_semantic/src/types/regex/character_names.rs b/crates/ty_python_semantic/src/types/regex/character_names.rs new file mode 100644 index 0000000000..eef33d747b --- /dev/null +++ b/crates/ty_python_semantic/src/types/regex/character_names.rs @@ -0,0 +1,288 @@ +//! this file is generated by `scripts/generate_regex_character_names.py` +//! +//! the unicode database each python minor ships: +//! +//! - python 3.8: unicode 12.1.0 +//! - python 3.9: unicode 13.0.0 +//! - python 3.10: unicode 13.0.0 +//! - python 3.11: unicode 14.0.0 +//! - python 3.12: unicode 15.0.0 +//! - python 3.13: unicode 15.1.0 +//! - python 3.14: unicode 16.0.0 +//! - python 3.15: unicode 17.0.0 + +/// the code points python 3.8 has no name for that a later python does, as +/// `(first, last, minor)` runs: every code point from `first` to `last` is named +/// from python 3.`minor` on. sorted by `first`, and the runs do not overlap +pub(super) const NAMED_SINCE: &[(u32, u32, u8)] = &[ + (0x061D, 0x061D, 11), + (0x0870, 0x088E, 11), + (0x088F, 0x088F, 15), + (0x0890, 0x0891, 11), + (0x0897, 0x0897, 14), + (0x0898, 0x089F, 11), + (0x08B5, 0x08B5, 11), + (0x08BE, 0x08C7, 9), + (0x08C8, 0x08D2, 11), + (0x0B55, 0x0B55, 9), + (0x0C3C, 0x0C3C, 11), + (0x0C5C, 0x0C5C, 15), + (0x0C5D, 0x0C5D, 11), + (0x0CDC, 0x0CDC, 15), + (0x0CDD, 0x0CDD, 11), + (0x0CF3, 0x0CF3, 12), + (0x0D04, 0x0D04, 9), + (0x0D81, 0x0D81, 9), + (0x0ECE, 0x0ECE, 12), + (0x170D, 0x170D, 11), + (0x1715, 0x1715, 11), + (0x171F, 0x171F, 11), + (0x180F, 0x180F, 11), + (0x1ABF, 0x1AC0, 9), + (0x1AC1, 0x1ACE, 11), + (0x1ACF, 0x1ADD, 15), + (0x1AE0, 0x1AEB, 15), + (0x1B4C, 0x1B4C, 11), + (0x1B4E, 0x1B4F, 14), + (0x1B7D, 0x1B7E, 11), + (0x1B7F, 0x1B7F, 14), + (0x1C89, 0x1C8A, 14), + (0x1DFA, 0x1DFA, 11), + (0x20C0, 0x20C0, 11), + (0x20C1, 0x20C1, 15), + (0x2427, 0x2429, 14), + (0x2B96, 0x2B96, 15), + (0x2B97, 0x2B97, 9), + (0x2C2F, 0x2C2F, 11), + (0x2C5F, 0x2C5F, 11), + (0x2E50, 0x2E52, 9), + (0x2E53, 0x2E5D, 11), + (0x2FFC, 0x2FFF, 13), + (0x31BB, 0x31BF, 9), + (0x31E4, 0x31E5, 14), + (0x31EF, 0x31EF, 13), + (0xA7C0, 0xA7C1, 11), + (0xA7C7, 0xA7CA, 9), + (0xA7CB, 0xA7CD, 14), + (0xA7CE, 0xA7CF, 15), + (0xA7D0, 0xA7D1, 11), + (0xA7D2, 0xA7D2, 15), + (0xA7D3, 0xA7D3, 11), + (0xA7D4, 0xA7D4, 15), + (0xA7D5, 0xA7D9, 11), + (0xA7DA, 0xA7DC, 14), + (0xA7F1, 0xA7F1, 15), + (0xA7F2, 0xA7F4, 11), + (0xA7F5, 0xA7F6, 9), + (0xA82C, 0xA82C, 9), + (0xAB68, 0xAB6B, 9), + (0xFBC2, 0xFBC2, 11), + (0xFBC3, 0xFBD2, 15), + (0xFD40, 0xFD4F, 11), + (0xFD90, 0xFD91, 15), + (0xFDC8, 0xFDCE, 15), + (0xFDCF, 0xFDCF, 11), + (0xFDFE, 0xFDFF, 11), + (0x1019C, 0x1019C, 9), + (0x10570, 0x1057A, 11), + (0x1057C, 0x1058A, 11), + (0x1058C, 0x10592, 11), + (0x10594, 0x10595, 11), + (0x10597, 0x105A1, 11), + (0x105A3, 0x105B1, 11), + (0x105B3, 0x105B9, 11), + (0x105BB, 0x105BC, 11), + (0x105C0, 0x105F3, 14), + (0x10780, 0x10785, 11), + (0x10787, 0x107B0, 11), + (0x107B2, 0x107BA, 11), + (0x10940, 0x10959, 15), + (0x10D40, 0x10D65, 14), + (0x10D69, 0x10D85, 14), + (0x10D8E, 0x10D8F, 14), + (0x10E80, 0x10EA9, 9), + (0x10EAB, 0x10EAD, 9), + (0x10EB0, 0x10EB1, 9), + (0x10EC2, 0x10EC4, 14), + (0x10EC5, 0x10EC7, 15), + (0x10ED0, 0x10ED8, 15), + (0x10EFA, 0x10EFB, 15), + (0x10EFC, 0x10EFC, 14), + (0x10EFD, 0x10EFF, 12), + (0x10F70, 0x10F89, 11), + (0x10FB0, 0x10FCB, 9), + (0x11070, 0x11075, 11), + (0x110C2, 0x110C2, 11), + (0x11147, 0x11147, 9), + (0x111CE, 0x111CF, 9), + (0x1123F, 0x11241, 12), + (0x11380, 0x11389, 14), + (0x1138B, 0x1138B, 14), + (0x1138E, 0x1138E, 14), + (0x11390, 0x113B5, 14), + (0x113B7, 0x113C0, 14), + (0x113C2, 0x113C2, 14), + (0x113C5, 0x113C5, 14), + (0x113C7, 0x113CA, 14), + (0x113CC, 0x113D5, 14), + (0x113D7, 0x113D8, 14), + (0x113E1, 0x113E2, 14), + (0x1145A, 0x1145A, 9), + (0x11460, 0x11461, 9), + (0x116B9, 0x116B9, 11), + (0x116D0, 0x116E3, 14), + (0x11740, 0x11746, 11), + (0x11900, 0x11906, 9), + (0x11909, 0x11909, 9), + (0x1190C, 0x11913, 9), + (0x11915, 0x11916, 9), + (0x11918, 0x11935, 9), + (0x11937, 0x11938, 9), + (0x1193B, 0x11946, 9), + (0x11950, 0x11959, 9), + (0x11AB0, 0x11ABF, 11), + (0x11B00, 0x11B09, 12), + (0x11B60, 0x11B67, 15), + (0x11BC0, 0x11BE1, 14), + (0x11BF0, 0x11BF9, 14), + (0x11DB0, 0x11DDB, 15), + (0x11DE0, 0x11DE9, 15), + (0x11F00, 0x11F10, 12), + (0x11F12, 0x11F3A, 12), + (0x11F3E, 0x11F59, 12), + (0x11F5A, 0x11F5A, 14), + (0x11FB0, 0x11FB0, 9), + (0x12F90, 0x12FF2, 11), + (0x1342F, 0x1342F, 12), + (0x13439, 0x13455, 12), + (0x13460, 0x143FA, 14), + (0x16100, 0x16139, 14), + (0x16A70, 0x16ABE, 11), + (0x16AC0, 0x16AC9, 11), + (0x16D40, 0x16D79, 14), + (0x16EA0, 0x16EB8, 15), + (0x16EBB, 0x16ED3, 15), + (0x16FE4, 0x16FE4, 9), + (0x16FF0, 0x16FF1, 9), + (0x16FF2, 0x16FF6, 15), + (0x17000, 0x187FF, 15), + (0x18AF3, 0x18CD5, 9), + (0x18CFF, 0x18CFF, 14), + (0x18D00, 0x18D1E, 15), + (0x18D80, 0x18DF2, 15), + (0x1AFF0, 0x1AFF3, 11), + (0x1AFF5, 0x1AFFB, 11), + (0x1AFFD, 0x1AFFE, 11), + (0x1B11F, 0x1B122, 11), + (0x1B132, 0x1B132, 12), + (0x1B155, 0x1B155, 12), + (0x1CC00, 0x1CCF9, 14), + (0x1CCFA, 0x1CCFC, 15), + (0x1CD00, 0x1CEB3, 14), + (0x1CEBA, 0x1CED0, 15), + (0x1CEE0, 0x1CEF0, 15), + (0x1CF00, 0x1CF2D, 11), + (0x1CF30, 0x1CF46, 11), + (0x1CF50, 0x1CFC3, 11), + (0x1D1E9, 0x1D1EA, 11), + (0x1D2C0, 0x1D2D3, 12), + (0x1DF00, 0x1DF1E, 11), + (0x1DF25, 0x1DF2A, 12), + (0x1E030, 0x1E06D, 12), + (0x1E08F, 0x1E08F, 12), + (0x1E290, 0x1E2AE, 11), + (0x1E4D0, 0x1E4F9, 12), + (0x1E5D0, 0x1E5FA, 14), + (0x1E5FF, 0x1E5FF, 14), + (0x1E6C0, 0x1E6DE, 15), + (0x1E6E0, 0x1E6F5, 15), + (0x1E6FE, 0x1E6FF, 15), + (0x1E7E0, 0x1E7E6, 11), + (0x1E7E8, 0x1E7EB, 11), + (0x1E7ED, 0x1E7EE, 11), + (0x1E7F0, 0x1E7FE, 11), + (0x1F10D, 0x1F10F, 9), + (0x1F16D, 0x1F16F, 9), + (0x1F1AD, 0x1F1AD, 9), + (0x1F6D6, 0x1F6D7, 9), + (0x1F6D8, 0x1F6D8, 15), + (0x1F6DC, 0x1F6DC, 12), + (0x1F6DD, 0x1F6DF, 11), + (0x1F6FB, 0x1F6FC, 9), + (0x1F774, 0x1F776, 12), + (0x1F777, 0x1F77A, 15), + (0x1F77B, 0x1F77F, 12), + (0x1F7D9, 0x1F7D9, 12), + (0x1F7F0, 0x1F7F0, 11), + (0x1F8B0, 0x1F8B1, 9), + (0x1F8B2, 0x1F8BB, 14), + (0x1F8C0, 0x1F8C1, 14), + (0x1F8D0, 0x1F8D8, 15), + (0x1F90C, 0x1F90C, 9), + (0x1F972, 0x1F972, 9), + (0x1F977, 0x1F978, 9), + (0x1F979, 0x1F979, 11), + (0x1F9A3, 0x1F9A4, 9), + (0x1F9AB, 0x1F9AD, 9), + (0x1F9CB, 0x1F9CB, 9), + (0x1F9CC, 0x1F9CC, 11), + (0x1FA54, 0x1FA57, 15), + (0x1FA74, 0x1FA74, 9), + (0x1FA75, 0x1FA77, 12), + (0x1FA7B, 0x1FA7C, 11), + (0x1FA83, 0x1FA86, 9), + (0x1FA87, 0x1FA88, 12), + (0x1FA89, 0x1FA89, 14), + (0x1FA8A, 0x1FA8A, 15), + (0x1FA8E, 0x1FA8E, 15), + (0x1FA8F, 0x1FA8F, 14), + (0x1FA96, 0x1FAA8, 9), + (0x1FAA9, 0x1FAAC, 11), + (0x1FAAD, 0x1FAAF, 12), + (0x1FAB0, 0x1FAB6, 9), + (0x1FAB7, 0x1FABA, 11), + (0x1FABB, 0x1FABD, 12), + (0x1FABE, 0x1FABE, 14), + (0x1FABF, 0x1FABF, 12), + (0x1FAC0, 0x1FAC2, 9), + (0x1FAC3, 0x1FAC5, 11), + (0x1FAC6, 0x1FAC6, 14), + (0x1FAC8, 0x1FAC8, 15), + (0x1FACD, 0x1FACD, 15), + (0x1FACE, 0x1FACF, 12), + (0x1FAD0, 0x1FAD6, 9), + (0x1FAD7, 0x1FAD9, 11), + (0x1FADA, 0x1FADB, 12), + (0x1FADC, 0x1FADC, 14), + (0x1FADF, 0x1FADF, 14), + (0x1FAE0, 0x1FAE7, 11), + (0x1FAE8, 0x1FAE8, 12), + (0x1FAE9, 0x1FAE9, 14), + (0x1FAEA, 0x1FAEA, 15), + (0x1FAEF, 0x1FAEF, 15), + (0x1FAF0, 0x1FAF6, 11), + (0x1FAF7, 0x1FAF8, 12), + (0x1FB00, 0x1FB92, 9), + (0x1FB94, 0x1FBCA, 9), + (0x1FBCB, 0x1FBEF, 14), + (0x1FBF0, 0x1FBF9, 9), + (0x1FBFA, 0x1FBFA, 15), +]; + +/// the aliases python 3.8 does not resolve that a later python does, as +/// `(alias, minor)`: `alias` resolves from python 3.`minor` on. sorted by alias +pub(super) const ALIASED_SINCE: &[(&str, u8)] = &[ + ("ARABIC SMALL HIGH LIGATURE ALEF WITH YEH BARREE", 12), + ("BAMUM LETTER PHASE-A MAEMGBIEE", 14), + ("BAMUM LETTER PHASE-B NGGOM", 15), + ("BAMUM LETTER PHASE-B PUNGGAAM", 15), + ("BAMUM LETTER PHASE-C SHETFON", 15), + ("BAMUM LETTER PHASE-E NGGOP", 15), + ("CUNEIFORM SIGN KALAM", 14), + ("EM", 12), + ("FVS4", 11), + ("MENDE KIKAKUI SYLLABLE M172 MBO", 14), + ("MENDE KIKAKUI SYLLABLE M174 MBOO", 14), + ("MYANMAR LETTER KHAMTI LLA", 11), + ("SUNDANESE LETTER ARCHAIC I", 12), +]; diff --git a/crates/ty_python_semantic/src/types/regex/parse.rs b/crates/ty_python_semantic/src/types/regex/parse.rs index 9e003e6658..5b55f3b769 100644 --- a/crates/ty_python_semantic/src/types/regex/parse.rs +++ b/crates/ty_python_semantic/src/types/regex/parse.rs @@ -14,8 +14,12 @@ use std::ops::Range; use ruff_python_ast::PythonVersion; use ruff_python_ast::name::Name; +use ruff_python_ast::str::TripleQuotes; +use ruff_python_literal::escape::UnicodeEscape; use rustc_hash::FxHashMap; +use super::character_names::{ALIASED_SINCE, NAMED_SINCE}; + /// the characters `sre_parse` treats as ignorable whitespace under `re.VERBOSE` const VERBOSE_WHITESPACE: [char; 6] = [' ', '\t', '\n', '\r', '\x0b', '\x0c']; @@ -436,7 +440,7 @@ impl Parser { Err(self.bad_escape(start)) } } - 'N' if text && self.dialect.version >= PythonVersion::PY38 => self.named_escape(), + 'N' if text && self.dialect.version >= PythonVersion::PY38 => self.named_escape(start), '0'..='7' if in_class => { self.eat_octal_digits(2); self.octal_in_range(start) @@ -480,10 +484,10 @@ impl Parser { } } - /// the `{NAME}` of a `\N{NAME}`, whose name is not looked up: python's - /// unicode database is not ours, so a name it lacks is a missed error rather - /// than a valid pattern reported as invalid - fn named_escape(&mut self) -> Result<(), Failure> { + /// the `{NAME}` of a `\N{NAME}` escape starting at `start`, whose name has to + /// be one the python version's unicode database has for a single character — + /// see [`character_name_resolves`] + fn named_escape(&mut self, start: usize) -> Result<(), Failure> { if !self.eat('{') { return Err(Failure::Invalid(RegexError::new("missing {", self.pos))); } @@ -510,7 +514,18 @@ impl Parser { ))); } Some('}') => { + let text = self.text(name..self.pos); self.pos += 1; + if character_name_resolves(&text, self.dialect.version) == Some(false) { + return Err(Failure::Invalid(RegexError::spanning( + format_args!( + "undefined character name {}", + UnicodeEscape::new_repr(&text).str_repr(TripleQuotes::No) + ), + start, + start..self.pos, + ))); + } return Ok(()); } Some(_) => self.pos += 1, @@ -569,23 +584,35 @@ impl Parser { start + 1..self.pos, ))); } + self.check_backreference(group, start, start..self.pos) + } + + /// what python checks of a backreference to `group`, by number or by name, whose text + /// is `span`: that the group is closed, which python reports at `start`, and that it + /// was not opened in the lookbehind the backreference is in + fn check_backreference( + &self, + group: u32, + start: usize, + span: Range, + ) -> Result<(), Failure> { if !self .closed - .get(group as usize - 1) + .get((group as usize).saturating_sub(1)) .copied() .unwrap_or_default() { return Err(Failure::Invalid(RegexError::spanning( "cannot refer to an open group", start, - start..self.pos, + span, ))); } if self.lookbehind_groups.is_some_and(|first| group >= first) { return Err(Failure::Invalid(RegexError::spanning( "cannot refer to group defined in the same lookbehind subpattern", self.pos, - start..self.pos, + span, ))); } Ok(()) @@ -766,13 +793,14 @@ impl Parser { let name = Self::validate_group_name(&name, start)?; // python resolves a backreference against the groups opened so // far, so a forward reference is an error there too - if !self.names.contains_key(&name) { + let Some(&group) = self.names.get(&name) else { return Err(Failure::Invalid(RegexError::spanning( format_args!("unknown group name '{name}'"), start, name_span(&name, start), ))); - } + }; + self.check_backreference(group, start, name_span(&name, start))?; Ok(Some(Node::Atom)) } Some(char) => Err(Failure::Invalid(RegexError::new( @@ -951,6 +979,59 @@ struct BracedQuantifier { } /// the characters of a group's name that starts at `start` +/// the newest python whose unicode database `unicode_names2` has every name of: +/// the crate's tables are unicode 17.0's, which python 3.15 ships +const NAMES_KNOWN_THROUGH: PythonVersion = PythonVersion::PY315; + +/// whether `re` on `version` resolves `name`, the name in a `\N{...}`, to one +/// character, or `None` where we cannot tell +/// +/// python looks the name up in the `unicodedata` of its own version, case +/// insensitively, among the characters' names and their aliases. a named +/// sequence resolves to several characters, which `re` refuses as it refuses a +/// name that resolves to nothing. names and aliases are never changed or +/// withdrawn, so a name python resolves is one `unicode_names2` resolves, through +/// the newest python whose unicode its tables hold, and the version decides only +/// whether the alias exists yet, which [`ALIASED_SINCE`] says, or else whether the +/// character the name is for has a name yet, which [`NAMED_SINCE`] says +fn character_name_resolves(name: &str, version: PythonVersion) -> Option { + if version > NAMES_KNOWN_THROUGH { + return None; + } + // names derived from the code point by a rule, which pythons do not all + // implement alike: 3.8 names `CJK UNIFIED IDEOGRAPH-2EBE1`, which no unicode + // assigns, only 3.13 and later resolve one written in lowercase or a + // `TANGUT IDEOGRAPH-17000`, and only 3.12 and earlier one whose code point is + // written with a leading zero + let upper = name.to_ascii_uppercase(); + if upper.starts_with("CJK UNIFIED IDEOGRAPH-") || upper.starts_with("TANGUT IDEOGRAPH-") { + return None; + } + if let Ok(index) = ALIASED_SINCE.binary_search_by(|&(alias, _)| alias.cmp(upper.as_str())) + && let Some(&(_, minor)) = ALIASED_SINCE.get(index) + { + return Some(version.minor >= minor); + } + let Some(character) = unicode_names2::character(name) else { + return Some(false); + }; + let code = u32::from(character); + let named_since = NAMED_SINCE + .binary_search_by(|&(first, last, _)| { + if last < code { + std::cmp::Ordering::Less + } else if first > code { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }) + .ok() + .and_then(|index| NAMED_SINCE.get(index)) + .map(|&(_, _, minor)| minor); + Some(named_since.is_none_or(|minor| version.minor >= minor)) +} + fn name_span(name: &str, start: usize) -> Range { start..start + name.chars().count() } @@ -1009,7 +1090,7 @@ fn collect_certainty(node: &Node, certain: bool, out: &mut [bool]) { mod tests { use ruff_python_ast::PythonVersion; - use super::{Dialect, ParsedGroup, PatternAnalysis, analyze}; + use super::{Dialect, NAMES_KNOWN_THROUGH, ParsedGroup, PatternAnalysis, analyze}; const STR: Dialect = Dialect { version: PythonVersion::PY313, @@ -1115,6 +1196,24 @@ mod tests { ); } + /// a backreference by name is checked as one by number is + #[test] + fn named_backreferences() { + assert_eq!( + shape("(?P(?P=a))"), + "invalid: cannot refer to an open group at position 10" + ); + assert_eq!( + shape("(?P(?Px)(?P=b))"), + "invalid: cannot refer to an open group at position 18" + ); + assert_eq!( + shape("(?<=(?Px)(?P=a))"), + "invalid: cannot refer to group defined in the same lookbehind subpattern at position 18" + ); + assert_eq!(shape("(?Px)(?<=(?P=a))"), "a"); + } + #[test] fn lookarounds() { // a positive lookahead's groups do capture @@ -1404,6 +1503,103 @@ mod tests { ); } + /// a `\N{...}` name is looked up as `unicodedata.lookup` looks it up: in any + /// case, among the names and their aliases, and for one character + #[test] + fn character_names() { + assert_eq!(shape(r"\N{em dash}\N{BYTE ORDER MARK}\N{LINE FEED}"), ""); + assert_eq!( + shape(r"a\N{EM DASH}(b)"), + "invalid: undefined character name 'EM DASH' at position 1" + ); + // a named sequence stands for several characters + assert_eq!( + shape(r"\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}"), + "invalid: undefined character name 'LATIN CAPITAL LETTER A WITH MACRON AND GRAVE' \ + at position 0" + ); + // the name is quoted as python's `repr` quotes it + assert_eq!( + shape(r"\N{O'CLOCK}"), + r#"invalid: undefined character name "O'CLOCK" at position 0"# + ); + // names derived from the code point, which pythons do not all derive alike, + // are left to python + assert_eq!(shape(r"\N{CJK UNIFIED IDEOGRAPH-2EBE1}"), ""); + assert_eq!(shape(r"\N{TANGUT IDEOGRAPH-FFFFF}"), ""); + } + + /// a name exists from the python whose unicode assigns its character, and past + /// the unicode `unicode_names2` holds, a name is not looked up at all + #[test] + fn character_names_by_version() { + let version = |minor| Dialect { + version: PythonVersion { major: 3, minor }, + ..STR + }; + // unicode 14.0, python 3.11 + let arabic = r"\N{ARABIC POUND MARK ABOVE}"; + assert_eq!( + shape_in(arabic, false, version(10)), + "invalid: undefined character name 'ARABIC POUND MARK ABOVE' at position 0" + ); + assert_eq!(shape_in(arabic, false, version(11)), ""); + // unicode 16.0, python 3.14 + let alien = r"\N{ALIEN MONSTER OPEN JAWS}"; + assert_eq!( + shape_in(alien, false, version(13)), + "invalid: undefined character name 'ALIEN MONSTER OPEN JAWS' at position 0" + ); + assert_eq!(shape_in(alien, false, version(14)), ""); + // unicode 17.0, python 3.15 + let bell = r"\N{ALARM BELL SYMBOL}"; + assert_eq!( + shape_in(bell, false, version(14)), + "invalid: undefined character name 'ALARM BELL SYMBOL' at position 0" + ); + assert_eq!(shape_in(bell, false, version(15)), ""); + assert_eq!(shape_in(r"\N{NO SUCH NAME}", false, version(16)), ""); + } + + /// an alias exists from the python whose unicode adds it, which can be long after the + /// character it is for was first named + #[test] + fn character_aliases_by_version() { + let version = |minor| Dialect { + version: PythonVersion { major: 3, minor }, + ..STR + }; + // `END OF MEDIUM`, named in every python, is aliased `EM` from python 3.12 + assert_eq!( + shape_in(r"\N{em}", false, version(11)), + "invalid: undefined character name 'em' at position 0" + ); + assert_eq!(shape_in(r"\N{em}", false, version(12)), ""); + let bamum = r"\N{BAMUM LETTER PHASE-B NGGOM}"; + assert_eq!( + shape_in(bamum, false, version(14)), + "invalid: undefined character name 'BAMUM LETTER PHASE-B NGGOM' at position 0" + ); + assert_eq!(shape_in(bamum, false, version(15)), ""); + // an alias every python resolves + assert_eq!(shape_in(r"\N{BYTE ORDER MARK}", false, version(8)), ""); + } + + /// `NAMES_KNOWN_THROUGH` is the python whose unicode `unicode_names2` holds: + /// the tables are built from the unicode 17.0 data vendored beside the crate, + /// and know a name 17.0 assigned. when the data moves to a newer unicode, + /// `NAMES_KNOWN_THROUGH` moves to the python that ships it + #[test] + fn unicode_names2_holds_the_unicode_of_names_known_through() { + let aliases = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../ruff_unicode_names2/data/NameAliases.txt" + )); + assert_eq!(aliases.lines().next(), Some("# NameAliases-17.0.0.txt")); + assert_eq!(NAMES_KNOWN_THROUGH, PythonVersion::PY315); + assert!(unicode_names2::character("ALARM BELL SYMBOL").is_some()); + } + /// `\z` is an anchor from python 3.14, and a bad escape before it #[test] fn escapes_by_version() { diff --git a/docs/basedpython/features/regex-groups.md b/docs/basedpython/features/regex-groups.md index 3a60c287c1..8025646642 100644 --- a/docs/basedpython/features/regex-groups.md +++ b/docs/basedpython/features/regex-groups.md @@ -124,8 +124,23 @@ escapes are read by `re`'s own rules for the python version checked against: an ASCII letter or digit after a backslash that is no escape, `\q`, is a bad escape rather than the letter, a `bytes` pattern has no `\u`, `\U` or `\N`, `\z` is an anchor only from 3.14, and a backreference has to name a group already closed. -each is reported at the whole escape. the name in a `\N{...}` is not looked up, -so a name python's unicode database lacks goes unreported +each is reported at the whole escape. + +the name in a `\N{...}` is looked up as `unicodedata.lookup` looks it up in the +python version checked against — in any case, among the characters' names and +their aliases — so a name that python's unicode database does not have yet, or a +named sequence, which stands for several characters, is reported too: + +```py +re.compile(r"\N{EM DASH}") # error: undefined character name 'EM DASH' at position 0 +``` + +a character is named from the python that ships the unicode version assigning it, +which is read off each python's own `unicodedata`. two kinds of name are left to +python: one derived from the code point, `CJK UNIFIED IDEOGRAPH-4E00` or +`TANGUT IDEOGRAPH-17000`, which pythons derive differently, and any name checked +against a python newer than 3.15, whose unicode is newer than the table the names +are read from a construct the pattern reader does not model produces neither a refinement nor a diagnostic: a pattern reported invalid when `re.compile` would have accepted it diff --git a/scripts/generate_regex_character_names.py b/scripts/generate_regex_character_names.py new file mode 100644 index 0000000000..5410f2861d --- /dev/null +++ b/scripts/generate_regex_character_names.py @@ -0,0 +1,244 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [] +# +# [tool.uv] +# no-build = true +# exclude-newer = "P7D" +# /// + +r"""Generate `crates/ty_python_semantic/src/types/regex/character_names.rs`. + +`re` looks the name in a `\N{...}` escape up in the `unicodedata` of the python +that compiles the pattern, and each python minor ships the unicode database of one +unicode version. Character names are never changed or withdrawn once assigned, so +what differs between two pythons is which code points have a name at all. This +asks each python which code points it names and writes the ones a python after +the oldest names first, with the minor version that first names them. + +A name can also be an alias, and unicode adds aliases to characters that have long +had a name. So this also asks each python which of the aliases in the unicode data +`unicode_names2` is built from it resolves, and writes the ones a python after the +oldest resolves first, with the minor version that first resolves them. + +The pythons are run with `uv run --managed-python`, which downloads any of them +that is not installed yet. +""" + +from __future__ import annotations + +import subprocess +import textwrap +from pathlib import Path +from typing import Final + +MODULE_CRATE: Final = "ty_python_semantic" +MODULE_PATH: Final = ( + Path("crates") / MODULE_CRATE / "src" / "types" / "regex" / "character_names.rs" +) +PYTHON_VERSIONS: Final = range(8, 16) # Python 3.8 through 3.15 + +# prints the unicode version, then each run of consecutive code points that have a +# name as `first last`, in hex. a `CJK UNIFIED IDEOGRAPH-` name is left out: those are +# derived from the code point by a rule each python implements a little differently +# (3.8 names `2EBE1` to `2EBEF`, which no unicode assigns), so the regex reader never +# reports one, and has no use for when each was first named +NAMED_RANGES: Final = """ +import unicodedata +print(unicodedata.unidata_version) +start = None +for cp in range(0x110001): + name = unicodedata.name(chr(cp), None) if cp < 0x110000 else None + named = name is not None and not name.startswith("CJK UNIFIED IDEOGRAPH-") + if named and start is None: + start = cp + elif not named and start is not None: + print(f"{start:x} {cp - 1:x}") + start = None +""" + + +NAME_ALIASES: Final = ( + Path("crates") / "ruff_unicode_names2" / "data" / "NameAliases.txt" +) + +# reads `alias code` lines, in hex, from stdin and prints each alias it resolves to +# that code point +RESOLVED_ALIASES: Final = """ +import sys +import unicodedata +for line in sys.stdin: + alias, code = line.rstrip("\\n").split(";") + try: + resolved = unicodedata.lookup(alias) + except KeyError: + continue + if resolved == chr(int(code, 16)): + print(alias) +""" + + +def name_aliases() -> list[tuple[str, str]]: + """every alias in the unicode data, with the code point it names in hex""" + aliases = [] + for line in NAME_ALIASES.read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + match line.split(";"): + case [code, alias, _kind]: + aliases.append((alias, code)) + case _: + raise ValueError(f"not a NameAliases.txt entry: {line!r}") + return aliases + + +def resolved_on_version(minor: int, aliases: list[tuple[str, str]]) -> set[str]: + proc = subprocess.run( + [ + "uv", + "run", + "--managed-python", + "--no-project", + "--python", + f"3.{minor}", + "python", + "-I", + "-c", + RESOLVED_ALIASES, + ], + input="".join(f"{alias};{code}\n" for alias, code in aliases), + check=True, + text=True, + capture_output=True, + ) + return set(proc.stdout.splitlines()) + + +def named_on_version(minor: int) -> tuple[str, set[int]]: + proc = subprocess.run( + [ + "uv", + "run", + "--managed-python", + "--no-project", + "--python", + f"3.{minor}", + "python", + "-I", + "-c", + NAMED_RANGES, + ], + check=True, + text=True, + capture_output=True, + ) + match proc.stdout.splitlines(): + case [unicode_version, *ranges]: + named: set[int] = set() + for line in ranges: + match line.split(): + case [first, last]: + named.update(range(int(first, 16), int(last, 16) + 1)) + case _: + raise ValueError(f"not a code point range: {line!r}") + return unicode_version, named + case _: + raise ValueError(f"python 3.{minor} printed no unicode version") + + +def generate_module() -> None: + by_version = {minor: named_on_version(minor) for minor in PYTHON_VERSIONS} + oldest = PYTHON_VERSIONS[0] + + # the code point -> the first minor that names it, for those the oldest does not + first_named: dict[int, int] = {} + previous = by_version[oldest][1] + for minor in PYTHON_VERSIONS[1:]: + named = by_version[minor][1] + withdrawn = previous - named + if withdrawn: + raise SystemExit( + f"python 3.{minor} names no code point {min(withdrawn):04X}, " + f"which python 3.{minor - 1} names" + ) + for cp in named - previous: + first_named[cp] = minor + previous = named + + runs: list[tuple[int, int, int]] = [] + for cp in sorted(first_named): + minor = first_named[cp] + if runs and runs[-1][1] == cp - 1 and runs[-1][2] == minor: + runs[-1] = (runs[-1][0], cp, minor) + else: + runs.append((cp, cp, minor)) + + aliases = name_aliases() + resolved = {minor: resolved_on_version(minor, aliases) for minor in PYTHON_VERSIONS} + newest = PYTHON_VERSIONS[-1] + unresolved = {alias for alias, _ in aliases} - resolved[newest] + if unresolved: + raise SystemExit( + f"python 3.{newest} does not resolve the alias {min(unresolved)}, " + f"which {NAME_ALIASES} lists" + ) + # the alias -> the first minor that resolves it, for those the oldest does not + first_resolved: dict[str, int] = {} + previous_aliases = resolved[oldest] + for minor in PYTHON_VERSIONS[1:]: + withdrawn = previous_aliases - resolved[minor] + if withdrawn: + raise SystemExit( + f"python 3.{minor} does not resolve the alias {min(withdrawn)}, " + f"which python 3.{minor - 1} resolves" + ) + for alias in resolved[minor] - previous_aliases: + first_resolved[alias] = minor + previous_aliases = resolved[minor] + alias_entries = "\n".join( + f' ("{alias}", {first_resolved[alias]}),' for alias in sorted(first_resolved) + ) + + versions = "\n".join( + f"//! - python 3.{minor}: unicode {by_version[minor][0]}" + for minor in PYTHON_VERSIONS + ) + entries = "\n".join( + f" (0x{first:04X}, 0x{last:04X}, {minor})," for first, last, minor in runs + ) + MODULE_PATH.write_text( + textwrap.dedent( + """\ + //! this file is generated by `scripts/generate_regex_character_names.py` + //! + //! the unicode database each python minor ships: + //! + {versions} + + /// the code points python 3.{oldest} has no name for that a later python does, as + /// `(first, last, minor)` runs: every code point from `first` to `last` is named + /// from python 3.`minor` on. sorted by `first`, and the runs do not overlap + pub(super) const NAMED_SINCE: &[(u32, u32, u8)] = &[ + {entries} + ]; + + /// the aliases python 3.{oldest} does not resolve that a later python does, as + /// `(alias, minor)`: `alias` resolves from python 3.`minor` on. sorted by alias + pub(super) const ALIASED_SINCE: &[(&str, u8)] = &[ + {alias_entries} + ]; + """ + ).format( + versions=versions, + oldest=oldest, + entries=entries, + alias_entries=alias_entries, + ) + ) + + subprocess.run(["rustfmt", "--edition", "2024", MODULE_PATH], check=True) + + +if __name__ == "__main__": + generate_module() diff --git a/scripts/generate_regex_character_names.py.lock b/scripts/generate_regex_character_names.py.lock new file mode 100644 index 0000000000..b51f89b1a9 --- /dev/null +++ b/scripts/generate_regex_character_names.py.lock @@ -0,0 +1,15 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[options.exclude-newer-package] +astral-dev-toolchain-hyperfine = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-insta = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-shear = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-codspeed = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-nextest = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-fuzz = "2026-09-01T00:00:00Z" From cdf1c9966af74738d9ab283ea90d8c08931c89b3 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:00:02 +1000 Subject: [PATCH 082/110] give a sealed class declared in a function or class body its __sealed_members__ tuple, assigned in that body from the subclasses declared alongside it, where it had none and raised AttributeError, and type the tuple from the class statements of the scope declaring the sealed class rather than from what the module's names hold at its end, which could name no subclass a function or class body declares; place the assignment after the whole if, for, try, ... a subclass is written in, at the scope's indentation, rather than right after the class, which put it between a branch and its else and did not parse --- .../src/transforms/ast_driver.rs | 7 +- .../by_transforms/src/transforms/modifiers.rs | 497 +++++++++++++----- crates/by_transforms/src/type_info.rs | 8 + crates/ty/tests/by_e2e.rs | 39 ++ .../resources/mdtest/basedpython_sealed.md | 139 ++++- .../src/types/class/static_literal.rs | 84 +-- .../src/types/ide_support.rs | 35 ++ docs/basedpython/features/modifiers.md | 18 +- docs/basedpython/features/sealed-classes.md | 62 ++- 9 files changed, 707 insertions(+), 182 deletions(-) diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 176ca47e37..d98374c1ec 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -1495,7 +1495,12 @@ pub(crate) fn run_against_source<'a>( raises_clause::RaisesGuardPass::new(source_ref, written, config.runtime_raises_checks); let type_fn_pass = type_fn::TypeFnPass::new(source_ref); let match_type_pass = match_type::MatchTypePass::new(source_ref, written); - let modifiers_pass = modifiers::ModifiersPass::new(source_ref, written, config); + let modifiers_pass = modifiers::ModifiersPass::new( + source_ref, + written, + modifiers::collect_sealed_listings(parsed_handle.suite(), &semantic_model), + config, + ); let main_function_pass = main_function::MainFunction::new(source_ref, written, config.session); let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index e8593e1ed7..d5d7bd74b1 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -36,10 +36,12 @@ use ruff_python_ast::{ self as ast, Expr, Stmt, StmtAnnAssign, StmtClassDef, StmtFunctionDef, StmtTypeAlias, }; use ruff_python_stdlib::basedpython::visibility_rename; -use ruff_text_size::{Ranged, TextRange, TextSize}; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; +use std::collections::HashMap; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; +use crate::type_info::TypeInfo; use crate::{Config, PythonVersion}; /// basedpython: the visibility a modifier keyword records, read off the @@ -79,19 +81,6 @@ fn is_class_var(annotation: &Expr) -> bool { } } -/// Returns the head identifier of a base-class expression: `A` for `A`, and -/// `A` for the generic form `A[int]`. Other base shapes have no simple name. -fn base_head_name(base: &Expr) -> Option<&str> { - match base { - Expr::Name(n) => Some(n.id.as_str()), - Expr::Subscript(s) => match s.value.as_ref() { - Expr::Name(n) => Some(n.id.as_str()), - _ => None, - }, - _ => None, - } -} - #[expect(clippy::struct_excessive_bools)] pub(crate) struct Modifiers<'src> { source: &'src str, @@ -121,21 +110,20 @@ pub(crate) struct Modifiers<'src> { exports: Vec, /// Module-level names renamed by `private` (original → `_original`). private_renames: Vec, - /// Module-level classes declared `sealed`, in source order. Each gets a - /// `.__sealed_members__` tuple of its same-module subclasses. - sealed_classes: Vec, - /// Module-level `(class name, base head names, statement end offset)` - /// triples, in source order. Used to resolve the subclasses of each sealed - /// class and to place the `__sealed_members__` assignment after the last one. - class_bases: Vec<(String, Vec, TextSize)>, + /// for each class statement that is a member of a `sealed` class, keyed by its range, + /// the ranges of the names of the sealed classes it is a member of + sealed_listings: &'src HashMap>, + /// the classes declared `sealed`, in source order + sealed_classes: Vec, + /// the members of sealed classes, in source order, each with the names of the sealed + /// classes it is a member of + sealed_members: Vec<(DeclaredClass, &'src [TextRange])>, /// Tracks the current class-nesting depth so visibility modifiers can /// distinguish module-level declarations from class members. class_depth: u32, - /// Tracks the current function-nesting depth. A class declared inside a - /// function body is not a module-level name, so it must not be recorded as - /// a sealed subclass (the runtime tuple assignment lives at module scope - /// and cannot reference a function-local name). + /// tracks the current function-nesting depth, so a declaration inside a + /// function body is not taken for a module-level one func_depth: u32, /// whether the innermost enclosing scope is a class body — where a /// visibility keyword names a member — rather than a function body @@ -143,7 +131,11 @@ pub(crate) struct Modifiers<'src> { } impl<'src> Modifiers<'src> { - fn new(source: &'src str, written: WrittenNames<'src>) -> Self { + fn new( + source: &'src str, + written: WrittenNames<'src>, + sealed_listings: &'src HashMap>, + ) -> Self { Self { source, written, @@ -162,8 +154,9 @@ impl<'src> Modifiers<'src> { needs_dataclass_slots: false, exports: Vec::new(), private_renames: Vec::new(), + sealed_listings, sealed_classes: Vec::new(), - class_bases: Vec::new(), + sealed_members: Vec::new(), class_depth: 0, func_depth: 0, class_body: false, @@ -255,19 +248,9 @@ impl<'src> Modifiers<'src> { } fn process_class(&mut self, class: &StmtClassDef) { - // Record module-level class → base-head-names so sealed-member tuples can - // be resolved after the whole module has been visited. - if self.at_module_level() { - let bases = class - .arguments - .iter() - .flat_map(|args| args.args.iter()) - .filter_map(base_head_name) - .map(str::to_owned) - .collect(); - self.class_bases - .push((class.name.as_str().to_owned(), bases, class.range().end())); - } + // the name the class statement binds once the modifiers below are lowered + let mut emitted = class.name.as_str().to_owned(); + let mut sealed = false; for dec in &class.decorator_list { let Some(name) = self.synthetic_name(dec) else { continue; @@ -280,13 +263,11 @@ impl<'src> Modifiers<'src> { .push(Fix::safe_edit(Edit::range_deletion(dec.range()))); } "sealed" => { - // Remove the modifier prefix; the `__sealed_members__` tuple is - // emitted after the last subclass once they are all known. + // the class gets its `__sealed_members__` tuple, and each member is + // added to it, once every class statement has been seen self.edits .push(Fix::safe_edit(Edit::range_deletion(dec.range()))); - if self.at_module_level() { - self.sealed_classes.push(class.name.as_str().to_owned()); - } + sealed = true; } "final" => { self.needs_final = true; @@ -346,18 +327,36 @@ impl<'src> Modifiers<'src> { .push(Fix::safe_edit(Edit::range_deletion(dec.range()))); if self.at_module_level() { self.private_renames.push(class.name.as_str().to_owned()); + emitted = self.written.module_private(class.name.as_str()); self.rename_with_underscore(class.name.range()); } else if self.class_body { // a nested class is a member like any other: its // visibility is spelled in its name, which for `private` // is the `__name` python name-mangles out of subclass // scope + if let Some(renamed) = visibility_rename( + class.name.as_str(), + visibility_of(name).name_prefix(), + ) { + emitted = renamed; + } self.rename_with_visibility(&class.name, visibility_of(name)); } } _ => {} } } + let declared = || DeclaredClass { + name: class.name.range(), + emitted: emitted.clone(), + statement: class.range(), + }; + if sealed { + self.sealed_classes.push(declared()); + } + if let Some(listing) = self.sealed_listings.get(&class.range()) { + self.sealed_members.push((declared(), listing)); + } } fn process_function(&mut self, func: &StmtFunctionDef) { @@ -744,8 +743,8 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { } Stmt::FunctionDef(f) => { self.process_function(f); - // Walk the body with `func_depth` incremented so a class defined - // inside the function is not treated as a module-level subclass. + // walk the body with `func_depth` incremented so declarations in it + // are not treated as module-level ones self.func_depth += 1; let enclosing = std::mem::replace(&mut self.class_body, false); walk_stmt(self, stmt); @@ -765,6 +764,46 @@ impl<'ast> Visitor<'ast> for Modifiers<'_> { } } +/// a class statement, as the `__sealed_members__` tuples need it +struct DeclaredClass { + /// the range of the name the class is written with + name: TextRange, + /// the name the class statement binds once it is lowered + emitted: String, + /// the class statement + statement: TextRange, +} + +/// basedpython: for each class statement of `stmts` that is a member of a `sealed` class, keyed +/// by its range, the ranges of the names of the sealed classes it is a member of. `stmts` must +/// come from the same parse `types` answers for +pub(crate) fn collect_sealed_listings( + stmts: &[Stmt], + types: &dyn TypeInfo, +) -> HashMap> { + struct Collector<'a> { + types: &'a dyn TypeInfo, + listings: HashMap>, + } + impl<'ast> Visitor<'ast> for Collector<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if let Stmt::ClassDef(class) = stmt { + let listing = self.types.sealed_classes_listing(class); + if !listing.is_empty() { + self.listings.insert(class.range(), listing); + } + } + walk_stmt(self, stmt); + } + } + let mut collector = Collector { + types, + listings: HashMap::new(), + }; + collector.visit_body(stmts); + collector.listings +} + /// The name a module-level `private` symbol is emitted under: one leading /// underscore, which is python's own mark for "not part of the interface". /// @@ -803,6 +842,8 @@ fn reads_class_cell(body: &[Stmt]) -> bool { pub(crate) struct ModifiersPass<'src> { source: &'src str, written: WrittenNames<'src>, + /// see [`collect_sealed_listings`] + sealed_listings: HashMap>, is_stub: bool, /// whether the python the module targets has `dataclass(slots=True)` dataclass_slots: bool, @@ -813,10 +854,16 @@ pub(crate) struct ModifiersPass<'src> { } impl<'src> ModifiersPass<'src> { - pub(crate) fn new(source: &'src str, written: WrittenNames<'src>, config: &Config) -> Self { + pub(crate) fn new( + source: &'src str, + written: WrittenNames<'src>, + sealed_listings: HashMap>, + config: &Config, + ) -> Self { Self { source, written, + sealed_listings, is_stub: config.is_stub, dataclass_slots: config.min_version >= PythonVersion::PY310, dataclass_slots_keep_class_cell: config.min_version >= PythonVersion::PY313, @@ -836,7 +883,7 @@ impl AstPass for ModifiersPass<'_> { } fn run(&self, module: &mut ruff_python_ast::ModModule, ctx: &mut PassContext) { - let mut inner = Modifiers::new(self.source, self.written); + let mut inner = Modifiers::new(self.source, self.written, &self.sealed_listings); // a stub is never run, so it keeps the option a checker reads inner.slots_by_helper = !self.dataclass_slots && !self.is_stub; inner.slots_by_helper_for_class_cell = @@ -846,14 +893,8 @@ impl AstPass for ModifiersPass<'_> { } let exports = std::mem::take(&mut inner.exports); let private_renames = std::mem::take(&mut inner.private_renames); - // the members tuple is assigned as the module runs, and a stub never - // runs. it declares the classes, which is all a checker reads of them - let sealed_classes = if self.is_stub { - Vec::new() - } else { - std::mem::take(&mut inner.sealed_classes) - }; - let class_bases = std::mem::take(&mut inner.class_bases); + let sealed_classes = std::mem::take(&mut inner.sealed_classes); + let sealed_members = std::mem::take(&mut inner.sealed_members); if inner.needs_dataclass_slots { ctx.runtime.insert(crate::runtime::DATACLASS_SLOTS); @@ -902,48 +943,87 @@ impl AstPass for ModifiersPass<'_> { .push(format!("__all__ = [{}]", entries.join(", "))); } - for sealed in &sealed_classes { - let subclasses: Vec<&str> = class_bases - .iter() - .filter(|(_, bases, _)| bases.iter().any(|base| base == sealed)) - .map(|(name, _, _)| name.as_str()) - .collect(); - let tuple = match subclasses.as_slice() { - [] => "()".to_owned(), - [single] => format!("({single},)"), - many => format!("({})", many.join(", ")), - }; + // the members tuple is built as the module runs, and a stub never runs. it + // declares the classes, which is all a checker reads of them + if !self.is_stub { + self.add_sealed_members(&sealed_classes, &sealed_members, ctx); + } + } +} - // Place the assignment right after the last subclass (or the sealed - // class itself when it has none), so module-level code that follows - // can read `__sealed_members__`. The module epilogue runs too late. - let anchor_end = class_bases - .iter() - .filter(|(name, bases, _)| { - name == sealed || bases.iter().any(|base| base == sealed) - }) - .map(|(_, _, end)| *end) - .max(); - let Some(anchor_end) = anchor_end else { - continue; - }; - let anchor_end = usize::from(anchor_end); - - let assignment = format!("{sealed}.__sealed_members__ = {tuple}"); - // Insert at the start of the line after the anchor statement so the - // edit never collides with `empty_declarations`' mid-line `: ...`. - // Without a blank line in between, that is where the next statement - // starts, so the insert is anchored to the statement boundary: a - // rewrite of that statement's first expression — a soundness check - // wrapping it — would otherwise absorb the assignment into itself. - if let Some(rel) = self.source[anchor_end..].find('\n') { - let pos = TextSize::try_from(anchor_end + rel + 1).expect("offset fits u32"); - ctx.statement_inserts - .push((pos, vec![Fragment::Lit(format!("{assignment}\n"))])); - } else { - let pos = TextSize::try_from(self.source.len()).expect("offset fits u32"); - ctx.text_edits - .push((TextRange::empty(pos), format!("\n{assignment}\n"))); +impl ModifiersPass<'_> { + /// give each sealed class its `__sealed_members__` tuple, right after its class + /// statement, and add each member to the tuple of each sealed class it is a member of, + /// right after the member's own class statement and in the block it is written in. + /// ty decides which classes are members, so a base written through an alias counts + /// + /// the tuple then holds the members whose statements have run, in the order they ran, + /// which is what ty types it as: a member declared under an `if` that is not taken, in + /// a loop, or after a `return`, is in it only once its statement has run, and code after + /// a member in the same block can read it. a member's statement is in the scope that + /// binds the sealed class, so the sealed class's name is bound wherever it runs + fn add_sealed_members( + &self, + sealed_classes: &[DeclaredClass], + sealed_members: &[(DeclaredClass, &[TextRange])], + ctx: &mut PassContext, + ) { + // the lines to insert after each class statement, which for a sealed class that is + // also a member of another is both its own tuple and its addition to the other's + let mut inserts: Vec<(TextRange, Vec)> = Vec::new(); + let mut insert = |statement: TextRange, line: String| match inserts + .iter_mut() + .find(|(after, _)| *after == statement) + { + Some((_, lines)) => lines.push(line), + None => inserts.push((statement, vec![line])), + }; + for sealed in sealed_classes { + insert( + sealed.statement, + format!("{}.__sealed_members__ = ()", sealed.emitted), + ); + } + for (member, listing) in sealed_members { + for name in *listing { + let Some(sealed) = sealed_classes.iter().find(|sealed| sealed.name == *name) else { + continue; + }; + insert( + member.statement, + format!( + "{}.__sealed_members__ += ({},)", + sealed.emitted, member.emitted + ), + ); + } + } + for (statement, lines) in inserts { + let indent = super::source_util::line_indent(self.source, statement.start()); + let mut text = String::new(); + for line in lines { + text.push_str(indent); + text.push_str(&line); + text.push('\n'); + } + let end = statement.end().to_usize(); + // at the start of the line after the class statement, so the edit never + // collides with `empty_declarations`' mid-line `: ...`. without a blank line + // in between, that is where the next statement starts, so the insert is + // anchored to the statement boundary: a rewrite of that statement's first + // expression — a soundness check wrapping it — would otherwise absorb the + // insert into itself + match self.source[end..] + .find('\n') + .and_then(|newline| TextSize::try_from(end + newline + 1).ok()) + { + Some(position) => ctx + .statement_inserts + .push((position, vec![Fragment::Lit(text)])), + None => ctx.text_edits.push(( + TextRange::empty(self.source.text_len()), + format!("\n{text}"), + )), } } } @@ -994,8 +1074,9 @@ mod tests { from typing import final @final class A: ... + A.__sealed_members__ = () class B(A): ... - A.__sealed_members__ = (B,) + A.__sealed_members__ += (B,) "}; check("sealed final class A\nclass B(A)\n", expected); check("final sealed class A\nclass B(A)\n", expected); @@ -2355,9 +2436,11 @@ mod tests { "}, indoc! {" class A: ... + A.__sealed_members__ = () class B(A): ... + A.__sealed_members__ += (B,) class C(A): ... - A.__sealed_members__ = (B, C) + A.__sealed_members__ += (C,) "}, ); } @@ -2376,32 +2459,18 @@ mod tests { "}, indoc! {" class A: ... + A.__sealed_members__ = () def make(): class C(A): ... class B(A): ... - A.__sealed_members__ = (B,) - "}, - ); - } - - #[test] - fn sealed_class_single_member() { - check( - indoc! {" - sealed class A - class B(A) - "}, - indoc! {" - class A: ... - class B(A): ... - A.__sealed_members__ = (B,) + A.__sealed_members__ += (B,) "}, ); } - /// with no blank line after the last subclass, the members tuple is assigned - /// where the next statement starts, and a soundness check wrapping that - /// statement's expression keeps it out of the call it opens there + /// with no blank line after a member, it is added to the tuple where the next + /// statement starts, and a soundness check wrapping that statement's expression + /// keeps the addition out of the call it opens there #[test] fn sealed_members_ahead_of_a_checked_statement() { let out = transpile( @@ -2420,7 +2489,7 @@ mod tests { assert!( out.contains(indoc! {" class B(A): ... - A.__sealed_members__ = (B,) + A.__sealed_members__ += (B,) _soundness_check(xs[0], str) "}), "got:\n{out}" @@ -2438,6 +2507,194 @@ mod tests { ); } + /// a sealed class declared in a function body gets its members tuple in that + /// body, which is the scope that can name it and its subclasses + #[test] + fn sealed_class_in_function_body() { + check( + indoc! {" + def make(): + sealed class A + class B(A) + class C(A) + return A + "}, + indoc! {" + def make(): + class A: ... + A.__sealed_members__ = () + class B(A): ... + A.__sealed_members__ += (B,) + class C(A): ... + A.__sealed_members__ += (C,) + return A + "}, + ); + } + + #[test] + fn sealed_class_in_class_body() { + check( + indoc! {" + class Outer: + sealed class A + class B(A) + print(Outer.A.__sealed_members__) + "}, + indoc! {" + class Outer: + class A: ... + A.__sealed_members__ = () + class B(A): ... + A.__sealed_members__ += (B,) + print(Outer.A.__sealed_members__) + "}, + ); + } + + /// a subclass declared in a scope nested inside the sealed class's own is not + /// a name the sealed class's scope can reach, and a same-named class in another + /// scope is not a member of this one + #[test] + fn sealed_members_are_the_binding_scopes_own() { + check( + indoc! {" + def make(): + sealed class A + def inner(): + class C(A): ... + class B(A) + return A + class A + class D(A) + "}, + indoc! {" + def make(): + class A: ... + A.__sealed_members__ = () + def inner(): + class C(A): ... + class B(A): ... + A.__sealed_members__ += (B,) + return A + class A: ... + class D(A): ... + "}, + ); + } + + /// a member is added in the block its statement is written in, right after it, + /// so a member whose statement does not run — in a branch not taken, or after a + /// `return` — is not added, and code after it in the same block can read it + #[test] + fn sealed_members_are_added_where_their_statement_runs() { + check( + indoc! {" + sealed class A + if flag: + class B(A) + else: + pass + for _ in range(2): + class C(A) + def make(flag): + sealed class D + class E(D) + if flag: + return D + class F(D) + return D + "}, + indoc! {" + class A: ... + A.__sealed_members__ = () + if flag: + class B(A): ... + A.__sealed_members__ += (B,) + else: + pass + for _ in range(2): + class C(A): ... + A.__sealed_members__ += (C,) + def make(flag): + class D: ... + D.__sealed_members__ = () + class E(D): ... + D.__sealed_members__ += (E,) + if flag: + return D + class F(D): ... + D.__sealed_members__ += (F,) + return D + "}, + ); + } + + /// a member is a class whose base is the sealed class, however the base is + /// written, as ty decides it + #[test] + fn sealed_members_through_another_name() { + check( + indoc! {" + sealed class A + Base = A + class B(Base) + "}, + indoc! {" + class A: ... + A.__sealed_members__ = () + Base = A + class B(Base): ... + A.__sealed_members__ += (B,) + "}, + ); + } + + /// a sealed class that is a member of another gets a tuple of its own and is + /// added to the other's + #[test] + fn sealed_member_of_a_sealed_class() { + check( + indoc! {" + sealed class A + sealed class B(A) + "}, + indoc! {" + class A: ... + A.__sealed_members__ = () + class B(A): ... + B.__sealed_members__ = () + A.__sealed_members__ += (B,) + "}, + ); + } + + /// a `private` class is added under the name it is emitted under, which python + /// mangles in a class body as it does the class statement's own + #[test] + fn private_sealed_class() { + check( + indoc! {" + private sealed class A + private class B(A) + class Outer: + private sealed class X + private class Y(X) + "}, + indoc! {" + class _A: ... + _A.__sealed_members__ = () + class _B(_A): ... + _A.__sealed_members__ += (_B,) + class Outer: + class __X: ... + __X.__sealed_members__ = () + class __Y(_Outer__X): ... + __X.__sealed_members__ += (__Y,) + "}, + ); + } + #[test] fn export_and_private_together_in_module() { check( diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index ad73032d85..f9b744b839 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -799,6 +799,10 @@ pub(crate) trait TypeInfo { /// `docs/basedpython/frameworks/index.md` fn framework_class_role(&self, class_def: &StmtClassDef) -> Option; + /// basedpython: the `sealed` classes `class_def` is a member of, each by the range of its + /// name, which ty types `__sealed_members__` from + fn sealed_classes_listing(&self, class_def: &StmtClassDef) -> Vec; + /// the inferred annotation to synthesize for a bare class-body assignment /// ` = ` — the promoted display of `value`'s type (`1` → /// `"int"`, `[1]` → `"list[int]"`, `int` → `"type[int]"`), with the modules @@ -1668,6 +1672,10 @@ impl TypeInfo for LoweringModel<'_> { ty_python_semantic::types::class_framework_role(self.db(), class) } + fn sealed_classes_listing(&self, class_def: &StmtClassDef) -> Vec { + ty_python_semantic::types::ide_support::sealed_classes_listing(self, class_def) + } + fn inferred_annotation( &self, expr: &Expr, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 04e1ccf930..3617039c64 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3371,6 +3371,45 @@ print(A.__sealed_members__ == (B, C)) assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "True"); } +/// a sealed class declared in a function or class body has its members tuple at +/// runtime too, assigned in the body that declares it +#[test] +fn nested_sealed_class_exposes_members_at_runtime() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +def make(): + sealed class A + class B(A) + class C(A) + return A, (B, C) + +class Outer: + sealed class X + class Y(X) + +sealed, members = make() +print(sealed.__sealed_members__ == members, Outer.X.__sealed_members__ == (Outer.Y,)) +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "True True"); +} + #[test] fn context_parameters_pass_implicitly_at_runtime() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sealed.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sealed.md index ee76749925..bf22aa7125 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sealed.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sealed.md @@ -2,43 +2,164 @@ A `sealed` class declares a closed hierarchy: it may be subclassed freely from anywhere within its own workspace, but not from a dependency. Every sealed class exposes a `__sealed_members__` tuple of -its same-module direct subclasses, which the transpiler materializes at runtime. +the direct subclasses declared in the scope that declares it. The transpiler adds each subclass to +the tuple as its class statement runs, so the tuple holds the subclasses whose statements have run, +in the order they ran. ## `__sealed_members__` lists the same-module subclasses +A subclass may be declared under an `if`, in a loop or after a `return`, so how many of them the +tuple holds is not known before the program runs. + ```by sealed class A class B(A) class C(A) -reveal_type(A.__sealed_members__) # revealed: (type[B], type[C]) +reveal_type(A.__sealed_members__) # revealed: (*: type[B | C]) +assert A.__sealed_members__ == (B, C) +``` + +## a sealed class with no subclasses + +```by +sealed class A + +reveal_type(A.__sealed_members__) # revealed: () +assert A.__sealed_members__ == () ``` -## a sealed class with a single subclass +## only direct subclasses are members ```by sealed class A class B(A) +class C(B) -reveal_type(A.__sealed_members__) # revealed: (type[B],) +reveal_type(A.__sealed_members__) # revealed: (*: type[B]) +assert A.__sealed_members__ == (B,) ``` -## a sealed class with no subclasses +## a sealed class declared in a function or class body + +The members are the direct subclasses declared alongside it, in the same body. + +```by +def make(): + sealed class A + class B(A) + class C(A) + reveal_type(A.__sealed_members__) # revealed: (*: type[B | C]) + return A + +class Outer: + sealed class X + class Y(X) + +reveal_type(Outer.X.__sealed_members__) # revealed: (*: type[Y]) +assert len(make().__sealed_members__) == 2 +assert Outer.X.__sealed_members__ == (Outer.Y,) +``` + +## subclasses declared in another scope are not members + +The tuple is added to in the scope that declares the sealed class, which cannot name a class +declared inside a nested function or class body. ```by sealed class A +class B(A) -reveal_type(A.__sealed_members__) # revealed: () +def make(): + class C(A): ... + return C + +class Holder: + class D(A): ... + +reveal_type(A.__sealed_members__) # revealed: (*: type[B]) +make() +assert A.__sealed_members__ == (B,) ``` -## only direct subclasses are members +## a subclass declared conditionally + +A subclass whose statement does not run is not in the tuple. One that has run is in it straight +away, so code after it in the same block reads it. + +```by +import sys + +flag = len(sys.argv) > 99 + +sealed class A +if flag: + class B(A) + assert A.__sealed_members__ == (B,) +class C(A) + +reveal_type(A.__sealed_members__) # revealed: (*: type[B | C]) +assert A.__sealed_members__[-1] == C +assert len(A.__sealed_members__) == (2 if flag else 1) +``` + +A function that returns before a subclass's statement runs has only the subclasses declared before +the `return` in its tuple. + +```by +def make(early: bool): + sealed class Shape + class Circle(Shape) + if early: + return Shape + class Square(Shape) + return Shape + +assert len(make(True).__sealed_members__) == 1 +assert len(make(False).__sealed_members__) == 2 +``` + +## a subclass declared more than once + +Each class statement adds the class it makes, so a name declared twice is in the tuple twice, once +for each class. ```by sealed class A class B(A) -class C(B) +first = B +class B(A) + +assert A.__sealed_members__ == (first, B) +``` + +## a subclass naming the sealed class another way + +A subclass is a member when its base is the sealed class, however the base is written. + +```by +sealed class A +Base = A +class B(Base) + +reveal_type(A.__sealed_members__) # revealed: (*: type[B]) +assert A.__sealed_members__ == (B,) +``` + +## a private sealed class + +```by +private sealed class A +class B(A) + +class Outer: + private sealed class X + class Y(X) + members = X.__sealed_members__ -reveal_type(A.__sealed_members__) # revealed: (type[B],) +reveal_type(A.__sealed_members__) # revealed: (*: type[B]) +assert A.__sealed_members__ == (B,) +assert Outer.members == (Outer.Y,) ``` ## subclassing from within the workspace is allowed diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index d85ee072a3..c0968c9a9b 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -119,7 +119,7 @@ bitflags::bitflags! { pub struct ClassLiteralFlags: u8 { /// basedpython: whether this class is declared `sealed`. A sealed class may /// only be subclassed from within its own workspace, and exposes a - /// `__sealed_members__` tuple of its same-module direct subclasses. + /// `__sealed_members__` tuple of the direct subclasses its own scope declares const SEALED = 1 << 0; /// Whether this class has any decorators. const HAS_DECORATORS = 1 << 1; @@ -1093,31 +1093,37 @@ impl<'db> StaticClassLiteral<'db> { }) } - /// basedpython: the module-level direct subclasses of this `sealed` class, - /// in source order. These make up the `__sealed_members__` tuple and match - /// what the runtime transform emits (it only sees same-file subclasses). + /// basedpython: the direct subclasses of this `sealed` class that the scope + /// binding it declares — the module for a module-level class, the function + /// or class body for one declared there — in source order, one per class + /// statement. these are the members of `__sealed_members__`, and the classes + /// the transpiler adds to it: it adds each one in that scope, right after its + /// class statement, and can name no subclass another scope declares + /// + /// the class statements are read off the scope's child scopes rather than + /// off what its names hold at the end of the scope: a function body that + /// ends in `return` has no reachable end, and a name a class statement binds + /// may be rebound after it, but the statement is still one that adds a member #[salsa::tracked(returns(deref), heap_size=ruff_memory_usage::heap_size)] pub(crate) fn sealed_members(self, db: &'db dyn Db) -> Box<[StaticClassLiteral<'db>]> { - let global = ty_python_core::global_scope(db, self.program_file(db)); - let mut members = Vec::new(); - for symbol in place_table(db, global).symbols() { - let Some(Type::ClassLiteral(ClassLiteral::Static(candidate))) = - class_member(db, global, symbol.name()).ignore_possibly_undefined() - else { - continue; - }; - if candidate == self { - continue; - } - let is_direct_subclass = candidate.explicit_bases(db).iter().any(|base| { - base.to_class_type(db) - .is_some_and(|class| class.class_literal(db) == ClassLiteral::Static(self)) - }); - if is_direct_subclass { - members.push(candidate); - } - } - members.into_boxed_slice() + let index = semantic_index(db, self.program_file(db)); + let scope = self.definition(db).file_scope(db); + index + .child_scopes(scope) + // a generic class's body is a child of its type-parameter scope, which + // is the one the class statement opens in the enclosing scope + .filter_map(|(_, child)| child.node().as_class_or_class_type_parameters()) + .filter_map(|class| original_class_type(db, index.expect_single_definition(class))) + .filter_map(ClassLiteral::as_static) + .filter(|candidate| { + *candidate != self + && candidate.explicit_bases(db).iter().any(|base| { + base.to_class_type(db).is_some_and(|class| { + class.class_literal(db) == ClassLiteral::Static(self) + }) + }) + }) + .collect() } /// Is this class final? @@ -1997,17 +2003,27 @@ impl<'db> StaticClassLiteral<'db> { inherited_generic_context: Option>, name: &str, ) -> Option> { - // basedpython: a `sealed` class exposes `__sealed_members__`, a tuple of - // its same-module direct subclasses (matching the runtime transform). + // basedpython: a `sealed` class exposes `__sealed_members__`, the tuple the + // transpiler adds each of its members to as the member's class statement runs. + // a statement may run conditionally, more than once, or not at all, and the tuple + // can be read before a later one has, so it holds any number of them if name == "__sealed_members__" && self.is_sealed(db) { - let elements = self.sealed_members(db).iter().map(|member| { - SubclassOfType::from( - db, - env, - ClassLiteral::Static(*member).default_specialization(db), - ) - }); - return Some(Type::heterogeneous_tuple(db, env, elements)); + let members = self.sealed_members(db); + if members.is_empty() { + return Some(Type::empty_tuple(db, env)); + } + let member = UnionType::from_elements( + db, + env, + members.iter().map(|member| { + SubclassOfType::from( + db, + env, + ClassLiteral::Static(*member).default_specialization(db), + ) + }), + ); + return Some(Type::homogeneous_tuple(db, env, member)); } // Handle `@functools.total_ordering`: synthesize comparison methods diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 09bd52a7f9..9d8c81d471 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -5261,6 +5261,41 @@ pub fn is_union_special_form(ty: Type) -> bool { matches!(ty, Type::SpecialForm(SpecialFormType::Union)) } +/// basedpython: the `sealed` classes whose `__sealed_members__` the class `class_def` declares +/// is a member of, each by the range of its name, which is in the same file and scope +/// +/// the transpiler adds the class to each of their tuples as its statement runs, and the checker +/// types the tuples from the same members, so the two agree on which classes are members however +/// a base is written +pub fn sealed_classes_listing( + model: &SemanticModel<'_>, + class_def: &ast::StmtClassDef, +) -> Vec { + let db = model.db(); + let file = db.program_file(model.file()); + let index = semantic_index(db, file); + let Some(ClassLiteral::Static(class)) = + original_class_type(db, index.expect_single_definition(class_def)) + else { + return Vec::new(); + }; + let module = parsed_module(db, file.python_file(db)).load(db); + let mut sealed: Vec = Vec::new(); + for base in class.explicit_bases(db) { + if let Some(base) = base.to_class_type(db) + && let ClassLiteral::Static(base) = base.class_literal(db) + && base.is_sealed(db) + && base.sealed_members(db).contains(&class) + { + let name = base.definition(db).focus_range(db, &module).range(); + if !sealed.contains(&name) { + sealed.push(name); + } + } + } + sealed +} + #[cfg(test)] mod tests { use super::{ diff --git a/docs/basedpython/features/modifiers.md b/docs/basedpython/features/modifiers.md index 23319c7198..453b701c55 100644 --- a/docs/basedpython/features/modifiers.md +++ b/docs/basedpython/features/modifiers.md @@ -36,15 +36,15 @@ ORIGIN: Final = Point(0, 0) ## class modifiers -| basedpython | Python output | -| ----------------------- | --------------------------------------------------- | -| `final class Foo` | `@final` + `class Foo` | -| `abstract class Foo` | `class Foo` (keyword stripped, no decorator) | -| `open class Foo` | `class Foo` (keyword stripped, no decorator) | -| `data class Foo` | `@dataclass(slots=True)` + `class Foo` | -| `frozen data class Foo` | `@dataclass(frozen=True, slots=True)` + `class Foo` | -| `protocol Foo` | `class Foo(Protocol)` (base added) | -| `sealed class Foo` | `class Foo` + `Foo.__sealed_members__ = (...)` | +| basedpython | Python output | +| ----------------------- | --------------------------------------------------------- | +| `final class Foo` | `@final` + `class Foo` | +| `abstract class Foo` | `class Foo` (keyword stripped, no decorator) | +| `open class Foo` | `class Foo` (keyword stripped, no decorator) | +| `data class Foo` | `@dataclass(slots=True)` + `class Foo` | +| `frozen data class Foo` | `@dataclass(frozen=True, slots=True)` + `class Foo` | +| `protocol Foo` | `class Foo(Protocol)` (base added) | +| `sealed class Foo` | `class Foo` + `Foo.__sealed_members__` built per subclass | `dataclass(slots=True)` is new in python 3.10. below it, a `data class` is decorated with a runtime helper that makes it slotted the way that option would. diff --git a/docs/basedpython/features/sealed-classes.md b/docs/basedpython/features/sealed-classes.md index 4fb12f8b00..4039e6cc34 100644 --- a/docs/basedpython/features/sealed-classes.md +++ b/docs/basedpython/features/sealed-classes.md @@ -17,28 +17,72 @@ transpiles to: ```python class Shape: ... +Shape.__sealed_members__ = () class Circle(Shape): ... +Shape.__sealed_members__ += (Circle,) class Square(Shape): ... -Shape.__sealed_members__ = (Circle, Square) +Shape.__sealed_members__ += (Square,) ``` ## `__sealed_members__` -each sealed class gains a `__sealed_members__` attribute: a tuple of its -same-module direct subclasses, in source order. the transpiler materializes the -tuple at runtime by emitting an assignment after the last subclass, as above +each sealed class gains a `__sealed_members__` attribute: a tuple of the direct +subclasses declared in the same scope as it. the tuple starts empty, right after +the sealed class's own statement, and each subclass is added right after its own +class statement, in the block it is written in, as above. so the tuple holds the +subclasses whose statements have run, in the order they ran: one written under an +`if` that is not taken, or in a loop, is in it once its statement has run ```by sealed class Shape class Circle(Shape) class Square(Shape) -reveal_type(Shape.__sealed_members__) # (type[Circle], type[Square]) +reveal_type(Shape.__sealed_members__) # (*: type[Circle | Square]) ``` -only direct, same-module subclasses are members — a subclass of a subclass, or a -subclass defined in another module, is not listed. this matches what the runtime -transform can see (it only ever sees one file) +the checker does not know the tuple's length, since which subclass statements have +run is not known statically, so it types it as any number of the subclasses + +only direct subclasses declared in the same scope are members — a subclass of a +subclass, a subclass declared inside a function or class body nested in that +scope, or a subclass defined in another module, is not listed. this matches what +the runtime transform can name: it only ever sees one file, and builds the tuple +in the scope that declares the sealed class + +a sealed class declared in a function or class body gets its tuple there, from +the subclasses declared alongside it: + +```by +def make(): + sealed class Shape + class Circle(Shape) + class Square(Shape) + return Shape + +class Shapes: + sealed class Shape + class Circle(Shape) +``` + +transpiles to: + +```python +def make(): + class Shape: ... + Shape.__sealed_members__ = () + class Circle(Shape): ... + Shape.__sealed_members__ += (Circle,) + class Square(Shape): ... + Shape.__sealed_members__ += (Square,) + return Shape + +class Shapes: + class Shape: ... + Shape.__sealed_members__ = () + class Circle(Shape): ... + Shape.__sealed_members__ += (Circle,) +``` ## workspace boundary @@ -67,5 +111,5 @@ class Circle(Shape): ... # error: [subclass-of-sealed-class] ## transpilation `sealed` is a forward-only transform: the modifier keyword is stripped and the -`__sealed_members__` assignment is inserted. there is no reverse transform, so +`__sealed_members__` assignments are inserted. there is no reverse transform, so round-tripping python back to basedpython does not recover the `sealed` keyword From 4753b239c3e615febfc48da1d27588f49aa8a267 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:11:22 +1000 Subject: [PATCH 083/110] send a workspace diagnostic request again from the ty_server test client when the server cancels it with retriggerRequest, as a client does, rather than failing the test: a change the server's own watcher takes in while it checks the workspace cancels the check, which under load overtook a pull in build_trees::a_build_written_while_the_server_watches_does_not_join_the_project, including for a write the test made before the server started that macOS reported late --- crates/ty_server/tests/e2e/main.rs | 32 ++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index d09fc2a172..cd2fe7bd3f 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -201,6 +201,18 @@ pub(crate) enum AwaitResponseError { DeserializationError(#[from] serde_json::Error), } +/// whether the server cancelled a diagnostic request and asks the client to send it again +fn asks_to_retrigger(error: &ResponseError) -> bool { + error.code == lsp_server::ErrorCode::ServerCancelled as i32 + && error + .data + .clone() + .and_then(|data| { + serde_json::from_value::(data).ok() + }) + .is_some_and(|data| data.retrigger_request) +} + impl From for AwaitResponseError { fn from(err: ReceiveError) -> Self { match err { @@ -1145,6 +1157,14 @@ impl TestServer { } /// Send a `workspace/diagnostic` request with optional previous result IDs. + /// + /// a change the server takes in while it checks the workspace cancels the check, and the + /// server answers `ServerCancelled` with `retriggerRequest: true`, which is it asking the + /// client to send the request again. this does, as a client does. a test that watches the + /// file system meets it whenever a pull overlaps a change the watcher reports, one the test + /// made or one the operating system reports late for a file the test wrote before the server + /// started + #[track_caller] pub(crate) fn workspace_diagnostic_request( &mut self, work_done_token: Option, @@ -1157,8 +1177,16 @@ impl TestServer { partial_result_params: PartialResultParams::default(), }; - let id = self.send_request::(params); - self.await_response::(&id) + loop { + let id = self.send_request::(params.clone()); + match self.try_await_response::(&id, None) { + Ok(report) => return report, + Err(AwaitResponseError::RequestFailed(error)) if asks_to_retrigger(&error) => { + tracing::debug!("Retriggering workspace diagnostic request {id}"); + } + Err(error) => panic!("Failed to receive response for request {id}: {error}"), + } + } } /// Send a `textDocument/definition` request for the document at the given path and position. From 7df3f1b271f7d2719018c497dc6c67cce77d5513 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:16:31 +1000 Subject: [PATCH 084/110] highlight a final written in an annotation, x: final float, where it is written, after the target, rather than ahead of the target as a declaration keyword is, which added the tokens out of file order and panicked semantic tokens on the class attribute --- crates/ty_ide/src/semantic_tokens.rs | 51 +++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 2845bb939c..b82fd52289 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -2075,13 +2075,19 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.visit_decorator(decorator); } - // A basedpython declaration keyword is written ahead of its - // target, so it is emitted before it rather than in annotation - // position, where the marker standing in for it sits. + // a basedpython declaration keyword is mostly written ahead of + // its target (`final x: float`), and is then emitted before it + // rather than in annotation position, where the marker standing + // in for it sits. `final` is also a type form, written in the + // annotation (`x: final float`), and is emitted there: tokens are + // added in the order they are written let declaration = declaration_keyword(&assignment.annotation); - if let Some((keyword, _)) = declaration { + let keyword = declaration.map(|(keyword, _)| self.keyword_range(keyword)); + let ahead_of_target = + |keyword: &TextRange| keyword.start() < assignment.target.start(); + if let Some(keyword) = keyword.filter(ahead_of_target) { self.add_token( - self.keyword_range(keyword), + keyword, SemanticTokenType::Keyword, SemanticTokenModifier::empty(), ); @@ -2091,6 +2097,14 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.visit_expr(&assignment.target); self.in_target_creating_definition = false; + if let Some(keyword) = keyword.filter(|keyword| !ahead_of_target(keyword)) { + self.add_token( + keyword, + SemanticTokenType::Keyword, + SemanticTokenModifier::empty(), + ); + } + match declaration { // only the type the user actually spelled is an annotation; // the marker itself is keyword text @@ -8178,6 +8192,33 @@ final c: str = \"x\" "#); } + #[test] + fn semantic_tokens_final_type_form_in_an_annotation() { + // `final` written in the annotation is a type form, not a declaration + // keyword ahead of the target, and is highlighted where it is written + let test = SemanticTokenTest::new_by( + " +class A: + x: final float = 2.0 + y: list[final float] = [] +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "A" @ 7..8: Class [definition] + "x" @ 14..15: Variable [definition] + "final" @ 17..22: Keyword + "float" @ 23..28: Class + "2.0" @ 31..34: Number + "y" @ 39..40: Variable [definition] + "list" @ 42..46: Class + "final" @ 47..52: Keyword + "float" @ 53..58: Class + "#); + } + #[test] fn semantic_tokens_class_variable_keywords() { // `class var x: T` / `class let x: T` declare through the same marker, From f3178238bc8f3f777e8099e04eef21d2fd5b2587 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:19:06 +1000 Subject: [PATCH 085/110] pull bases up into a superclass whose body is only pass without touching that body, rather than replacing its pass with the no members pulled alongside them, which panicked by/pullMembersUp and would have left the superclass without a body --- crates/ty_ide/src/refactor/class_members.rs | 1329 ++++++++++++++--- .../src/refactor/class_members/bases.rs | 162 +- crates/ty_ide/src/refactor/flow.rs | 16 +- crates/ty_ide/src/refactor/move_symbol.rs | 390 +++-- crates/ty_python_semantic/src/importer.rs | 4 +- .../src/types/ide_support.rs | 115 +- .../src/server/api/requests/move_members.rs | 59 +- crates/ty_server/tests/e2e/move_symbol.rs | 66 + 8 files changed, 1706 insertions(+), 435 deletions(-) diff --git a/crates/ty_ide/src/refactor/class_members.rs b/crates/ty_ide/src/refactor/class_members.rs index 9d9da9d56e..b5a6ed6594 100644 --- a/crates/ty_ide/src/refactor/class_members.rs +++ b/crates/ty_ide/src/refactor/class_members.rs @@ -18,16 +18,22 @@ //! meant: //! //! - pulled up, a member that reads an attribute of its receiver the superclass does not have, -//! which the superclass's other subclasses would then miss; that uses `super()` from a class -//! whose base is not the superclass, since `super()` would then start from elsewhere; that -//! another subclass of the superclass inherits from further up already, and would inherit the -//! pulled one instead; or that the superclass binds already +//! which the superclass's other subclasses would then miss; that reads, where the class body +//! evaluates it — a value, a default, a decorator — a name the class body binds and that is +//! not pulled with it, or a name bound after the superclass is defined; that uses `super()` +//! from a class whose base is not the superclass, since `super()` would then start from +//! elsewhere; that a class below the superclass would then find in another class than it does, +//! by its method resolution order — another subclass that finds it in a mixin or above the +//! superclass, or the class itself, in a class it finds before the superclass; or that the +//! superclass binds already //! - pushed down, a member something reads off the class itself or an instance that may be the //! class rather than a subclass — another of its methods, through `self`, included — or of a //! class without subclasses in the project //! - either way, a name-mangled `__member`, whose name depends on the class it is written in, -//! and, between modules, a member that reads a name of its module that the other would have -//! to import from a module that imports it +//! a method that reads `__class__`, which is the class it is written in, a member that reads a +//! name one class finds in a scope around it — a function it is defined in, say — and the +//! other finds elsewhere, and, between modules, a member that reads a name of its module that +//! the other would have to import from a module that imports it //! //! `override` follows the move: a method pulled up overrides nothing where it lands unless the //! superclass inherits the name, so it loses the modifier there, and the methods of the same @@ -40,7 +46,7 @@ use ruff_db::source::source_text; use ruff_diagnostics::Edit; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor, walk_expr}; -use ruff_python_ast::{self as ast, Expr, Stmt}; +use ruff_python_ast::{self as ast, Expr, ExprContext, Stmt}; use ruff_python_codegen::Stylist; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -48,20 +54,27 @@ use rustc_hash::FxHashSet; use ty_module_resolver::file_to_module; use ty_project::Db; use ty_python_core::ProgramFile; +use ty_python_core::scope::{FileScopeId, NodeWithScopeRef}; +use ty_python_core::semantic_index; use ty_python_semantic::importer::{ImportAction, ImportRequest, MembersInScope}; use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::{ + decorated_function, is_enum_class, is_override_function, is_super, reads_class_cell, +}; use ty_python_semantic::{ - HasType, ProgramEnvironment, SemanticModel, class_bases, instance_member_names, + HasDefinition, HasType, ProgramEnvironment, SemanticModel, class_bases, instance_member_names, is_within_classes, type_hierarchy_prepare, type_hierarchy_subtypes, }; +use super::flow::{bindings_of_symbol, resolving_scope, resolving_scope_among}; use super::move_symbol::{ - Dependency, OwnedRequest, SymbolMove, cut_range, dependencies, imports_module, line_number, + Dependency, OwnedRequest, SymbolMove, checked_dependencies, cut_range, imports_module, + line_number, loads, }; use crate::code_action::FileEdit; use crate::goto::find_goto_target; use crate::import_order; -use crate::method_kind::{MethodKind, method_kind}; +use crate::method_kind::{MethodKind, resolved_method_kind}; use crate::references::{ReferencesMode, references}; mod bases; @@ -74,20 +87,20 @@ pub enum MemberKind { StaticMethod, Property, Attribute, - /// A base the class statement names, which a pull up moves into the superclass's bases. + /// a base the class statement names, which a pull up moves into the superclass's bases Base, } -/// A member [`pull_members_up`] or [`push_members_down`] can move, or a base a pull up can. +/// a member [`pull_members_up`] or [`push_members_down`] can move, or a base a pull up can #[derive(Debug, Clone, PartialEq, Eq)] pub struct ClassMember { - /// Its name, or a base as the class statement writes it. + /// its name, or a base as the class statement writes it pub name: String, pub kind: MemberKind, - /// The range of its name in its first statement, or of the base. + /// the range of its name in its first statement, or of the base pub name_range: TextRange, - /// How it reads in a list of members: a method or property as its name and its parameters - /// as its first statement writes them, `fetch(self, key: str)`, an attribute as its name. + /// how it reads in a list of members: a method or property as its name and its parameters + /// as its first statement writes them, `fetch(self, key: str)`, an attribute as its name pub detail: String, } @@ -123,9 +136,11 @@ pub fn movable_members( let class = ClassSite::at(db, file, offset)?; let def = class.def()?; let source = source_text(db, class.file_id); + let model = SemanticModel::new(db, class.file); let mut members: Vec = Vec::new(); for statement in &def.body { - let Some((name, kind)) = member_of(statement) else { + let (Some(name), Some(kind)) = (member_name(statement), member_kind(&model, statement)) + else { continue; }; if let Some(existing) = members @@ -145,7 +160,7 @@ pub fn movable_members( }); } members.extend( - bases::written_bases(def, source.as_str()) + bases::written_bases(db, &class, def, source.as_str()) .into_iter() .map(|base| ClassMember { name: base.text.to_string(), @@ -170,13 +185,15 @@ pub fn movable_members( }) } -/// Moves the members `names` of the class around `offset` into its superclass `target`, named by -/// its qualified name as [`movable_members`] lists it. `Err` says why they cannot be moved. +/// moves the members `names` and the bases `bases`, each as the class statement writes it, of the +/// class around `offset` into its superclass `target`, named by its qualified name as +/// [`movable_members`] lists it. `Err` says why they cannot be moved pub fn pull_members_up( db: &dyn Db, file: ProgramFile<'_>, offset: TextSize, names: &[String], + bases: &[String], target: &str, ) -> Result { let class = @@ -201,28 +218,24 @@ pub fn pull_members_up( let mut super_members = instance_member_names(db, &env, superclass.ty); let source = source_text(db, class.file.file(db)); let source = source.as_str(); + let model = SemanticModel::new(db, class.file); + let super_model = SemanticModel::new(db, superclass.file); - // a name the class body binds is a member; any other is one of the bases it writes - let (names, pulled_bases): (Vec<&String>, Vec<&String>) = names - .iter() - .partition(|name| !member_statements(def, name).is_empty()); let mut moved = SymbolMove::default(); - if !pulled_bases.is_empty() { + if !bases.is_empty() { let pulled = bases::pull_bases_up( db, &class, superclass, - &pulled_bases - .iter() - .map(|name| name.as_str()) - .collect::>(), + &bases.iter().map(String::as_str).collect::>(), source, )?; // what the bases bring is the superclass's from now on, for the members pulled with them super_members.extend(pulled.gained); moved.edits.extend(pulled.edits); } - let pulled: FxHashSet<&str> = names.iter().map(|name| name.as_str()).collect(); + let pulled: FxHashSet<&str> = names.iter().map(String::as_str).collect(); + let below_superclass = superclass.descendants(db); let mut texts: Vec<(usize, MemberKind, String)> = Vec::new(); let mut removed: Vec = Vec::new(); @@ -238,31 +251,17 @@ pub fn pull_members_up( .body .iter() .any(|statement| binds(statement, name)) - || assigns_on_receiver(super_def, name) + || assigns_on_receiver(&super_model, super_def, name)? { return Err(format!("{} has a `{name}` already", superclass.name)); } let inherited_above = super_members.contains(&Name::new(name)); - // a class between the two that has the name would be what the class inherits instead - if let Some(between) = ancestors - .iter() - .take_while(|ancestor| !ancestor.same(superclass)) - .find(|ancestor| { - superclass.is_ancestor_of(db, ancestor) - && ancestor - .def() - .is_some_and(|def| !member_statements(def, name).is_empty()) - }) - { - return Err(format!( - "{} would inherit {}'s `{name}` rather than its own", - class.name, between.name - )); - } + bases::refuse_changed_lookups(db, &class, superclass, &below_superclass, name)?; for &index in &statements { let statement = &def.body[index]; + let kind = member_kind(&model, statement).ok_or_else(|| unknown_kind(name))?; // what it reads off its receiver has to be there in the superclass - for (attribute, range) in receiver_attributes(statement) { + for (attribute, range) in receiver_attributes(&model, statement)? { if pulled.contains(attribute.as_str()) || super_members.contains(&attribute) { continue; } @@ -279,23 +278,89 @@ pub fn pull_members_up( if is_member { "; pull it up too" } else { "" } )); } - for (sibling, _) in class_scope_reads(def, statement) { - if !pulled.contains(sibling.as_str()) && !super_members.contains(&sibling) { - return Err(format!( - "`{name}` reads {}'s `{sibling}`, which {} does not have; pull it up too", - class.name, superclass.name - )); + for read in class_body_reads(db, &class, def, statement) { + match read { + BodyRead::Member { + name: sibling, + index: read_index, + } => { + if !pulled.contains(sibling.as_str()) { + return Err(format!( + "`{name}` reads {}'s `{sibling}`, which it would no longer read \ + in {}'s body; pull it up too", + class.name, superclass.name + )); + } + // an attribute lands before the superclass's methods, and a method + // after them + if is_attribute(statement) + && read_index + .is_some_and(|read_index| !is_attribute(&def.body[read_index])) + { + return Err(format!( + "`{name}` reads `{sibling}`, which {} would define after it, \ + among its methods", + superclass.name + )); + } + } + BodyRead::Outside { + name: read, + resolves_to, + } => { + if superclass.file_id != class.file_id { + // what another module would have to import is asked below + continue; + } + let index = semantic_index(db, class.file); + let super_scope = index + .try_node_scope(NodeWithScopeRef::Class(super_def)) + .ok_or_else(|| format!("{} cannot be read", superclass.name))?; + if resolving_scope(index, super_scope, read.as_str()) != resolves_to { + return Err(format!( + "`{name}` reads `{read}`, which is another name where {} is \ + defined", + superclass.name + )); + } + if let Some(scope) = resolves_to + && bound_between(db, &class, scope, read.as_str(), super_def, def) + { + return Err(format!( + "`{name}` reads `{read}`, which is bound after {} is defined", + superclass.name + )); + } + } } } - if uses_super(statement) && !class.has_only_base(db, superclass) { + if let Some(read) = + read_rebound_around(db, &class, def, statement, superclass, super_def) + { + return Err(format!( + "`{name}` reads `{read}`, which is another name where {} is defined", + superclass.name + )); + } + if uses_super(&model, statement) && !class.has_only_base(db, superclass) { return Err(format!( "`{name}` calls `super()`, which would start from {} rather than from {}", superclass.name, class.name )); } + if member_reads_class_cell(&model, statement) { + return Err(class_cell_refusal(name, &superclass.name, &class.name)); + } if superclass.file_id != class.file_id { - let parsed = &class.parsed; - for dependency in dependencies(db, class.file, parsed, statement, name) { + for dependency in checked_dependencies( + db, + class.file, + &class.parsed, + statement, + name, + None, + superclass.file, + )? { match dependency { Dependency::Imported { request, .. } => target_imports.push(request), Dependency::Sibling(sibling) => { @@ -311,31 +376,17 @@ pub fn pull_members_up( } texts.push(( index, - member_kind(statement), + kind, member_text( source, &def.body, index, - !inherited_above && has_override(statement), + override_marker(db, &class, statement).filter(|_| !inherited_above), ), )); removed.push(index); } - if inherited_above { - // another subclass of the superclass that inherits it from further up would - // inherit this one instead - for below in superclass.descendants(db) { - if below.same(&class) || class.is_ancestor_of(db, &below) { - continue; - } - if !below.inherits_own(db, name, superclass) { - return Err(format!( - "{} inherits `{name}` from above {}, and would inherit {}'s instead", - below.name, superclass.name, class.name - )); - } - } - } else { + if !inherited_above { overriding.push(name.clone()); } } @@ -377,7 +428,7 @@ pub fn pull_members_up( } if let Some(below_def) = below.def() && let Some(&first) = member_statements(below_def, name).first() - && !has_override(&below_def.body[first]) + && override_marker(db, &below, &below_def.body[first]).is_none() { moved .edits @@ -474,7 +525,14 @@ pub fn push_members_down( for &index in &statements { let statement = &def.body[index]; - if uses_super(statement) { + if member_reads_class_cell(&model, statement) { + return Err(class_cell_refusal( + name, + "the subclass it lands in", + &class.name, + )); + } + if uses_super(&model, statement) { for subclass in &subclasses { if !subclass.has_only_base(db, &class) { return Err(format!( @@ -485,12 +543,12 @@ pub fn push_members_down( } } texts.push(( - member_kind(statement), + member_kind(&model, statement).ok_or_else(|| unknown_kind(name))?, member_text( source, &def.body, index, - !inherited_above && has_override(statement), + override_marker(db, &class, statement).filter(|_| !inherited_above), ), vec![index], )); @@ -514,23 +572,39 @@ pub fn push_members_down( let mut imports = Vec::new(); for (kind, text, indices) in &texts { let statement = &def.body[indices[0]]; - let Some((name, _)) = member_of(statement) else { + let Some(name) = member_name(statement) else { continue; }; let own = member_statements(sub_def, name.as_str()); if let Some(&first) = own.first() { // it keeps its own, which overrides nothing once the class no longer has it let own_statement = &sub_def.body[first]; - if !inherited.contains(name) && has_override(own_statement) { + if !inherited.contains(name) + && override_marker(db, subclass, own_statement).is_some() + { moved .edits .extend(remove_override(db, subclass, own_statement)); } continue; } + if let Some(read) = read_rebound_around(db, &class, def, statement, subclass, sub_def) { + return Err(format!( + "`{name}` reads `{read}`, which is another name where {} is defined", + subclass.name + )); + } copies.push((*kind, reindent(text, &from_indent, &to_indent))); if subclass.file_id != class.file_id { - for dependency in dependencies(db, class.file, &class.parsed, statement, name) { + for dependency in checked_dependencies( + db, + class.file, + &class.parsed, + statement, + name, + None, + subclass.file, + )? { match dependency { Dependency::Imported { request, .. } => imports.push(request), Dependency::Sibling(sibling) => { @@ -579,7 +653,8 @@ pub fn push_members_down( /// the class's bases and its members `names`, and makes it the class's one base. the class's /// method resolution order is what it was with the new class inserted right after it, so every /// member is found where it was found, and `super()` in a moved method starts where it started. -/// `Err` says why it cannot be extracted. +/// a class that passes a keyword other than `metaclass` when it is made is refused, since the +/// keyword would be passed for the new class instead. `Err` says why it cannot be extracted pub fn extract_superclass( db: &dyn Db, file: ProgramFile<'_>, @@ -613,6 +688,8 @@ pub fn extract_superclass( class.module(db).unwrap_or_default() )); } + let source = source_text(db, class.file.file(db)); + let source = source.as_str(); let env = ProgramEnvironment::from_file(class.file); let bases = class_bases(db, &env, class.ty); // a base that is a special form rather than a class — `Protocol`, `NamedTuple`, `TypedDict`, @@ -622,25 +699,34 @@ pub fn extract_superclass( .arguments .as_ref() .map_or(0, |arguments| arguments.args.len()); - if bases.len() != written_bases - || bases.iter().any(|base| { - instance_member_names(db, &env, *base).contains(&Name::new_static("_member_map_")) - }) - { + if bases.len() != written_bases || bases.iter().any(|base| is_enum_class(db, &env, *base)) { return Err(format!( "{} has a base whose subclasses python treats specially, which {name} would take \ in its place", class.name )); } + // a keyword is passed to `__init_subclass__` and the metaclass for the class it is written on + // alone, while a metaclass the new class is made by makes the class too + if let Some(keyword) = def.arguments.as_ref().and_then(|arguments| { + arguments + .keywords + .iter() + .find(|keyword| keyword.arg.as_ref().is_none_or(|arg| arg != "metaclass")) + }) { + return Err(format!( + "{} passes `{}` when it is made, which {name} would be passed instead", + class.name, + &source[keyword.range()] + )); + } // what the new class has: everything the class's bases give it, and what moves into it let inherited: FxHashSet = bases .into_iter() .flat_map(|base| instance_member_names(db, &env, base)) .collect(); let extracted: FxHashSet<&str> = names.iter().map(String::as_str).collect(); - let source = source_text(db, class.file.file(db)); - let source = source.as_str(); + let model = SemanticModel::new(db, class.file); let mut texts: Vec<(usize, MemberKind, String)> = Vec::new(); let mut removed: Vec = Vec::new(); @@ -652,12 +738,13 @@ pub fn extract_superclass( } for &index in &statements { let statement = &def.body[index]; - if !def.decorator_list.is_empty() && member_kind(statement) == MemberKind::Attribute { + if !def.decorator_list.is_empty() && is_attribute(statement) { return Err(format!( - "`{member}` is an attribute of a decorated class, which its decorator may read as a field" + "`{member}` is an attribute of a decorated class, which its decorator may \ + read as a field" )); } - for (attribute, range) in receiver_attributes(statement) { + for (attribute, range) in receiver_attributes(&model, statement)? { if extracted.contains(attribute.as_str()) || inherited.contains(&attribute) { continue; } @@ -673,19 +760,26 @@ pub fn extract_superclass( if is_member { "; extract it too" } else { "" } )); } - for (sibling, _) in class_scope_reads(def, statement) { - if !extracted.contains(sibling.as_str()) && !inherited.contains(&sibling) { + if member_reads_class_cell(&model, statement) { + return Err(class_cell_refusal(member, name, &class.name)); + } + // the new class is written right above the class, so a name read from outside the + // class body is the same name there + for read in class_body_reads(db, &class, def, statement) { + if let BodyRead::Member { name: sibling, .. } = read + && !extracted.contains(sibling.as_str()) + { return Err(format!( - "`{member}` reads {}'s `{sibling}`, which {name} would not have; \ - extract it too", + "`{member}` reads {}'s `{sibling}`, which it would no longer read in \ + {name}'s body; extract it too", class.name )); } } texts.push(( index, - member_kind(statement), - member_text(source, &def.body, index, false), + member_kind(&model, statement).ok_or_else(|| unknown_kind(member))?, + member_text(source, &def.body, index, None), )); removed.push(index); } @@ -946,26 +1040,6 @@ impl<'db> ClassSite<'db> { } found } - - /// Whether this class, or a class between it and `top`, defines `name` itself, so that it - /// does not inherit `name` from `top` or above. - fn inherits_own(&self, db: &'db dyn Db, name: &str, top: &ClassSite<'db>) -> bool { - if self - .def() - .is_some_and(|def| !member_statements(def, name).is_empty()) - { - return true; - } - self.ancestors(db) - .iter() - .take_while(|ancestor| !ancestor.same(top)) - .any(|ancestor| { - top.is_ancestor_of(db, ancestor) - && ancestor - .def() - .is_some_and(|def| !member_statements(def, name).is_empty()) - }) - } } /// every class definition of a module, at any depth @@ -988,27 +1062,32 @@ fn all_classes(parsed: &ParsedModuleRef) -> Vec<&ast::StmtClassDef> { classes.found } -/// The member a class body statement binds, and what kind of member it is. -fn member_of(statement: &Stmt) -> Option<(&Name, MemberKind)> { +/// the member a class body statement binds +fn member_name(statement: &Stmt) -> Option<&Name> { match statement { - Stmt::FunctionDef(function) => Some((&function.name.id, member_kind(statement))), + Stmt::FunctionDef(function) => Some(&function.name.id), Stmt::Assign(assign) => match assign.targets.as_slice() { - [Expr::Name(name)] => Some((&name.id, MemberKind::Attribute)), + [Expr::Name(name)] => Some(&name.id), _ => None, }, Stmt::AnnAssign(assign) => match assign.target.as_ref() { - Expr::Name(name) => Some((&name.id, MemberKind::Attribute)), + Expr::Name(name) => Some(&name.id), _ => None, }, _ => None, } } -/// See [`ClassMember::detail`]. Each parameter is written as the source writes it, but on one -/// line, with the `/` and `*` that separate the kinds. +/// whether a class body statement binds an attribute rather than a method +fn is_attribute(statement: &Stmt) -> bool { + !matches!(statement, Stmt::FunctionDef(_)) +} + +/// see [`ClassMember::detail`]. each parameter is written as the source writes it, but on one +/// line, with the `/` and `*` that separate the kinds fn member_detail(source: &str, statement: &Stmt) -> String { let Stmt::FunctionDef(function) = statement else { - return member_of(statement).map_or_else(String::new, |(name, _)| name.to_string()); + return member_name(statement).map_or_else(String::new, ToString::to_string); }; let written = |range: TextRange| { source[range] @@ -1043,23 +1122,25 @@ fn member_detail(source: &str, statement: &Stmt) -> String { format!("{}({})", function.name, parts.join(", ")) } -fn member_kind(statement: &Stmt) -> MemberKind { +/// what kind of member the class body statement `statement` of `model`'s file binds: a method +/// is a property where its decorators bind its name to one, and is passed what its decorators +/// say as the checker resolves them. `None` where the checker has no function for a `def` +fn member_kind(model: &SemanticModel<'_>, statement: &Stmt) -> Option { let Stmt::FunctionDef(function) = statement else { - return MemberKind::Attribute; + return Some(MemberKind::Attribute); }; - let is_property = function.decorator_list.iter().any(|decorator| { - matches!(&decorator.expression, Expr::Name(name) if name.id.as_str() == "property") - || matches!(&decorator.expression, Expr::Attribute(attribute) - if matches!(attribute.attr.as_str(), "setter" | "getter" | "deleter")) - }); - if is_property { - return MemberKind::Property; + if decorated_function(model.db(), function.definition(model))?.is_property() { + return Some(MemberKind::Property); } - match method_kind(function) { + Some(match resolved_method_kind(model, function)? { MethodKind::Instance => MemberKind::Method, MethodKind::Class => MemberKind::ClassMethod, MethodKind::Static => MemberKind::StaticMethod, - } + }) +} + +fn unknown_kind(name: &str) -> String { + format!("the checker cannot tell what kind of method `{name}` is") } fn member_name_range(statement: &Stmt) -> Option { @@ -1072,7 +1153,7 @@ fn member_name_range(statement: &Stmt) -> Option { } fn binds(statement: &Stmt, name: &str) -> bool { - member_of(statement).is_some_and(|(bound, _)| bound.as_str() == name) + member_name(statement).is_some_and(|bound| bound.as_str() == name) } /// the indices of the statements of `class`'s body that bind `name` @@ -1095,18 +1176,25 @@ fn refuse_mangled(name: &str) -> Result<(), String> { Ok(()) } -/// Whether a method of `class` assigns `name` on its receiver: `self.name = ...`. -fn assigns_on_receiver(class: &ast::StmtClassDef, name: &str) -> bool { - class.body.iter().any(|statement| { - receiver_attributes(statement) - .iter() - .any(|(attribute, _)| attribute.as_str() == name) - && matches!(statement, Stmt::FunctionDef(_)) - && assigned_on_receiver(statement, name) - }) +/// whether a method of `class`, a class of `model`'s file, assigns `name` on its receiver: +/// `self.name = ...` +fn assigns_on_receiver( + model: &SemanticModel<'_>, + class: &ast::StmtClassDef, + name: &str, +) -> Result { + for statement in &class.body { + if let Stmt::FunctionDef(function) = statement + && let Some(receiver) = receiver_name(model, function)? + && assigned_on_receiver(statement, receiver.as_str(), name) + { + return Ok(true); + } + } + Ok(false) } -fn assigned_on_receiver(statement: &Stmt, name: &str) -> bool { +fn assigned_on_receiver(statement: &Stmt, receiver: &str, name: &str) -> bool { struct Stores<'a> { receiver: &'a str, name: &'a str, @@ -1124,11 +1212,8 @@ fn assigned_on_receiver(statement: &Stmt, name: &str) -> bool { walk_expr(self, expr); } } - let Some(receiver) = receiver_name(statement) else { - return false; - }; let mut stores = Stores { - receiver: receiver.as_str(), + receiver, name, found: false, }; @@ -1136,26 +1221,32 @@ fn assigned_on_receiver(statement: &Stmt, name: &str) -> bool { stores.found } -/// The name of a method's receiver parameter, `self` or `cls`; `None` for a static method or -/// what is no method. -fn receiver_name(statement: &Stmt) -> Option { - let Stmt::FunctionDef(function) = statement else { - return None; - }; - if method_kind(function) == MethodKind::Static { - return None; - } - function +/// the name of the receiver parameter of `function`, a method of `model`'s file, `self` or +/// `cls`; `None` for a static method +fn receiver_name( + model: &SemanticModel<'_>, + function: &ast::StmtFunctionDef, +) -> Result, String> { + let kind = resolved_method_kind(model, function) + .ok_or_else(|| unknown_kind(function.name.as_str()))?; + if kind == MethodKind::Static { + return Ok(None); + } + Ok(function .parameters .posonlyargs .iter() .chain(&function.parameters.args) .next() - .map(|parameter| parameter.parameter.name.id.clone()) + .map(|parameter| parameter.parameter.name.id.clone())) } -/// The attributes a method reads or writes on its receiver, `x` of `self.x`, with where. -fn receiver_attributes(statement: &Stmt) -> Vec<(Name, TextRange)> { +/// the attributes a method of `model`'s file reads or writes on its receiver, `x` of `self.x`, +/// with where +fn receiver_attributes( + model: &SemanticModel<'_>, + statement: &Stmt, +) -> Result, String> { struct Attributes<'a> { receiver: &'a str, found: Vec<(Name, TextRange)>, @@ -1175,94 +1266,279 @@ fn receiver_attributes(statement: &Stmt) -> Vec<(Name, TextRange)> { walk_expr(self, expr); } } - let Some(receiver) = receiver_name(statement) else { - return Vec::new(); + let Stmt::FunctionDef(function) = statement else { + return Ok(Vec::new()); + }; + let Some(receiver) = receiver_name(model, function)? else { + return Ok(Vec::new()); }; let mut attributes = Attributes { receiver: receiver.as_str(), found: Vec::new(), }; - if let Stmt::FunctionDef(function) = statement { - attributes.visit_body(&function.body); - } - attributes.found + attributes.visit_body(&function.body); + Ok(attributes.found) } -/// The names an attribute's value reads from the class body it is in: other members bound -/// before it, like `B` in `B = A + 1`. -fn class_scope_reads(class: &ast::StmtClassDef, statement: &Stmt) -> Vec<(Name, TextRange)> { - struct Loads { - found: Vec<(Name, TextRange)>, +/// a name a member reads where its class body is evaluated: an attribute's annotation and value, +/// a method's decorators, defaults and annotations, a lambda's defaults and a comprehension's +/// first iterable +enum BodyRead { + /// a name the class body binds, bound by the member statement `index` where one binds it + Member { name: Name, index: Option }, + /// a name the class body does not bind, and the scope it resolves to from there; `None` for + /// a builtin or a name nothing binds + Outside { + name: Name, + resolves_to: Option, + }, +} + +fn class_body_reads( + db: &dyn Db, + class: &ClassSite<'_>, + def: &ast::StmtClassDef, + statement: &Stmt, +) -> Vec { + #[derive(Default)] + struct Loads<'a> { + found: Vec<&'a ast::ExprName>, } - impl<'a> SourceOrderVisitor<'a> for Loads { + impl<'a> SourceOrderVisitor<'a> for Loads<'a> { fn visit_expr(&mut self, expr: &'a Expr) { match expr { - Expr::Name(name) if name.ctx.is_load() => { - self.found.push((name.id.clone(), name.range())); + Expr::Name(name) if name.ctx.is_load() => self.found.push(name), + // what is evaluated later, in a scope of its own, is not read in the class body + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + for default in parameters.iter().filter_map(ast::AnyParameterRef::default) { + self.visit_expr(default); + } + } + } + Expr::ListComp(ast::ExprListComp { generators, .. }) + | Expr::SetComp(ast::ExprSetComp { generators, .. }) + | Expr::Generator(ast::ExprGenerator { generators, .. }) => { + if let Some(first) = generators.first() { + self.visit_expr(&first.iter); + } + } + Expr::DictComp(ast::ExprDictComp { generators, .. }) => { + if let Some(first) = generators.first() { + self.visit_expr(&first.iter); + } } - // a lambda or comprehension does not see the class body - Expr::Lambda(_) - | Expr::ListComp(_) - | Expr::SetComp(_) - | Expr::DictComp(_) - | Expr::Generator(_) => {} _ => walk_expr(self, expr), } } } - let value = match statement { - Stmt::Assign(assign) => &*assign.value, - Stmt::AnnAssign(assign) => match &assign.value { - Some(value) => value, - None => return Vec::new(), - }, - _ => return Vec::new(), - }; - let mut loads = Loads { found: Vec::new() }; - loads.visit_expr(value); + let mut loads = Loads::default(); + match statement { + Stmt::Assign(assign) => loads.visit_expr(&assign.value), + Stmt::AnnAssign(assign) => { + loads.visit_expr(&assign.annotation); + if let Some(value) = &assign.value { + loads.visit_expr(value); + } + } + Stmt::FunctionDef(function) => { + for decorator in &function.decorator_list { + loads.visit_expr(&decorator.expression); + } + for parameter in &function.parameters { + if let Some(default) = parameter.default() { + loads.visit_expr(default); + } + if let Some(annotation) = parameter.annotation() { + loads.visit_expr(annotation); + } + } + if let Some(returns) = &function.returns { + loads.visit_expr(returns); + } + } + _ => {} + } + let index = semantic_index(db, class.file); + let body_scope = index.try_node_scope(NodeWithScopeRef::Class(def)); loads .found .into_iter() - .filter(|(name, _)| { - class - .body - .iter() - .any(|other| !std::ptr::eq(other, statement) && binds(other, name)) + .filter_map(|load| { + let scope = index.try_expression_scope_id(&ast::ExprRef::Name(load))?; + let resolves_to = resolving_scope(index, scope, load.id.as_str()); + Some(if resolves_to.is_some() && resolves_to == body_scope { + BodyRead::Member { + name: load.id.clone(), + index: member_statements(def, load.id.as_str()).first().copied(), + } + } else { + BodyRead::Outside { + name: load.id.clone(), + resolves_to, + } + }) }) .collect() } -/// Whether a method calls `super()`, or reads basedpython's `super`. -fn uses_super(statement: &Stmt) -> bool { - struct Super { +/// whether `scope` binds `name` after the statement of `first` begins and before the statement of +/// `second`, where a class body run between them would read another value +fn bound_between( + db: &dyn Db, + class: &ClassSite<'_>, + scope: FileScopeId, + name: &str, + first: &ast::StmtClassDef, + second: &ast::StmtClassDef, +) -> bool { + let index = semantic_index(db, class.file); + let Some(symbol) = index.place_table(scope).symbol_id(name) else { + return false; + }; + let in_stub = class.file_id.is_stub(db); + let between = TextRange::new(first.start(), second.start().max(first.start())); + bindings_of_symbol(db, index, &class.parsed, scope, symbol, in_stub) + .into_iter() + .any(|definition| between.contains_range(definition.focus_range(db, &class.parsed).range())) +} + +/// whether a member of `model`'s file reads the builtin `super`, under whatever name it is bound +/// to, or basedpython's `super` keyword: either names the class the member is written in, the +/// one by its `__class__` cell, the other by the class it is passed +fn uses_super(model: &SemanticModel<'_>, statement: &Stmt) -> bool { + struct Super<'m, 'db> { + model: &'m SemanticModel<'db>, found: bool, } - impl<'a> SourceOrderVisitor<'a> for Super { + impl<'a> SourceOrderVisitor<'a> for Super<'_, '_> { fn visit_expr(&mut self, expr: &'a Expr) { - if matches!(expr, Expr::Name(name) if name.id.as_str() == "super") { + if let Expr::Name(name) = expr + && name.ctx.is_load() + && name + .inferred_type(self.model) + .is_some_and(|ty| is_super(self.model.db(), ty)) + { self.found = true; } walk_expr(self, expr); } } - let mut uses = Super { found: false }; + let mut uses = Super { + model, + found: false, + }; uses.visit_stmt(statement); uses.found } -/// The `override` a method is marked with: basedpython's modifier, or `@override`. -fn override_marker(statement: &Stmt) -> Option<&ast::Decorator> { - let Stmt::FunctionDef(function) = statement else { - return None; - }; - function.decorator_list.iter().find(|decorator| { - matches!(&decorator.expression, Expr::Name(name) if name.id.as_str() == "override") - || matches!(&decorator.expression, Expr::Attribute(attribute) if attribute.attr.as_str() == "override") +/// a name `statement`, a member of the class `def`, reads from around the class that names +/// something else around `target`'s class `target_def`, where the member would read it: a scope +/// around one binds it and the same scope is not around the other, or the class body of +/// `target_def` binds what the class body of `def` reads from around it. a name both read from +/// their module's top level, or that no scope binds, is for [`checked_dependencies`] to compare +/// between modules +fn read_rebound_around( + db: &dyn Db, + class: &ClassSite<'_>, + def: &ast::StmtClassDef, + statement: &Stmt, + target: &ClassSite<'_>, + target_def: &ast::StmtClassDef, +) -> Option { + let index = semantic_index(db, class.file); + let target_index = semantic_index(db, target.file); + let body = index.try_node_scope(NodeWithScopeRef::Class(def))?; + let target_body = target_index.try_node_scope(NodeWithScopeRef::Class(target_def))?; + let around: FxHashSet = index + .ancestor_scopes(body) + .map(|(scope, _)| scope) + .filter(|scope| *scope != body) + .collect(); + let in_module = |scope: Option| scope.is_none_or(FileScopeId::is_global); + loads(statement).into_iter().find_map(|load| { + let scope = index.try_expression_scope_id(&ast::ExprRef::Name(load))?; + let name = load.id.as_str(); + let seen: Vec = index + .visible_ancestor_scopes(scope) + .map(|(scope, _)| scope) + .collect(); + let resolves_to = resolving_scope_among(index, seen.iter().copied(), name); + // what the member binds itself moves with it, and what the class body binds is a sibling + if resolves_to.is_some_and(|scope| !around.contains(&scope)) { + return None; + } + // what a method reads skips its class body, and what the class body evaluates does not + let sees_body = seen.contains(&body); + let target_resolves_to = resolving_scope_among( + target_index, + target_index + .visible_ancestor_scopes(target_body) + .map(|(scope, _)| scope) + .skip(usize::from(!sees_body)), + name, + ); + let rebound = if class.file_id == target.file_id { + resolves_to != target_resolves_to + } else { + !in_module(resolves_to) || !in_module(target_resolves_to) + }; + rebound.then(|| load.id.clone()) }) } -fn has_override(statement: &Stmt) -> bool { - override_marker(statement).is_some() +/// whether a member of `model`'s file reads the `__class__` cell python gives a method, which is +/// the class the method is written in, wherever it is called from +fn member_reads_class_cell(model: &SemanticModel<'_>, statement: &Stmt) -> bool { + struct ClassCell<'m, 'db> { + model: &'m SemanticModel<'db>, + found: bool, + } + impl<'a> SourceOrderVisitor<'a> for ClassCell<'_, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + && reads_class_cell(self.model, name) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut reads = ClassCell { + model, + found: false, + }; + reads.visit_stmt(statement); + reads.found +} + +fn class_cell_refusal(member: &str, to: &str, from: &str) -> String { + format!("`{member}` reads `__class__`, which would be {to} rather than {from}") +} + +/// the `override` a method of `class` is marked with: basedpython's modifier, or a decorator +/// that is `typing.override` +fn override_marker<'a>( + db: &dyn Db, + class: &ClassSite<'_>, + statement: &'a Stmt, +) -> Option<&'a ast::Decorator> { + let Stmt::FunctionDef(function) = statement else { + return None; + }; + let model = SemanticModel::new(db, class.file); + function + .decorator_list + .iter() + .find(|decorator| match &decorator.expression { + // the modifier is parsed into a decorator named for it, which no source spells, so + // it is told apart from a real name by its context + Expr::Name(name) if name.ctx == ExprContext::Invalid => name.id.as_str() == "override", + expression => expression + .inferred_type(&model) + .is_some_and(|ty| is_override_function(db, ty)), + }) } /// the range that takes `decorator` out: an `@override` line with its line break, or the @@ -1285,12 +1561,16 @@ fn override_range(source: &str, decorator: &ast::Decorator) -> TextRange { } } -/// The text of the body statement `index`, as [`cut_range`] cuts it, without its `override` when -/// `strip_override`. -fn member_text(source: &str, body: &[Stmt], index: usize, strip_override: bool) -> String { - let statement = &body[index]; +/// the text of the body statement `index`, as [`cut_range`] cuts it, without the `override` +/// `strip_override` when there is one +fn member_text( + source: &str, + body: &[Stmt], + index: usize, + strip_override: Option<&ast::Decorator>, +) -> String { let cut = cut_range(source, body, index); - if strip_override && let Some(decorator) = override_marker(statement) { + if let Some(decorator) = strip_override { let local = override_range(source, decorator) - cut.start; let mut text = cut.text; text.replace_range(local.to_std_range(), ""); @@ -1420,6 +1700,11 @@ fn insert_members( members: &[(MemberKind, String)], line_ending: &str, ) -> Vec { + // a pull of bases alone moves no member, and the superclass's body stays as it is: its + // placeholder too, which replaced with nothing would leave the class without a body + if members.is_empty() { + return Vec::new(); + } let attributes: Vec<&str> = members .iter() .filter(|(kind, _)| *kind == MemberKind::Attribute) @@ -1596,7 +1881,7 @@ fn add_override(db: &dyn Db, class: &ClassSite<'_>, statement: &Stmt) -> Vec, statement: &Stmt) -> Vec { - let Some(decorator) = override_marker(statement) else { + let Some(decorator) = override_marker(db, class, statement) else { return Vec::new(); }; let source = source_text(db, class.file_id); @@ -1674,13 +1959,19 @@ class GrandChild(Child): impl CursorTest { fn pull_up(&self, names: &[&str], target: &str) -> String { + self.pull_up_with_bases(names, &[], target) + } + + fn pull_up_with_bases(&self, names: &[&str], bases: &[&str], target: &str) -> String { let names: Vec = names.iter().map(ToString::to_string).collect(); + let bases: Vec = bases.iter().map(ToString::to_string).collect(); let moved = salsa::attach(&self.db, || { pull_members_up( &self.db, self.program_file(self.cursor.file), self.cursor.offset, &names, + &bases, target, ) }); @@ -1747,7 +2038,7 @@ class GrandChild(Child): "); } - /// A method reads as its parameters, on one line whatever lines they are written on. + /// a method reads as its parameters, on one line whatever lines they are written on #[test] fn a_member_reads_as_its_name_and_parameters() { let test = CursorTestBuilder::default() @@ -1783,8 +2074,8 @@ class GrandChild(Child): .build() } - /// A base the class writes is pulled up into the superclass's bases, after its own, where - /// every class below the superclass finds what it found before. + /// a base the class writes is pulled up into the superclass's bases, after its own, where + /// every class below the superclass finds what it found before #[test] fn a_base_is_pulled_up_into_the_superclass() { let test = mixins( @@ -1809,7 +2100,7 @@ class Child(Base, Mixin, metaclass=type): superclasses: main.Base, main.Mixin subclasses: "); - assert_snapshot!(test.pull_up(&["Mixin", "own"], "main.Base"), @" + assert_snapshot!(test.pull_up_with_bases(&["own"], &["Mixin"], "main.Base"), @" --- /main.py class Mixin: def mix(self): @@ -1829,7 +2120,43 @@ class Child(Base, Mixin, metaclass=type): "); } - /// Whatever a class below the superclass finds, it has to find in the same class afterwards. + /// bases pulled up on their own move no member, and leave a superclass whose body is only + /// `pass` as it is + #[test] + fn a_base_alone_is_pulled_up_into_a_superclass_with_an_empty_body() { + let test = mixins( + r#"class Mixin: + def mix(self): + return 1 + + +class Base: + pass + + +class Child(Base, Mixin): + def own(self): + return self.mix() +"#, + ); + assert_snapshot!(test.pull_up_with_bases(&[], &["Mixin"], "main.Base"), @" + --- /main.py + class Mixin: + def mix(self): + return 1 + + + class Base(Mixin): + pass + + + class Child(Base): + def own(self): + return self.mix() + "); + } + + /// whatever a class below the superclass finds, it has to find in the same class afterwards #[test] fn a_base_that_changes_what_a_class_finds_is_refused() { let test = mixins( @@ -1847,7 +2174,7 @@ class Child(Mixin, Base): pass "#, ); - assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: Child would find `keep` in another class than it does"); + assert_snapshot!(test.pull_up_with_bases(&[], &["Mixin"], "main.Base"), @"refused: Child would find `keep` in another class than it does"); let test = mixins( r#"class Mixin: @@ -1872,11 +2199,11 @@ class Sibling(Base, Other): pass "#, ); - assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: Sibling would find `shared` in another class than it does"); + assert_snapshot!(test.pull_up_with_bases(&[], &["Mixin"], "main.Base"), @"refused: Sibling would find `shared` in another class than it does"); } - /// The superclass names the base where it is defined, so the name has to be bound there, and - /// the class cannot lose the base it reaches the superclass through. + /// the superclass names the base where it is defined, so the name has to be bound there, and + /// the class cannot lose the base it reaches the superclass through #[test] fn a_base_the_superclass_cannot_take_is_refused() { let test = mixins( @@ -1892,7 +2219,7 @@ class Child(Base, Mixin): pass "#, ); - assert_snapshot!(test.pull_up(&["Mixin"], "main.Base"), @"refused: `Mixin` is not bound above Base, where it would be written"); + assert_snapshot!(test.pull_up_with_bases(&[], &["Mixin"], "main.Base"), @"refused: `Mixin` is not bound above Base, where it would be written"); let test = mixins( r#"class Base: @@ -1907,7 +2234,7 @@ class Child(Middle): pass "#, ); - assert_snapshot!(test.pull_up(&["Middle"], "main.Base"), @"refused: Child inherits Base through `Middle`, so it cannot lose it"); + assert_snapshot!(test.pull_up_with_bases(&[], &["Middle"], "main.Base"), @"refused: Child inherits Base through `Middle`, so it cannot lose it"); } #[test] @@ -2252,4 +2579,540 @@ class Leaf(Mid): .build(); assert_snapshot!(test.extract(&["x"], "Base"), @"refused: Point has a base whose subclasses python treats specially, which Base would take in its place"); } + + /// a default, a decorator and a comprehension's first iterable are evaluated in the class + /// body, so what they read there has to move with the member + #[test] + fn a_member_whose_class_body_reads_stay_behind_is_refused() { + let test = mixins( + r#"class Base: + pass + + +class Child(Base): + LIMIT = 3 + RAW = ["a", "b"] + + def tag(function): + return function + + def clamp(self, x, limit=LIMIT): + return min(x, limit) + + @tag + def run(self): + return 1 + + NAMES = [name.upper() for name in RAW] +"#, + ); + assert_snapshot!(test.pull_up(&["clamp"], "main.Base"), @"refused: `clamp` reads Child's `LIMIT`, which it would no longer read in Base's body; pull it up too"); + assert_snapshot!(test.pull_up(&["run"], "main.Base"), @"refused: `run` reads Child's `tag`, which it would no longer read in Base's body; pull it up too"); + assert_snapshot!(test.pull_up(&["NAMES"], "main.Base"), @"refused: `NAMES` reads Child's `RAW`, which it would no longer read in Base's body; pull it up too"); + assert_snapshot!(test.extract(&["clamp"], "Shape"), @"refused: `clamp` reads Child's `LIMIT`, which it would no longer read in Shape's body; extract it too"); + } + + /// `Base`'s own `LIMIT` is not what `Child`'s body reads: a class body does not see another + /// class's members, and `SIZE` would read `Base.LIMIT` once it is written in `Base` + #[test] + fn a_member_that_reads_a_sibling_the_superclass_also_has_is_refused() { + let test = mixins( + r#"class Base: + LIMIT = 1 + + +class Child(Base): + LIMIT = 3 + SIZE = LIMIT * 2 +"#, + ); + assert_snapshot!(test.pull_up(&["SIZE"], "main.Base"), @"refused: `SIZE` reads Child's `LIMIT`, which it would no longer read in Base's body; pull it up too"); + } + + /// `LIMIT` is bound after `Base`, so `Base`'s body would read it before it is bound + #[test] + fn a_member_that_reads_a_name_bound_between_the_classes_is_refused() { + let test = mixins( + r#"class Base: + pass + + +LIMIT = 3 + + +class Child(Base): + SIZE = LIMIT * 2 +"#, + ); + assert_snapshot!(test.pull_up(&["SIZE"], "main.Base"), @"refused: `SIZE` reads `LIMIT`, which is bound after Base is defined"); + } + + /// `Other` finds `describe` in `Mixin`, after `Shape` in its method resolution order, and + /// would find the one pulled into `Shape` instead + #[test] + fn a_pull_up_another_subclass_would_find_before_its_mixin_is_refused() { + let test = mixins( + r#"class Shape: + pass + + +class Mixin: + def describe(self) -> str: + return "mixin" + + +class Square(Shape): + def describe(self) -> str: + return "square" + + +class Other(Shape, Mixin): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["describe"], "main.Shape"), @"refused: Other would find `describe` in Shape rather than in Mixin"); + } + + /// `Square` finds `Mixin`'s `describe` before `Shape`'s, so it would lose its own + #[test] + fn a_pull_up_past_a_mixin_before_the_superclass_is_refused() { + let test = mixins( + r#"class Shape: + pass + + +class Mixin: + def describe(self) -> str: + return "mixin" + + +class Square(Mixin, Shape): + def describe(self) -> str: + return "square" +"#, + ); + assert_snapshot!(test.pull_up(&["describe"], "main.Shape"), @"refused: Square would inherit Mixin's `describe` rather than its own"); + } + + /// `Both` finds `Square`'s `describe` before `Other`'s, and would find `Other`'s once + /// `Square` loses it to `Shape`, which comes after `Other` + #[test] + fn a_pull_up_a_class_below_would_find_elsewhere_is_refused() { + let test = mixins( + r#"class Shape: + pass + + +class Square(Shape): + def describe(self) -> str: + return "square" + + +class Other(Shape): + def describe(self) -> str: + return "other" + + +class Both(Square, Other): + pass +"#, + ); + assert_snapshot!(test.pull_up(&["describe"], "main.Shape"), @"refused: Both would find `describe` in Other rather than in Square"); + } + + /// a decorator the program itself calls `override` is no `typing.override`, and stays + #[test] + fn a_decorator_named_override_that_is_not_typing_s_stays() { + let test = mixins( + r#"def override(function): + return function + + +class Base: + pass + + +class Child(Base): + @override + def describe(self) -> str: + return "child" +"#, + ); + assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @r#" + --- /main.py + def override(function): + return function + + + class Base: + @override + def describe(self) -> str: + return "child" + + + class Child(Base): + pass + "#); + } + + /// a member and a base written with one name are told apart by what the pull is asked to + /// move: here the base, while the class keeps its member + #[test] + fn a_base_is_pulled_up_apart_from_a_member_of_its_name() { + let test = mixins( + r#"class Mixin: + pass + + +class Base: + pass + + +class Child(Base, Mixin): + Mixin = 1 +"#, + ); + assert_snapshot!(test.pull_up_with_bases(&[], &["Mixin"], "main.Base"), @" + --- /main.py + class Mixin: + pass + + + class Base(Mixin): + pass + + + class Child(Base): + Mixin = 1 + "); + } + + /// `Root` is `object`, which every class inherits already, so it is no base to pull up + #[test] + fn object_is_no_movable_base_whatever_it_is_called() { + let test = mixins( + r#"Root = object + + +class Base: + pass + + +class Child(Base, Root): + pass +"#, + ); + assert_snapshot!(test.movable(), @" + main.Child + members: Base (Base) + superclasses: main.Base + subclasses: + "); + } + + /// a class keyword is passed to `__init_subclass__` for the class it is written on alone, + /// so it cannot move to the new class + #[test] + fn extracting_from_a_class_with_a_keyword_is_refused() { + let test = mixins( + r#"class Base: + def __init_subclass__(cls, flag: bool = False, **kwargs): + super().__init_subclass__(**kwargs) + + +class Child(Base, flag=True): + x = 1 +"#, + ); + assert_snapshot!(test.extract(&["x"], "Shape"), @"refused: Child passes `flag=True` when it is made, which Shape would be passed instead"); + } + + /// a metaclass makes the new class too, and the class still gets it from the new class + #[test] + fn extracting_from_a_class_with_a_metaclass_moves_it() { + let test = mixins( + r#"from abc import ABCMeta + + +class Child(metaclass=ABCMeta): + x = 1 +"#, + ); + assert_snapshot!(test.extract(&["x"], "Shape"), @" + --- /main.py + from abc import ABCMeta + + + class Shape(metaclass=ABCMeta): + x = 1 + + + class Child(Shape): + pass + "); + } + + #[test] + fn extracting_an_attribute_of_a_decorated_class_is_refused() { + let test = mixins( + r#"from dataclasses import dataclass + + +@dataclass +class Point: + x: int +"#, + ); + assert_snapshot!(test.extract(&["x"], "Base"), @"refused: `x` is an attribute of a decorated class, which its decorator may read as a field"); + } + + fn two_modules(base: &str, child: &str) -> CursorTest { + CursorTestBuilder::default() + .source("pkg/__init__.py", "") + .source("pkg/base.py", base) + .source("pkg/child.py", child) + .build() + } + + /// pulled into another module, a member reads what it read where it was written, or is + /// refused as a function moved there is + #[test] + fn a_member_pulled_into_another_module_is_checked_as_a_moved_function_is() { + let base = "class Base:\n pass\n"; + let test = two_modules( + base, + "from pkg.base import Base\n\n\nclass Child(Base):\n def origin(self):\n return __name__\n", + ); + assert_snapshot!(test.pull_up(&["origin"], "pkg.base.Base"), @"refused: `origin` reads `__name__`, which is pkg.child's own and would be pkg.base's once moved"); + let test = two_modules( + base, + "from pkg.base import Base\n\nCOUNT = 0\n\n\nclass Child(Base):\n def bump(self):\n global COUNT\n COUNT += 1\n", + ); + assert_snapshot!(test.pull_up(&["bump"], "pkg.base.Base"), @"refused: `bump` assigns pkg.child's `COUNT` through `global`, which would name pkg.base's once moved"); + let test = two_modules( + "len = 3\n\n\nclass Base:\n pass\n", + "from pkg.base import Base\n\n\nclass Child(Base):\n def size(self, x):\n return len(x)\n", + ); + assert_snapshot!(test.pull_up(&["size"], "pkg.base.Base"), @"refused: `size` uses the builtin `len`, which pkg.base binds to something else"); + let test = two_modules( + base, + "import os\nfrom pkg.base import Base\n\nimport os.path as os\n\n\nclass Child(Base):\n def sep(self):\n return os.sep\n", + ); + assert_snapshot!(test.pull_up(&["sep"], "pkg.base.Base"), @"refused: `sep` uses `os`, which pkg.child binds more than once, and an import of it would keep only the value it had when imported"); + } + + /// pushed into another module, a member reads what it read where it was written: a name + /// private to its module is not imported into the other + #[test] + fn a_member_pushed_into_another_module_is_checked_as_a_moved_function_is() { + let test = two_modules( + "_RATE = 2\n\n\nclass Base:\n def rate(self):\n return _RATE\n", + "from pkg.base import Base\n\n\nclass Child(Base):\n pass\n", + ); + assert_snapshot!(test.push_down(&["rate"]), @"refused: `rate` uses `_RATE`, which is private to pkg.base; make it public, or move it too"); + } + + /// what a decorator makes of a method is what the checker resolves it to, under whatever + /// name it is bound to + #[test] + fn a_member_s_kind_is_what_its_decorators_resolve_to() { + let test = mixins( + r#"prop = property +static = staticmethod + + +class Base: + pass + + +class Child(Base): + @prop + def size(self): + return 1 + + @static + def make(other): + return other.missing +"#, + ); + assert_snapshot!(test.movable(), @" + main.Child + members: size(self) (Property), make(other) (StaticMethod), Base (Base) + superclasses: main.Base + subclasses: + "); + // a static method is passed no receiver, so `other.missing` is no attribute it needs + assert_snapshot!(test.pull_up(&["make"], "main.Base"), @" + --- /main.py + prop = property + static = staticmethod + + + class Base: + @static + def make(other): + return other.missing + + + class Child(Base): + @prop + def size(self): + return 1 + "); + } + + /// `super` read under another name still names the class the method is written in, and a + /// parameter called `super` does not + #[test] + fn a_use_of_super_is_the_builtin_s_under_whatever_name() { + let test = mixins( + r#"sup = super + + +class Root: + def describe(self): + return "root" + + +class Base(Root): + pass + + +class Mixin: + pass + + +class Child(Base, Mixin): + def describe(self): + return sup(Child, self).describe() + + def tag(self, super): + return super.tag +"#, + ); + assert_snapshot!(test.pull_up(&["describe"], "main.Base"), @"refused: `describe` calls `super()`, which would start from Base rather than from Child"); + assert_snapshot!(test.pull_up(&["tag"], "main.Base"), @r#" + --- /main.py + sup = super + + + class Root: + def describe(self): + return "root" + + + class Base(Root): + def tag(self, super): + return super.tag + + + class Mixin: + pass + + + class Child(Base, Mixin): + def describe(self): + return sup(Child, self).describe() + "#); + } + + /// a name a pushed member reads from around its class has to be the same name around the + /// subclass: a function the subclass is defined in, or the subclass's body for what the class + /// body evaluates, may bind it to something else + #[test] + fn a_push_down_into_a_scope_that_binds_a_name_it_reads_is_refused() { + let test = mixins( + "def helper():\n return 1\n\n\nclass Base:\n def run(self):\n return helper()\n\n\ndef make():\n def helper():\n return 2\n\n class Child(Base):\n pass\n\n return Child\n", + ); + assert_snapshot!(test.push_down(&["run"]), @"refused: `run` reads `helper`, which is another name where Child is defined"); + let test = mixins( + "def helper():\n return 1\n\n\nclass Base:\n def run(self, value=helper()):\n return value\n\n\nclass Child(Base):\n helper = None\n", + ); + assert_snapshot!(test.push_down(&["run"]), @"refused: `run` reads `helper`, which is another name where Child is defined"); + let test = mixins( + "def make():\n def helper():\n return 2\n\n class Base:\n def run(self):\n return helper()\n\n class Child(Base):\n pass\n\n return Child\n", + ); + assert_snapshot!(test.push_down(&["run"]), @" + --- /main.py + def make(): + def helper(): + return 2 + + class Base: + pass + + class Child(Base): + def run(self): + return helper() + + return Child + "); + } + + /// a name a pulled member reads from around its class has to be the same name around the + /// superclass, as it does for what the class body evaluates + #[test] + fn a_pull_up_out_of_a_scope_that_binds_a_name_it_reads_is_refused() { + let test = mixins( + "class Base:\n pass\n\n\ndef make():\n def helper():\n return 2\n\n class Child(Base):\n def run(self):\n return helper()\n\n return Child\n", + ); + assert_snapshot!(test.pull_up(&["run"], "main.Base"), @"refused: `run` reads `helper`, which is another name where Base is defined"); + } + + /// `__class__` in a method is the class the method is written in, so it names another class + /// wherever the method moves to, and a `__class__` the method binds itself does not + #[test] + fn a_method_reading_its_class_cell_stays_in_its_class() { + let test = mixins( + "class Base:\n def make(self):\n return __class__()\n\n\nclass Child(Base):\n def copy(self):\n return __class__()\n\n def local(self):\n __class__ = int\n return __class__()\n", + ); + assert_snapshot!(test.pull_up(&["copy"], "main.Base"), @"refused: `copy` reads `__class__`, which would be Base rather than Child"); + assert_snapshot!(test.extract(&["copy"], "Middle"), @"refused: `copy` reads `__class__`, which would be Middle rather than Child"); + assert_snapshot!(test.pull_up(&["local"], "main.Base"), @" + --- /main.py + class Base: + def make(self): + return __class__() + + def local(self): + __class__ = int + return __class__() + + + class Child(Base): + def copy(self): + return __class__() + "); + let test = mixins( + "class Base:\n def make(self):\n return __class__()\n\n\nclass Child(Base):\n pass\n", + ); + assert_snapshot!(test.push_down(&["make"]), @"refused: `make` reads `__class__`, which would be the subclass it lands in rather than Base"); + } + + /// an enum is a base python treats specially, whatever it holds, and a class that holds what + /// an enum does is not one + #[test] + fn an_enum_base_is_told_by_its_class() { + let test = mixins( + "from enum import Enum\n\n\nclass Flavor(Enum):\n pass\n\n\nclass Child(Flavor):\n def describe(self):\n return 1\n", + ); + assert_snapshot!(test.extract(&["describe"], "Base"), @"refused: Child has a base whose subclasses python treats specially, which Base would take in its place"); + let test = mixins( + "class Registry:\n _member_map_ = {}\n\n\nclass Child(Registry):\n def describe(self):\n return 1\n", + ); + assert_snapshot!(test.extract(&["describe"], "Base"), @" + --- /main.py + class Registry: + _member_map_ = {} + + + class Base(Registry): + def describe(self): + return 1 + + + class Child(Base): + pass + "); + } } diff --git a/crates/ty_ide/src/refactor/class_members/bases.rs b/crates/ty_ide/src/refactor/class_members/bases.rs index 93d2365276..ad716a7640 100644 --- a/crates/ty_ide/src/refactor/class_members/bases.rs +++ b/crates/ty_ide/src/refactor/class_members/bases.rs @@ -1,15 +1,15 @@ -//! Pulling a class's bases up into a superclass, as python's own pull members up offers a mixin: +//! pulling a class's bases up into a superclass, as python's own pull members up offers a mixin: //! //! ```py //! class Shape: ... class Shape(Mixin): ... //! class Square(Shape, Mixin): ... ──► class Square(Shape): ... //! ``` //! -//! Every class below the superclass then inherits the base, so the pull is refused unless every +//! every class below the superclass then inherits the base, so the pull is refused unless every //! class whose bases change — the superclass and every class of the project below it — finds each -//! name it found before in the class it found it in. A name only the base brings is gained, which -//! is what the pull is for. Python has to be able to order each class's bases afterwards, and the -//! superclass has to be able to name the base where it is defined. +//! name it found before in the class it found it in. a name only the base brings is gained, which +//! is what the pull is for. python has to be able to order each class's bases afterwards, and the +//! superclass has to be able to name the base where it is defined use ruff_db::files::File; use ruff_db::parsed::parsed_module; @@ -23,37 +23,48 @@ use ty_project::Db; use ty_python_core::ProgramFile; use ty_python_semantic::types::Type; use ty_python_semantic::{ - ProgramEnvironment, class_bases, instance_member_names, type_hierarchy_mro, - type_hierarchy_prepare, + HasType, ProgramEnvironment, SemanticModel, class_bases, instance_member_names, + type_hierarchy_mro, type_hierarchy_prepare, }; -use super::{ClassSite, all_classes}; +use super::{ClassSite, all_classes, member_statements}; use crate::code_action::FileEdit; -/// A base a class statement writes, by name, which a pull up can move. +/// a base a class statement writes, by name, which a pull up can move pub(super) struct WrittenBase<'a> { - /// The base as written, which is how a client names it. + /// the base as written, which is how a client names it pub(super) text: &'a str, pub(super) expression: &'a Expr, } -/// The bases `def` writes as a name or a dotted name, the kind a superclass can write too. +/// the bases `def`, the statement of `class`, writes as a name or a dotted name, the kind a +/// superclass can write too, but `object`, which every class inherits already pub(super) fn written_bases<'a>( + db: &dyn Db, + class: &ClassSite<'_>, def: &'a ast::StmtClassDef, source: &'a str, ) -> Vec> { + let model = SemanticModel::new(db, class.file); + let env = ProgramEnvironment::from_file(class.file); + // `object` is the one class with nothing above it + let is_object = |expression: &Expr| { + expression.inferred_type(&model).is_some_and(|ty| { + type_hierarchy_prepare(db, &env, ty).is_some() + && type_hierarchy_mro(db, &env, ty).is_empty() + }) + }; def.bases() .iter() - .filter(|base| root_name(base).is_some()) + .filter(|base| root_name(base).is_some() && !is_object(base)) .map(|expression| WrittenBase { text: &source[expression.range()], expression, }) - .filter(|base| base.text != "object") .collect() } -/// The name a base's name or dotted name starts with. +/// the name a base's name or dotted name starts with fn root_name(expression: &Expr) -> Option<&Name> { match expression { Expr::Name(name) => Some(&name.id), @@ -62,16 +73,16 @@ fn root_name(expression: &Expr) -> Option<&Name> { } } -/// What pulling bases up does: the edits, and the names the superclass gains with them. +/// what pulling bases up does: the edits, and the names the superclass gains with them pub(super) struct PulledBases { pub(super) edits: Vec, pub(super) gained: FxHashSet, } -/// A class, as the file and name range its statement has. +/// a class, as the file and name range its statement has type ClassKey = (File, TextRange); -/// Pulls the bases `pulled`, as `class` writes them, up into `superclass`, after its own bases. +/// pulls the bases `pulled`, as `class` writes them, up into `superclass`, after its own bases pub(super) fn pull_bases_up<'db>( db: &'db dyn Db, class: &ClassSite<'db>, @@ -90,7 +101,7 @@ pub(super) fn pull_bases_up<'db>( type_hierarchy_prepare(db, &env, ty) .map(|class| (class.file.file(db), class.selection_range)) }; - let written = written_bases(def, source); + let written = written_bases(db, class, def, source); let resolved = class_bases(db, &env, class.ty); if resolved.len() != def.bases().len() { return Err(format!( @@ -274,7 +285,92 @@ pub(super) fn pull_bases_up<'db>( Ok(PulledBases { edits, gained }) } -/// The classes `ty` inherits from in its method resolution order, itself not among them. +/// refuses pulling the member `name` of `class` up into `superclass` where a class would then +/// find `name` elsewhere than it should: `class` and the classes below it that found its own +/// have to find the pulled one, and every other class below the superclass, `below_superclass`, +/// what it found before. a class that found no `name` gains the pulled one, which is what the +/// pull is for +pub(super) fn refuse_changed_lookups<'db>( + db: &'db dyn Db, + class: &ClassSite<'db>, + superclass: &ClassSite<'db>, + below_superclass: &[ClassSite<'db>], + name: &str, +) -> Result<(), String> { + let def = class + .def() + .ok_or_else(|| "there is no class here".to_string())?; + let env = ProgramEnvironment::from_file(class.file); + let name = Name::new(name); + let class_key = (class.file_id, class.name_range); + let super_key = (superclass.file_id, superclass.name_range); + // the class keeps what binds the name besides the members pulled + let pulled = member_statements(def, name.as_str()); + let mut keeps = Binder::default(); + for (index, statement) in def.body.iter().enumerate() { + if !pulled.contains(&index) { + keeps.visit_stmt(statement); + } + } + let class_keeps = keeps.names.contains(&name); + let mut names = OwnNames { + db, + program: class.file, + cache: FxHashMap::default(), + }; + let sites = + std::iter::once(class).chain(below_superclass.iter().filter(|site| !site.same(class))); + for site in sites { + let order: Vec<(ClassKey, Name)> = + std::iter::once(((site.file_id, site.name_range), Name::new(&site.name))) + .chain( + type_hierarchy_mro(db, &env, site.ty) + .into_iter() + .map(|class| ((class.file.file(db), class.selection_range), class.name)), + ) + .collect(); + let mut before = None; + let mut after = None; + for (key, owner) in &order { + let owns = names.of(*key).contains(&name); + if before.is_none() && owns { + before = Some((*key, owner)); + } + let owns_after = if *key == class_key { + class_keeps + } else { + *key == super_key || owns + }; + if after.is_none() && owns_after { + after = Some((*key, owner)); + } + } + let before_key = before.map(|(key, _)| key); + let after_key = after.map(|(key, _)| key); + let moves_with_it = + after_key == Some(super_key) && (before_key.is_none() || before_key == Some(class_key)); + if before_key == after_key || moves_with_it { + continue; + } + let after_name = + after.map_or_else(|| "nothing".to_string(), |(_, owner)| owner.to_string()); + return Err(if site.same(class) { + format!( + "{} would inherit {after_name}'s `{name}` rather than its own", + class.name + ) + } else { + format!( + "{} would find `{name}` in {after_name} rather than in {}", + site.name, + before.map_or_else(|| "nothing".to_string(), |(_, owner)| owner.to_string()) + ) + }); + } + Ok(()) +} + +/// the classes `ty` inherits from in its method resolution order, itself not among them fn mro_keys<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) -> Vec { type_hierarchy_mro(db, env, ty) .into_iter() @@ -282,9 +378,9 @@ fn mro_keys<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>) .collect() } -/// The method resolution order of `key` with the bases `new_bases` gives the classes whose bases -/// change, by python's C3 linearization; `None` where python could not order them. A class whose -/// bases stay as they are keeps the order python gives it, as `types` says which class it is. +/// the method resolution order of `key` with the bases `new_bases` gives the classes whose bases +/// change, by python's C3 linearization; `None` where python could not order them. a class whose +/// bases stay as they are keeps the order python gives it, as `types` says which class it is fn linearize<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, @@ -315,8 +411,8 @@ fn linearize<'db>( order } -/// The C3 merge of `sequences`: python's order for a class's ancestors, given each base's order -/// and the bases' own. +/// the C3 merge of `sequences`: python's order for a class's ancestors, given each base's order +/// and the bases' own fn merge(mut sequences: Vec>) -> Option> { let mut merged = Vec::new(); loop { @@ -338,7 +434,7 @@ fn merge(mut sequences: Vec>) -> Option> { } } -/// The names each class's own body binds, the ones a lookup through it finds there. +/// the names each class's own body binds, the ones a lookup through it finds there struct OwnNames<'db> { db: &'db dyn Db, program: ProgramFile<'db>, @@ -346,7 +442,7 @@ struct OwnNames<'db> { } impl OwnNames<'_> { - /// Where a lookup through `order`, a method resolution order, finds each name it finds. + /// where a lookup through `order`, a method resolution order, finds each name it finds fn found_through(&mut self, order: &[ClassKey]) -> FxHashMap { let mut found = FxHashMap::default(); for owner in order { @@ -377,9 +473,9 @@ impl OwnNames<'_> { } } -/// The names statements bind in the scope they are in, not in the functions and classes they -/// define. Every name a store writes counts, and every name an import or a definition binds, so -/// that no name a class could be found to have is missed. +/// the names statements bind in the scope they are in, not in the functions and classes they +/// define. every name a store writes counts, and every name an import or a definition binds, so +/// that no name a class could be found to have is missed #[derive(Default)] struct Binder { names: FxHashSet, @@ -432,15 +528,15 @@ impl<'a> SourceOrderVisitor<'a> for Binder { } } -/// Whether a top-level statement binds `name`. +/// whether a top-level statement binds `name` fn binds_at_top(statement: &Stmt, name: &Name) -> bool { let mut binder = Binder::default(); binder.visit_stmt(statement); binder.names.contains(name) } -/// The edits that take `bases` out of the list `def` writes. A run of them is deleted up to the -/// argument after it, or, at the end of the list, from the argument before it. +/// the edits that take `bases` out of the list `def` writes. a run of them is deleted up to the +/// argument after it, or, at the end of the list, from the argument before it fn remove_bases( file: File, def: &ast::StmtClassDef, @@ -485,7 +581,7 @@ fn remove_bases( edits } -/// The edit that writes `bases` after the bases `def` writes, before its keywords. +/// the edit that writes `bases` after the bases `def` writes, before its keywords fn add_bases(def: &ast::StmtClassDef, bases: &str) -> Edit { let Some(arguments) = &def.arguments else { let after = def diff --git a/crates/ty_ide/src/refactor/flow.rs b/crates/ty_ide/src/refactor/flow.rs index 1de2eee0eb..fdf9885dd8 100644 --- a/crates/ty_ide/src/refactor/flow.rs +++ b/crates/ty_ide/src/refactor/flow.rs @@ -116,7 +116,21 @@ pub(crate) fn resolving_scope( scope: FileScopeId, name: &str, ) -> Option { - for (scope_id, _) in index.visible_ancestor_scopes(scope) { + resolving_scope_among( + index, + index.visible_ancestor_scopes(scope).map(|(scope, _)| scope), + name, + ) +} + +/// the scope a load of `name` resolves to that sees `scopes`, innermost first, as +/// [`resolving_scope`] reads them +pub(crate) fn resolving_scope_among( + index: &SemanticIndex<'_>, + scopes: impl IntoIterator, + name: &str, +) -> Option { + for scope_id in scopes { let table = index.place_table(scope_id); let Some(symbol_id) = table.symbol_id(name) else { continue; diff --git a/crates/ty_ide/src/refactor/move_symbol.rs b/crates/ty_ide/src/refactor/move_symbol.rs index c4b46398dc..17caa40ce7 100644 --- a/crates/ty_ide/src/refactor/move_symbol.rs +++ b/crates/ty_ide/src/refactor/move_symbol.rs @@ -1,4 +1,4 @@ -//! Moving a module's top-level function, class or variable to another module. +//! moving a module's top-level function, class or variable to another module //! //! ```py //! # shapes/util.py # shapes/geometry.py @@ -10,16 +10,16 @@ //! return math.pi * r * RATE return math.pi * r * RATE //! ``` //! -//! The definition is cut from its module, with its decorators and the comments directly above -//! it, and appended to the target. What it reads from its old module comes with it: an import it +//! the definition is cut from its module, with its decorators and the comments directly above +//! it, and appended to the target. what it reads from its old module comes with it: an import it //! uses is written in the target too (and dropped from the old module when nothing else there //! uses it and no other module imports it from there), and another of the old module's names is -//! imported from there. Every reference to it in the project follows it: +//! imported from there. every reference to it in the project follows it: //! `from shapes.util import area` imports it from `shapes.geometry`, `shapes.util.area` reads //! `shapes.geometry.area`, a module that reached it through a star import of the old module -//! imports it by name, and the old module imports it back when it still uses it. +//! imports it by name, and the old module imports it back when it still uses it //! -//! The move is refused, with the reason, when the result would not run as the original did: +//! the move is refused, with the reason, when the result would not run as the original did: //! when the definition reads a name private to its module, which nothing outside the module is //! meant to import; when it reads a name its module binds more than once, which an import would //! see only as it was when imported; when it assigns a name of its module through `global`, which @@ -27,7 +27,7 @@ //! or `__file__`, which would be the target's; when it reads a builtin the target binds to //! something else; when the two modules would end up importing each other, which python cannot //! load; when the target already binds the name; when the name is bound more than once in its -//! module; and when the modules are not the same kind of file. +//! module; and when the modules are not the same kind of file use ruff_db::files::File; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; @@ -49,6 +49,9 @@ use ty_python_core::scope::FileScopeId; use ty_python_core::{SemanticIndex, semantic_index}; use ty_python_semantic::importer::{ImportAction, ImportRequest, MembersInScope}; use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::{ + is_module_implicit_global, reads_module_implicit_global, +}; use ty_python_semantic::{HasType, SemanticModel}; use super::flow::{bindings_of_symbol, resolving_scope}; @@ -57,7 +60,7 @@ use crate::goto::find_goto_target; use crate::import_order; use crate::references::{ReferencesMode, references}; -/// What a top-level definition that can be moved is. +/// what a top-level definition that can be moved is #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MovableKind { Function, @@ -65,25 +68,25 @@ pub enum MovableKind { Variable, } -/// A top-level definition [`move_symbol`] can move. +/// a top-level definition [`move_symbol`] can move #[derive(Debug, Clone, PartialEq, Eq)] pub struct MovableSymbol { - /// The file it is defined in, which need not be the one asked about. + /// the file it is defined in, which need not be the one asked about pub file: File, pub name: String, pub kind: MovableKind, - /// The range of its name where it is defined. + /// the range of its name where it is defined pub name_range: TextRange, - /// The qualified name of the module it is defined in. + /// the qualified name of the module it is defined in pub module: String, } -/// What [`move_symbol`] does to the project. +/// what [`move_symbol`] does to the project #[derive(Debug, Default)] pub struct SymbolMove { pub edits: Vec, - /// References the move leaves as they are, which may no longer reach the definition once it - /// has moved, each described with where it is. + /// references the move leaves as they are, which may no longer reach the definition once it + /// has moved, each described with where it is pub warnings: Vec, } @@ -108,8 +111,8 @@ pub fn movable_symbol( }) } -/// Moves the top-level definition named at `offset` to the end of the module `target`, and -/// updates the project to find it there. `Err` says why it cannot be moved. +/// moves the top-level definition named at `offset` to the end of the module `target`, and +/// updates the project to find it there. `Err` says why it cannot be moved pub fn move_symbol( db: &dyn Db, file: ProgramFile<'_>, @@ -176,90 +179,30 @@ pub fn move_symbol( } let target_parsed = parsed_module(db, target_file.python_file(db)).load(db); - let target_index = semantic_index(db, target_file); - let target_globals = target_index.place_table(FileScopeId::global()); - let target_binds = |name: &str| { - target_globals - .symbol_id(name) - .is_some_and(|symbol| target_globals.symbol(symbol).is_bound()) - }; - if target_binds(name) { + if module_binds(db, target_file, name) { return Err(format!("{target_module} has a `{name}` already")); } - // a `global` in the definition names the module it is in, which the move changes - if let Some(written) = global_written_by(db, index, declaration.statement.range()) { - return Err(format!( - "`{name}` assigns {source_module}'s `{written}` through `global`, which would name \ - {target_module}'s once moved" - )); - } - - // What the definition reads from its module comes with it. - let dependencies = dependencies(db, source, &parsed, declaration.statement, name); + // what the definition reads from its module comes with it + let dependencies = checked_dependencies( + db, + source, + &parsed, + declaration.statement, + name, + Some(name), + target_file, + )?; let mut target_imports: Vec = Vec::new(); let mut from_source: Vec = Vec::new(); - for builtin in builtins_read(db, source, declaration.statement) { - if target_binds(&builtin) { - return Err(format!( - "`{name}` uses the builtin `{builtin}`, which {target_module} binds to something \ - else" - )); - } - } for dependency in &dependencies { - let read = match dependency { - Dependency::Imported { request, .. } => &request.binds, - Dependency::Sibling(sibling) => sibling, - }; - if globals.symbol_id(read).is_some_and(|symbol| { - bindings_of_symbol( - db, - index, - &parsed, - FileScopeId::global(), - symbol, - source_type.is_stub(), - ) - .len() - > 1 - }) { - return Err(format!( - "`{name}` uses `{read}`, which {source_module} binds more than once, and an import \ - of it would keep only the value it had when imported" - )); - } match dependency { - Dependency::Imported { request, .. } => { - if target_binds(&request.binds) - && !target_imports_same(db, target_file, &target_parsed, request) - { - return Err(format!( - "`{name}` uses `{}`, which {target_module} binds to something else", - request.binds - )); - } - target_imports.push(request.clone()); - } - Dependency::Sibling(sibling) => { - if is_private(sibling) { - return Err(format!( - "`{name}` uses `{sibling}`, which is private to {source_module}; make it \ - public, or move it too" - )); - } - if target_binds(sibling) { - return Err(format!( - "`{name}` uses {source_module}'s `{sibling}`, which {target_module} binds \ - to something else" - )); - } - from_source.push(sibling.clone()); - } + Dependency::Imported { request, .. } => target_imports.push(request.clone()), + Dependency::Sibling(sibling) => from_source.push(sibling.clone()), } } - // Where the definition is referred to, other than by itself. + // where the definition is referred to, other than by itself let statement_range = declaration.statement.range(); let model = SemanticModel::new(db, source); let goto_target = find_goto_target(&model, &parsed, declaration.name_range.start()) @@ -278,8 +221,8 @@ pub fn move_symbol( } let source_uses: Vec = by_file.remove(&source_file).unwrap_or_default(); - // Two modules that import each other at the top cannot both be loaded: the second one would - // read a name from the first before the first has bound it. + // two modules that import each other at the top cannot both be loaded: the second one would + // read a name from the first before the first has bound it let source_needs_target = !source_uses.is_empty() || imports_module(db, source, &parsed, &target_module); let target_needs_source = @@ -310,8 +253,8 @@ pub fn move_symbol( let mut moved = SymbolMove::default(); - // The old module: the definition cut, the imports only it used dropped, and the definition - // imported back where the module still uses it. + // the old module: the definition cut, the imports only it used dropped, and the definition + // imported back where the module still uses it let source_text = source_text(db, source_file); let source_text = source_text.as_str(); let cut = cut_range(source_text, parsed.suite(), declaration.index); @@ -371,7 +314,7 @@ pub fn move_symbol( ); } - // The target: what the definition imports, then the definition. + // the target: what the definition imports, then the definition let target_text = ruff_db::source::source_text(db, target); let target_text = target_text.as_str(); let target_end = TextSize::of(target_text); @@ -422,7 +365,7 @@ pub fn move_symbol( moved.edits.push(FileEdit { file: target, edit }); } - // Every other module that refers to it. + // every other module that refers to it let mut files: Vec = by_file.keys().copied().collect(); files.sort_by_key(|file| file.path(db).as_str().to_string()); for file in files { @@ -501,7 +444,7 @@ fn disjoint(db: &dyn Db, edits: Vec) -> Result, String> Ok(disjoint) } -/// The file and offset of the top-level declaration named at `offset`: `offset` itself when it is +/// the file and offset of the top-level declaration named at `offset`: `offset` itself when it is /// in a declaration's name, or else where the name at `offset` is defined, or else the name of the /// top-level declaration `offset` is anywhere in, as a move from inside a definition's body moves /// the definition @@ -542,7 +485,7 @@ fn referenced_declaration<'db>( Some((target_file, declaration.name_range.start())) } -/// A top-level statement that defines one name. +/// a top-level statement that defines one name struct Declaration<'a> { index: usize, statement: &'a Stmt, @@ -568,7 +511,7 @@ impl<'a> Declaration<'a> { Self::at(suite, name) } - /// The top-level declaration whose name `offset` is in. + /// the top-level declaration whose name `offset` is in fn at(suite: &'a [Stmt], offset: TextSize) -> Option { suite.iter().enumerate().find_map(|(index, statement)| { let (name, kind) = match statement { @@ -611,13 +554,13 @@ impl<'a> Declaration<'a> { } } -/// An import request whose strings it owns. +/// an import request whose strings it owns #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct OwnedRequest { pub(super) module: String, pub(super) member: Option, pub(super) asname: Option, - /// The name the import binds. + /// the name the import binds pub(super) binds: String, } @@ -634,28 +577,29 @@ impl OwnedRequest { } } -/// A name a definition reads from the top level of its module. +/// a name a definition reads from the top level of its module #[derive(Debug)] pub(super) enum Dependency { - /// A name one of the module's imports binds, as the import that binds it. + /// a name one of the module's imports binds, as the import that binds it Imported { - /// The start of the statement that imports it, or `None` for a star import, which binds - /// more than this name and so is never dropped. + /// the start of the statement that imports it, or `None` for a star import, which binds + /// more than this name and so is never dropped binding_statement: Option, alias_index: usize, request: OwnedRequest, }, - /// Another of the module's own top-level names. + /// another of the module's own top-level names Sibling(String), } -/// The names `statement` reads from the top level of `file`, other than `name`, which it defines. -pub(super) fn dependencies( +/// the names `statement` reads from the top level of `file`, other than `defines`, the name it +/// binds there itself +fn dependencies( db: &dyn Db, file: ProgramFile<'_>, parsed: &ParsedModuleRef, statement: &Stmt, - name: &str, + defines: Option<&str>, ) -> Vec { let index = semantic_index(db, file); let globals = index.place_table(FileScopeId::global()); @@ -664,7 +608,7 @@ pub(super) fn dependencies( let mut dependencies = Vec::new(); for load in loads(statement) { let id = load.id.as_str(); - if id == name || !seen.insert(id) { + if defines == Some(id) || !seen.insert(id) { continue; } let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) else { @@ -739,8 +683,133 @@ pub(super) fn dependencies( dependencies } +/// the names `statement`, moved from the top of `source` into `target`, reads from `source`'s +/// top level, each of which `target` has to import, or why the move would change what it reads: +/// a `global` write to a name of `source`, which would name `target`'s once moved; a name +/// `source` binds more than once, an import of which would keep only the value it had when +/// imported; a name private to `source`; an attribute of the module object, `__name__` or +/// `__file__`, which would be `target`'s; and a builtin, an imported name or a name of `source` +/// that `target` binds to something else. `subject` names what moves in a refusal, and `defines` +/// is the name `statement` itself binds at the top of `source`, which it reads as itself +pub(super) fn checked_dependencies( + db: &dyn Db, + source: ProgramFile<'_>, + parsed: &ParsedModuleRef, + statement: &Stmt, + subject: &str, + defines: Option<&str>, + target: ProgramFile<'_>, +) -> Result, String> { + let module_name = |file: ProgramFile<'_>| { + file_to_module(db, file.resolver_file(db)) + .map(|module| module.name(db).to_string()) + .unwrap_or_default() + }; + let source_module = module_name(source); + let target_module = module_name(target); + let index = semantic_index(db, source); + let globals = index.place_table(FileScopeId::global()); + + // a `global` in it names the module it is in, which the move changes + if let Some(written) = global_written_by(db, index, statement.range()) { + return Err(format!( + "`{subject}` assigns {source_module}'s `{written}` through `global`, which would \ + name {target_module}'s once moved" + )); + } + + let model = SemanticModel::new(db, source); + let mut builtins: Vec<&str> = Vec::new(); + for load in loads(statement) { + let id = load.id.as_str(); + if reads_module_implicit_global(&model, load) { + return Err(format!( + "`{subject}` reads `{id}`, which is {source_module}'s own and would be \ + {target_module}'s once moved" + )); + } + if let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) + && resolving_scope(index, scope, id).is_none() + && !builtins.contains(&id) + { + builtins.push(id); + } + } + for builtin in builtins { + if module_binds(db, target, builtin) { + return Err(format!( + "`{subject}` uses the builtin `{builtin}`, which {target_module} binds to \ + something else" + )); + } + } + + let dependencies = dependencies(db, source, parsed, statement, defines); + let target_parsed = parsed_module(db, target.python_file(db)).load(db); + for dependency in &dependencies { + let read = match dependency { + Dependency::Imported { request, .. } => &request.binds, + Dependency::Sibling(sibling) => sibling, + }; + if globals.symbol_id(read).is_some_and(|symbol| { + bindings_of_symbol( + db, + index, + parsed, + FileScopeId::global(), + symbol, + source.file(db).is_stub(db), + ) + .len() + > 1 + }) { + return Err(format!( + "`{subject}` uses `{read}`, which {source_module} binds more than once, and an \ + import of it would keep only the value it had when imported" + )); + } + match dependency { + Dependency::Imported { request, .. } => { + if module_binds(db, target, &request.binds) + && !target_imports_same(db, target, &target_parsed, request) + { + return Err(format!( + "`{subject}` uses `{}`, which {target_module} binds to something else", + request.binds + )); + } + } + Dependency::Sibling(sibling) => { + if is_private(sibling) { + return Err(format!( + "`{subject}` uses `{sibling}`, which is private to {source_module}; make \ + it public, or move it too" + )); + } + if module_binds(db, target, sibling) { + return Err(format!( + "`{subject}` uses {source_module}'s `{sibling}`, which {target_module} \ + binds to something else" + )); + } + } + } + } + Ok(dependencies) +} + +/// whether the top level of `file` binds `name`, or the module object gives it to every module +/// as one of its attributes, `__name__` or `__doc__` +fn module_binds(db: &dyn Db, file: ProgramFile<'_>, name: &str) -> bool { + let globals = semantic_index(db, file).place_table(FileScopeId::global()); + globals + .symbol_id(name) + .is_some_and(|symbol| globals.symbol(symbol).is_bound()) + || is_module_implicit_global(db, file, name) +} + /// the names `statement` reads -fn loads(statement: &Stmt) -> Vec<&ast::ExprName> { +pub(super) fn loads(statement: &Stmt) -> Vec<&ast::ExprName> { struct Loads<'a> { names: Vec<&'a ast::ExprName>, } @@ -759,22 +828,6 @@ fn loads(statement: &Stmt) -> Vec<&ast::ExprName> { loads.names } -/// the names `statement` reads that no scope of `file` binds, the builtins among them -fn builtins_read(db: &dyn Db, file: ProgramFile<'_>, statement: &Stmt) -> Vec { - let index = semantic_index(db, file); - let mut read: Vec = Vec::new(); - for load in loads(statement) { - let id = load.id.as_str(); - if let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(load)) - && resolving_scope(index, scope, id).is_none() - && !read.iter().any(|seen| seen == id) - { - read.push(id.to_string()); - } - } - read -} - /// a name of the module's own that a `global` in a scope inside `range` assigns fn global_written_by(db: &dyn Db, index: &SemanticIndex<'_>, range: TextRange) -> Option { index @@ -871,13 +924,13 @@ fn imported_from( }) } -/// Whether `name` is private to its module by python's convention: it starts with an underscore -/// and is not a dunder. -pub(super) fn is_private(name: &str) -> bool { +/// whether `name` is private to its module by python's convention: it starts with an underscore +/// and is not a dunder +fn is_private(name: &str) -> bool { name.starts_with('_') && !(name.starts_with("__") && name.ends_with("__")) } -/// Whether `file` already imports what `request` asks for, under the name it asks for. +/// whether `file` already imports what `request` asks for, under the name it asks for fn target_imports_same( db: &dyn Db, file: ProgramFile<'_>, @@ -905,7 +958,7 @@ fn target_imports_same( }) } -/// Whether a top-level import of `file` imports `module`, or something from it. +/// whether a top-level import of `file` imports `module`, or something from it pub(super) fn imports_module( db: &dyn Db, file: ProgramFile<'_>, @@ -929,7 +982,7 @@ pub(super) fn imports_module( }) } -/// Whether `name` is read anywhere in the module outside `except`, or listed in its `__all__`. +/// whether `name` is read anywhere in the module outside `except`, or listed in its `__all__` fn used_elsewhere(parsed: &ParsedModuleRef, name: &str, except: TextRange) -> bool { struct Uses<'a> { name: &'a str, @@ -965,15 +1018,15 @@ fn used_elsewhere(parsed: &ParsedModuleRef, name: &str, except: TextRange) -> bo uses.found } -/// The text a cut takes, and the range it removes from the module. +/// the text a cut takes, and the range it removes from the module pub(super) struct Cut { - /// The definition with its decorators and the comments directly above it, ending in a line - /// break. + /// the definition with its decorators and the comments directly above it, ending in a line + /// break pub(super) text: String, /// where `text` starts in the module pub(super) start: TextSize, - /// Its lines and the blank lines that separated it from the statement after it, or, when it is - /// the last statement, from the one before. + /// its lines and the blank lines that separated it from the statement after it, or, when it is + /// the last statement, from the one before pub(super) removed: TextRange, } @@ -1052,8 +1105,8 @@ pub(super) fn cut_range(source: &str, suite: &[Stmt], index: usize) -> Cut { } } -/// The text appended to the end of the target: the definition, set apart from what is above it -/// as PEP 8 sets apart a top-level definition. +/// the text appended to the end of the target: the definition, set apart from what is above it +/// as PEP 8 sets apart a top-level definition fn appended_text( target: &str, suite: &[Stmt], @@ -1095,8 +1148,8 @@ fn appended_text( text } -/// The edit removing alias `alias_index` from the import statement starting at `statement`, or the -/// whole statement when it is its only alias. +/// the edit removing alias `alias_index` from the import statement starting at `statement`, or the +/// whole statement when it is its only alias fn remove_alias( source: &str, suite: &[Stmt], @@ -1126,7 +1179,7 @@ fn remove_alias( Some(Edit::range_deletion(range)) } -/// The `__all__ = [...]` (or tuple) assignment at the top level of a module. +/// the `__all__ = [...]` (or tuple) assignment at the top level of a module fn all_list(suite: &[Stmt]) -> Option<&[Expr]> { suite.iter().find_map(|stmt| { let (target, value) = match stmt { @@ -1145,7 +1198,7 @@ fn all_list(suite: &[Stmt]) -> Option<&[Expr]> { }) } -/// The edit taking `name` out of the module's `__all__`, when it lists it. +/// the edit taking `name` out of the module's `__all__`, when it lists it fn remove_from_all(suite: &[Stmt], name: &str) -> Option { let elements = all_list(suite)?; let position = elements.iter().position( @@ -1162,7 +1215,7 @@ fn remove_from_all(suite: &[Stmt], name: &str) -> Option { Some(Edit::range_deletion(range)) } -/// The edit adding `name` to the end of the module's `__all__`, when it has one. +/// the edit adding `name` to the end of the module's `__all__`, when it has one fn add_to_all(source: &str, suite: &[Stmt], name: &str) -> Option { let elements = all_list(suite)?; let quote = elements @@ -1175,7 +1228,7 @@ fn add_to_all(source: &str, suite: &[Stmt], name: &str) -> Option { .map(|last| Edit::insertion(format!(", {quote}{name}{quote}"), last.end())) } -/// Points the references `ranges` in `file` at `name`'s new module. +/// points the references `ranges` in `file` at `name`'s new module fn rewrite_references( db: &dyn Db, file: ProgramFile<'_>, @@ -1389,7 +1442,7 @@ fn from_module_range(parsed: &ParsedModuleRef, from: &ast::StmtImportFrom) -> Op Some(TextRange::new(first.start(), end)) } -/// The one-based line `offset` is on. +/// the one-based line `offset` is on pub(super) fn line_number(source: &str, offset: TextSize) -> usize { source[..offset.to_usize()].matches('\n').count() + 1 } @@ -1440,8 +1493,8 @@ import os X = os.sep "#; - /// A project of the modules above and modules using `area`, with the cursor where `src` - /// gives it, in `mv/src.py`. + /// a project of the modules above and modules using `area`, with the cursor where `src` + /// gives it, in `mv/src.py` fn project(src: &str) -> CursorTest { CursorTestBuilder::default() .source("mv/__init__.py", "") @@ -1517,7 +1570,7 @@ X = os.sep } impl CursorTest { - /// Every module the move to `mv/dst.py` changes, as it is afterwards, or why it refuses. + /// every module the move to `mv/dst.py` changes, as it is afterwards, or why it refuses fn move_to_dst(&self) -> String { let target = system_path_to_file(&self.db, "mv/dst.py").expect("dst to exist"); let moved = salsa::attach(&self.db, || { @@ -1532,9 +1585,9 @@ X = os.sep } } - /// The definition goes to the end of the target with the import it uses and the name of + /// the definition goes to the end of the target with the import it uses and the name of /// its old module it reads; the old module drops the import nothing else uses and its - /// `__all__` entry; and every way the project refers to it follows it. + /// `__all__` entry; and every way the project refers to it follows it #[test] fn a_function_moves_with_what_it_reads() { let test = project(&SRC.replace("def area(r)", "def area(r)")); @@ -1600,8 +1653,8 @@ X = os.sep "#); } - /// The old module still calls it, and it reads `RATE` from the old module, so the two - /// modules would import each other, which python cannot load. + /// the old module still calls it, and it reads `RATE` from the old module, so the two + /// modules would import each other, which python cannot load #[test] fn a_move_that_makes_two_modules_import_each_other_is_refused() { let test = project(&format!( @@ -1652,7 +1705,7 @@ X = os.sep assert_snapshot!(test.move_to_dst(), @"refused: `uses_private` uses `_SECRET`, which is private to mv.src; make it public, or move it too"); } - /// A variable the old module still reads is imported back from its new module. + /// a variable the old module still reads is imported back from its new module #[test] fn a_variable_the_old_module_reads_is_imported_back() { let test = project(&SRC.replace("RATE = 2", "RATE = 2")); @@ -1694,7 +1747,7 @@ X = os.sep "#); } - /// Asked at a reference, the definition the reference is to is moved. + /// asked at a reference, the definition the reference is to is moved #[test] fn asked_at_a_reference() { let test = CursorTestBuilder::default() @@ -1912,6 +1965,29 @@ X = os.sep assert_snapshot!(test.move_to_dst(), @"refused: `size` uses the builtin `len`, which mv.dst binds to something else"); } + /// `__name__` is the module's own, and would be the target's once moved + #[test] + fn a_definition_that_reads_an_attribute_of_its_module_is_refused() { + let test = modules("def origin():\n return __name__\n", "", &[]); + assert_snapshot!(test.move_to_dst(), @"refused: `origin` reads `__name__`, which is mv.src's own and would be mv.dst's once moved"); + } + + /// a `__name__` the definition binds itself is read as any other name of its own + #[test] + fn a_definition_that_binds_a_module_attribute_s_name_itself_moves() { + let test = modules( + "def origin(__name__):\n return __name__\n", + "", + &[], + ); + assert_snapshot!(test.move_to_dst(), @" + --- /mv/dst.py + def origin(__name__): + return __name__ + --- /mv/src.py + "); + } + /// the module of an import is found by its tokens, so a comment that says `import` is not /// taken for the keyword #[test] diff --git a/crates/ty_python_semantic/src/importer.rs b/crates/ty_python_semantic/src/importer.rs index ea08eeed09..d80337c780 100644 --- a/crates/ty_python_semantic/src/importer.rs +++ b/crates/ty_python_semantic/src/importer.rs @@ -1194,10 +1194,10 @@ pub struct MembersInScope<'ast> { impl<'ast> MembersInScope<'ast> { /// An empty scope for importing a name that is already known to be unbound. - /// This avoids querying inferred types while constructing a diagnostic fix, and serves a + /// this avoids querying inferred types while constructing a diagnostic fix, and serves a /// refactoring that has checked itself that the names it imports are free where it imports /// them, as one moving a definition does: the definition is still in scope where it is - /// moving from, and would otherwise be taken for a conflict. + /// moving from, and would otherwise be taken for a conflict pub fn empty(at: TextSize) -> Self { Self { at, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 9d8c81d471..7187729d46 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -9,7 +9,8 @@ use crate::place::{ imported_symbol, place_by_id, place_from_bindings, }; use crate::place_load::{ - PlaceLoadMode, PlaceLoadResolutionStep, PlaceLoadSourceKind, resolve_place_load, + ImplicitPlaceLoad, PlaceLoadMode, PlaceLoadResolutionStep, PlaceLoadSourceKind, + resolve_place_load, }; use crate::reachability::is_range_reachable; use crate::types::EnumLiteralType; @@ -2757,6 +2758,108 @@ pub fn is_within_classes<'db>( }) } +/// basedpython: whether the name `name`, read in `model`'s file, may read one of the attributes +/// python gives the module object itself, `__name__`, `__file__` or `__doc__`, which each module +/// has its own value of. it does where the read finds nothing sure to be bound before it reaches +/// the module's implicit globals +pub fn reads_module_implicit_global(model: &SemanticModel<'_>, name: &ast::ExprName) -> bool { + let db = model.db(); + load_may_reach(model, name, |implicit| match implicit { + ImplicitPlaceLoad::ModuleImplicitGlobal { file, name } => { + Some(is_module_implicit_global(db, *file, name)) + } + _ => None, + }) +} + +/// basedpython: whether the name `name`, read in `model`'s file, may read the `__class__` cell +/// python gives a method, the class the method is written in. it does where the read finds +/// nothing sure to be bound before it reaches the cell +pub fn reads_class_cell(model: &SemanticModel<'_>, name: &ast::ExprName) -> bool { + load_may_reach(model, name, |implicit| { + matches!(implicit, ImplicitPlaceLoad::DunderClass(_)).then_some(true) + }) +} + +/// whether the load of `name` in `model`'s file may reach an implicit source `decides` answers +/// for, and its answer there: the load goes on past a source while nothing it has passed is sure +/// to be bound +fn load_may_reach<'db>( + model: &SemanticModel<'db>, + name: &ast::ExprName, + decides: impl Fn(&ImplicitPlaceLoad<'db>) -> Option, +) -> bool { + let db = model.db(); + let file = model.program_file(); + let env = ProgramEnvironment::from_file(file); + let index = semantic_index(db, file); + let Some(scope) = index.try_expression_scope_id(&ast::ExprRef::Name(name)) else { + return false; + }; + let scope = scope.to_scope_id(db, file); + let mut resolution = resolve_place_load( + db, + index, + scope, + PlaceExpr::from_expr_name(name), + PlaceLoadMode::AtExpression(name.into()), + ); + while let Some(PlaceLoadResolutionStep::Source(source)) = resolution.next() { + let place = match source.kind { + PlaceLoadSourceKind::Bindings(bindings) => { + place_from_bindings(db, &env, bindings).place + } + PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { + place_by_id( + db, + scope, + id, + RequiresExplicitReExport::No, + ConsideredDefinitions::AllReachable, + ) + .place + } + PlaceLoadSourceKind::Implicit(implicit) => { + if let Some(answer) = decides(&implicit) { + return answer; + } + implicit.place(db, &env, scope).place + } + // only a program carrying a debugger's assumptions observes a place + PlaceLoadSourceKind::Observed(_) => return false, + }; + if place.is_definitely_bound() { + return false; + } + } + false +} + +/// basedpython: whether `name` is one of the attributes python gives every module object, which +/// the module reads as a global where it binds no `name` of its own: `__name__`, `__file__`, +/// `__doc__` +pub fn is_module_implicit_global(db: &dyn Db, file: ProgramFile<'_>, name: &str) -> bool { + !crate::place::module_type_implicit_global_symbol(db, file, name) + .place + .is_undefined() +} + +/// basedpython: whether `class` is an enum class, a subclass of `enum.Enum` or a class whose +/// metaclass is `enum.EnumType`, which python makes a subclass of something other than what the +/// class itself is +pub fn is_enum_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: Type<'db>, +) -> bool { + match class { + Type::ClassLiteral(ClassLiteral::Static(class)) => { + crate::types::enums::is_enum_class_by_inheritance(db, env, class) + } + _ => false, + } +} + /// Enumerates the reachable class definitions in `file`. fn reachable_class_literals_in_file<'db>( db: &'db dyn Db, @@ -4785,6 +4888,16 @@ pub fn is_dataclass_field_function<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool ) } +/// basedpython: whether `ty` is `typing.override`, the decorator that marks a method as +/// overriding one of a superclass +pub fn is_override_function<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + matches!( + ty.as_function_literal() + .and_then(|function| function.known(db)), + Some(KnownFunction::Override) + ) +} + /// Whether `ty` is the `reveal_type` function. pub fn is_reveal_type_function<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { matches!( diff --git a/crates/ty_server/src/server/api/requests/move_members.rs b/crates/ty_server/src/server/api/requests/move_members.rs index cdac5ebe84..fb14d6e520 100644 --- a/crates/ty_server/src/server/api/requests/move_members.rs +++ b/crates/ty_server/src/server/api/requests/move_members.rs @@ -8,11 +8,13 @@ use lsp_types::{ LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, }; use ruff_db::files::FileRange; +use ruff_text_size::TextSize; use ty_ide::{ HierarchyClass, MemberKind, extract_superclass, movable_members, pull_members_up, push_members_down, }; use ty_project::{ProjectDatabase, SemanticDb as _}; +use ty_python_core::ProgramFile; use super::move_symbol::MoveSymbolAnswer; use crate::document::{FileRangeExt, PositionExt}; @@ -194,13 +196,18 @@ impl Request for PullMembersUpRequest { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } -/// the class, the names of the members to pull up, and the superclass to pull them into +/// the class, the members and bases to pull up, and the superclass to pull them into #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct PullMembersUpParams { text_document: TextDocumentIdentifier, position: Position, + /// the names of the members to pull up, as `by/prepareMoveMembers` lists them. without + /// `bases`, the bases to pull up are named here too, each as listed with the kind `base` members: Vec, + /// the bases to pull up, each as `by/prepareMoveMembers` lists one of the kind `base` + #[serde(default, skip_serializing_if = "Option::is_none")] + bases: Option>, /// the superclass's `qualifiedName`, as `by/prepareMoveMembers` lists it target: String, } @@ -226,20 +233,56 @@ impl BackgroundDocumentRequestHandler for PullMembersUpRequestHandler { let Some((file, offset)) = position(db, snapshot, params.position) else { return Ok(None); }; + let file = db.program_file(file); + let parts = match params.bases { + Some(bases) => Ok((params.members, bases)), + None => members_and_bases(db, file, offset, params.members), + }; Ok(Some(MoveSymbolAnswer::of( db, snapshot, - pull_members_up( - db, - db.program_file(file), - offset, - ¶ms.members, - ¶ms.target, - ), + parts.and_then(|(members, bases)| { + pull_members_up(db, file, offset, &members, &bases, ¶ms.target) + }), ))) } } +/// `names`, a request's `members` that names the bases to pull up among the members, split into +/// the members and the bases by the kind `by/prepareMoveMembers` lists each with. a name it lists +/// as both is refused, and one it does not list is left with the members, for the pull to refuse +fn members_and_bases( + db: &ProjectDatabase, + file: ProgramFile<'_>, + offset: TextSize, + names: Vec, +) -> Result<(Vec, Vec), String> { + let movable = + movable_members(db, file, offset).ok_or_else(|| "there is no class here".to_string())?; + let mut members = Vec::new(); + let mut bases = Vec::new(); + for name in names { + let listed = |base: bool| { + movable + .members + .iter() + .any(|member| member.name == name && (member.kind == MemberKind::Base) == base) + }; + match (listed(false), listed(true)) { + (true, true) => { + return Err(format!( + "`{name}` is both a member of {} and one of its bases; name the bases to pull \ + up in `bases`", + movable.class.name + )); + } + (false, true) => bases.push(name), + _ => members.push(name), + } + } + Ok((members, bases)) +} + impl RetriableRequestHandler for PullMembersUpRequestHandler { /// the user is waiting on the move they asked for const RETRY_ON_CANCELLATION: bool = true; diff --git a/crates/ty_server/tests/e2e/move_symbol.rs b/crates/ty_server/tests/e2e/move_symbol.rs index 2868cdb16f..619ecf0e99 100644 --- a/crates/ty_server/tests/e2e/move_symbol.rs +++ b/crates/ty_server/tests/e2e/move_symbol.rs @@ -343,6 +343,72 @@ fn a_pull_up_answers_its_edit_and_a_push_down_with_nowhere_to_go_is_refused() -> Ok(()) } +const MIXED: &str = "\ +class Mixin: + pass + + +class Shape: + pass + + +class Square(Shape, Mixin): + Mixin = 1 +"; + +/// a pull names the bases it moves apart from the members, so a base and a member written with +/// one name are told apart; a request that names its bases among the members, as an older client +/// sends it, is read by the kinds `by/prepareMoveMembers` lists them with +#[test] +fn a_pull_up_names_its_bases_apart_from_its_members() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/shapes.py"), MIXED)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/shapes.py"), MIXED, 1); + let uri = server.file_uri(SystemPath::new("src/shapes.py")); + let pulled = server + .send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 8, "character": 7 }, + "members": [], + "bases": ["Mixin"], + "target": "shapes.Shape", + })) + .expect("an answer"); + assert_eq!( + pulled["edit"]["changes"][uri.as_str()], + json!([ + { + "range": { "start": { "line": 8, "character": 18 }, "end": { "line": 8, "character": 25 } }, + "newText": "", + }, + { + "range": { "start": { "line": 4, "character": 11 }, "end": { "line": 4, "character": 11 } }, + "newText": "(Mixin)", + }, + ]), + "{pulled:#}" + ); + + let ambiguous = server.send_request_await::(json!({ + "textDocument": { "uri": uri }, + "position": { "line": 8, "character": 7 }, + "members": ["Mixin"], + "target": "shapes.Shape", + })); + assert_eq!( + ambiguous, + Some(json!({ + "refused": "`Mixin` is both a member of Square and one of its bases; name the bases \ + to pull up in `bases`", + "warnings": [], + })) + ); + Ok(()) +} + /// `by/extractSuperclass`, in json throughout enum ExtractSuperclass {} From 80f6ffc83efd84e3052d654b10b0f8dc9ba1b466 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:29:30 +1000 Subject: [PATCH 086/110] leave out without includeDeclaration only what declares the name in the scope that declares it, rather than every import of it and every assignment that is the first in its own scope, so references and by/references keep an import of the name, a write to it through its module or an instance, from another method or through global, as uses, and the usage counts drawn over a declaration count its imports --- crates/ty_ide/src/find_references.rs | 113 +++++++++++++++++++++ crates/ty_ide/src/references.rs | 57 +++++++++-- crates/ty_ide/src/usage_counts.rs | 2 +- crates/ty_server/tests/e2e/usage_counts.rs | 8 +- docs/basedpython/features/editor.md | 7 ++ 5 files changed, 175 insertions(+), 12 deletions(-) diff --git a/crates/ty_ide/src/find_references.rs b/crates/ty_ide/src/find_references.rs index f553501806..b7d328b70d 100644 --- a/crates/ty_ide/src/find_references.rs +++ b/crates/ty_ide/src/find_references.rs @@ -2469,6 +2469,119 @@ print(x) "); } + /// an import of the target declares a name of the importing module's own, and is a use of + /// the target, as is an assignment to it through the module + #[test] + fn without_declaration_keeps_imports_and_writes_from_other_modules() { + let test = CursorTest::builder() + .source("lib.py", "def foo(): ...\nx = 1\n") + .source( + "main.py", + "from lib import foo\nimport lib\nfoo()\nlib.foo = foo\n", + ) + .build(); + + assert_snapshot!(test.references_without_declaration(), @" + info[references]: Found 4 references + --> main.py:1:17 + | + 1 | from lib import foo + | --- + 2 | import lib + 3 | foo() + | --- + 4 | lib.foo = foo + | --- --- + "); + } + + /// only the assignment that declares an attribute is its declaration: one from another method, + /// or through an instance, writes it + #[test] + fn without_declaration_keeps_attribute_writes_outside_the_declaring_scope() { + let test = cursor_test( + " +class C: + def __init__(self): + self.x = 1 + + def reset(self): + self.x = 2 + +c = C() +c.x = 3 +print(c.x) +", + ); + + assert_snapshot!(test.references_without_declaration(), @" + info[references]: Found 3 references + --> main.py:7:14 + | + 7 | self.x = 2 + | - + 8 | + 9 | c = C() + 10 | c.x = 3 + | - + 11 | print(c.x) + | - + "); + } + + /// a function that rebinds a module's name through `global` writes it + #[test] + fn without_declaration_keeps_a_global_write() { + let test = cursor_test( + " +x = 1 +def f(): + global x + x = 2 +print(x) +", + ); + + assert_snapshot!(test.references_without_declaration(), @" + info[references]: Found 3 references + --> main.py:4:12 + | + 4 | global x + | - + 5 | x = 2 + | - + 6 | print(x) + | - + "); + } + + /// a decorated `def` resolves to what its decorator returns, but is still where its name is + /// declared + #[test] + fn without_declaration_excludes_a_decorated_def() { + let test = cursor_test( + " +def logged(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + +@logged +def run(): ... + +run() +", + ); + + assert_snapshot!(test.references_without_declaration(), @" + info[references]: Found 1 references + --> main.py:10:1 + | + 10 | run() + | --- + "); + } + #[test] fn without_declaration_excludes_initial_attribute_assignment() { let test = cursor_test( diff --git a/crates/ty_ide/src/references.rs b/crates/ty_ide/src/references.rs index 0f2a961948..62e40b0286 100644 --- a/crates/ty_ide/src/references.rs +++ b/crates/ty_ide/src/references.rs @@ -28,9 +28,9 @@ use rustc_hash::{FxHashMap, FxHashSet}; use ty_project::parallel::{ParallelIteratorExt, minimum_parallel_job_len}; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind, DefinitionState}; -use ty_python_core::scope::{FileScopeId, NodeWithScopeKind, ScopeKind}; +use ty_python_core::scope::{FileScopeId, NodeWithScopeKind, ScopeId, ScopeKind}; use ty_python_semantic::{ - Db as SemanticDb, FixtureExposure, FixtureNameSource, ImportAliasResolution, + Db as SemanticDb, FixtureExposure, FixtureNameSource, HasDefinition, ImportAliasResolution, ResolvedDefinition, SemanticModel, fixture_bindings_for_parameter, fixture_exposures_for_definition, pytest_global_plugin_files, }; @@ -243,9 +243,26 @@ impl<'db> PreparedSearch<'db> { ) -> Option { let target_text = goto_target.to_string()?.into_owned(); - let target_definitions = goto_target - .definitions(model, mode.to_import_alias_resolution())? - .goto_declaration(model, goto_target, ClassCallName::Class)?; + let definitions = goto_target.definitions(model, mode.to_import_alias_resolution())?; + // what the target is bound by as written, as well as what declares it + let mut declaring_scopes: FxHashSet> = definitions + .iter() + .filter_map(ResolvedDefinition::definition) + .map(|definition| definition.scope(model.db())) + .collect(); + // a decorated `def` resolves to what its decorator returns, which is declared somewhere + // else, but the `def` declares the name where it is written + if let GotoTarget::FunctionDef(function) = goto_target { + declaring_scopes.insert(function.definition(model).scope(model.db())); + } + let target_definitions = + definitions.goto_declaration(model, goto_target, ClassCallName::Class)?; + declaring_scopes.extend( + target_definitions + .iter() + .filter_map(ResolvedDefinition::definition) + .map(|definition| definition.scope(model.db())), + ); let import_alias_resolution = mode.to_import_alias_resolution(); // An identifier can have both ordinary Python references and pytest fixture references. // Keep its Python definitions alongside any fixture roots used for the same search. @@ -270,6 +287,7 @@ impl<'db> PreparedSearch<'db> { search: LocalReferenceSearch { target_text, target_definitions, + declaring_scopes, import_alias_resolution, fixture_resolution, }, @@ -558,6 +576,7 @@ impl<'db> FixtureReferenceTarget<'db> { let search = LocalReferenceSearch { target_text: self.name.to_string(), target_definitions: Definitions::new(Vec::new()), + declaring_scopes: FxHashSet::default(), import_alias_resolution: ImportAliasResolution::PreserveAliases, fixture_resolution: Some(self.resolution), }; @@ -604,6 +623,10 @@ impl<'db> FixtureReferenceTarget<'db> { struct LocalReferenceSearch<'db> { target_text: String, target_definitions: Definitions<'db>, + /// the scopes the target is declared in. without `includeDeclaration`, what declares the + /// target in one of them is left out, and what binds it anywhere else is a use of it: an + /// import of it, an attribute assignment through an instance or from another method + declaring_scopes: FxHashSet>, import_alias_resolution: ImportAliasResolution, fixture_resolution: Option>, } @@ -969,9 +992,15 @@ impl<'a> LocalReferencesFinder<'a> { // A parameter declares a Python local but references the fixture it requests. // Keep fixture requests even when the fixture declaration is excluded. OccurrenceKind::Declaration if fixture_request_match => false, - OccurrenceKind::Declaration => true, + // a fixture is exposed under the name its definition or an import binds, each + // a declaration of that exposure wherever it is + OccurrenceKind::Declaration if fixture_match => true, + OccurrenceKind::Declaration => self.is_in_declaring_scope(covering_node), OccurrenceKind::Reference => false, - OccurrenceKind::Binding => self.is_declaration(covering_node), + OccurrenceKind::Binding => { + (fixture_match || self.is_in_declaring_scope(covering_node)) + && self.is_declaration(covering_node) + } }; if is_declaration { @@ -1171,6 +1200,20 @@ impl<'a> LocalReferencesFinder<'a> { }) } + /// whether the occurrence binds the target in one of the scopes the target is declared in, + /// where binding it declares it, as opposed to using it. an import declares a name of the + /// importing module's own, an attribute assignment through an instance or from another + /// method writes the attribute, and a `global` statement names it + fn is_in_declaring_scope(&self, covering_node: &CoveringNode<'_>) -> bool { + self.model + .first_local_definition(covering_node) + .is_some_and(|definition| { + self.search + .declaring_scopes + .contains(&definition.scope(self.model.db())) + }) + } + fn is_declaration(&self, covering_node: &CoveringNode<'_>) -> bool { let db = self.model.db(); diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 93b98e85ba..45c0064809 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -386,7 +386,7 @@ print(Shape().area(), Square().area()) ) .build(); assert_snapshot!(test.usage_counts(), @" - class Shape 2 refs=2 impls=1 + class Shape 2 refs=3 impls=1 method area 3 refs=1 impls=1 function unused 6 refs=0 "); diff --git a/crates/ty_server/tests/e2e/usage_counts.rs b/crates/ty_server/tests/e2e/usage_counts.rs index 3d84c1e49b..374e672717 100644 --- a/crates/ty_server/tests/e2e/usage_counts.rs +++ b/crates/ty_server/tests/e2e/usage_counts.rs @@ -168,10 +168,10 @@ fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result assert_eq!( counts, [ - ("Shape".to_string(), Some(2), Some(1), Some(false)), + ("Shape".to_string(), Some(3), Some(1), Some(false)), ("area".to_string(), Some(1), Some(1), Some(true)), ("describe".to_string(), Some(1), Some(0), Some(false)), - ("total".to_string(), Some(1), None, Some(false)), + ("total".to_string(), Some(2), None, Some(false)), ] ); @@ -348,8 +348,8 @@ fn a_count_the_client_cancels_while_it_runs_is_answered_once_and_the_next_is_ans .expect("an answer for a document"); let shape = &declarations.as_array().expect("a list of declarations")[0]; assert_eq!(shape["name"], "Shape"); - // `total` names it, and `main.by` and every module subclass it (an import is not a usage) - assert_eq!(shape["references"].as_u64(), Some(MODULES as u64 + 2)); + // `total` names it, and `main.by` and every module import it and subclass it + assert_eq!(shape["references"].as_u64(), Some(2 * MODULES as u64 + 3)); assert_eq!(shape["implementations"].as_u64(), Some(MODULES as u64 + 1)); // and the cancelled count does not bring a second answer diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 69bad9b246..8763f1ddbc 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1258,6 +1258,13 @@ a reference is the first of these that holds, from the top, so an import is an class in `cast`'s first argument in a base list is a `typeHint`. the answer is `null` where `textDocument/references` answers `null` +without `includeDeclaration`, both leave out only where the name is declared, in +the scope that declares it: its `def` or `class`, an annotation of it, or an +assignment no other assignment of it comes before. what binds it anywhere else +uses it and stays, as PyCharm's Python counts it: an import of it, an assignment +to it through its module or through an instance, from another method than the +one that declares the attribute, or from a function naming it `global` + ## type info an editor's *type info* shows the type of an expression, and which expression From 21e305f6550d5da58d5d733ddb96b81ec908b78d Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:37:32 +1000 Subject: [PATCH 087/110] indent a postfix statement's new line one level past the line it is written on for a client that indents each new line of a completion as that line, which a client is taken to do unless its completion.insertTextMode says asIs, and tell one that takes insertTextMode per item so, rather than writing the line's own indentation in as well, which the JetBrains client indented a second time --- crates/ty_ide/src/completion.rs | 49 +++++++++++- crates/ty_server/src/capabilities.rs | 37 ++++++++++ .../src/server/api/requests/completion.rs | 20 ++++- .../server/api/requests/smart_completion.rs | 10 +-- crates/ty_server/tests/e2e/completions.rs | 74 +++++++++++++++++++ crates/ty_server/tests/e2e/main.rs | 12 +++ docs/basedpython/features/editor.md | 8 ++ 7 files changed, 198 insertions(+), 12 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index b597db3760..ebdc1ce1aa 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -382,12 +382,25 @@ fn replacing(completions: Vec>, typed: TextRange) -> Vec Self { - Self { snippets } + Self { snippets, ..self } + } + + #[must_use] + pub fn adjusts_indentation(self, adjusts_indentation: bool) -> Self { + Self { + adjusts_indentation, + ..self + } } } @@ -4065,9 +4078,15 @@ fn add_postfix_completions<'db>( // the client's own idea of the word under the cursor is the attribute // alone, but a rewrite consumes the expression it is written on as well let replace = TextRange::new(value.start(), expr.range().end().max(cursor.offset)); - // a statement-shaped rewrite needs a line of its own to continue onto + // a statement-shaped rewrite needs a line of its own to continue onto, indented a level past + // the statement's own. a client that indents each new line as the line the completion is + // accepted on adds the statement's own indentation itself let indent = cursor.statement_expression(expr).map(|statement| { - let own = line_indentation(source.as_str(), statement.start()); + let own = if capabilities.adjusts_indentation { + "" + } else { + line_indentation(source.as_str(), statement.start()) + }; let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); format!("{own}{}", stylist.indentation().as_str()) }); @@ -14496,6 +14515,25 @@ count = 3 ); } + /// A client that indents each new line as the line the completion is + /// accepted on gets the new line indented one level past that line, and + /// adds the statement's own indentation itself. + #[test] + fn basedpython_postfix_statement_indentation_for_a_client_that_adjusts_it() { + let builder = CursorTest::builder() + .source( + "main.by", + "def f(x: list[int]):\n if x:\n x.", + ) + .completion_test_builder() + .adjusts_indentation(true) + .filter(|c| c.name == "for"); + assert_eq!( + inserted_text(&builder.build(), "for"), + "for item in x:\n " + ); + } + /// A statement-shaped rewrite has no statement to replace when the /// expression is only part of one. #[test] @@ -17018,6 +17056,11 @@ raise self } + fn adjusts_indentation(mut self, adjusts: bool) -> CompletionTestBuilder { + self.capabilities = self.capabilities.adjusts_indentation(adjusts); + self + } + /// When set, builtins from completions are skipped. This is /// useful in tests to reduce noise for scope based completions. /// diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index 959d3a61d6..0ebfcb0f5d 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -50,6 +50,8 @@ bitflags::bitflags! { const FOLDING_RANGE_KIND_STRING = 1 << 31; const FOLDING_RANGE_KIND_COLLECTION = 1 << 32; const FOLDING_RANGE_KIND_ANNOTATION = 1 << 33; + const COMPLETION_ITEM_ADJUST_INDENTATION = 1 << 34; + const COMPLETION_INSERTS_AS_IS = 1 << 35; } } @@ -232,6 +234,23 @@ impl ResolvedClientCapabilities { self.contains(Self::COMPLETION_ITEM_SNIPPET_SUPPORT) } + /// Returns `true` if a completion item can ask the client to indent each new line of its + /// text as the line the item is accepted on (`insertTextMode: adjustIndentation`). + pub(crate) const fn supports_completion_item_adjust_indentation(self) -> bool { + self.contains(Self::COMPLETION_ITEM_ADJUST_INDENTATION) + } + + /// Returns `true` if the client indents each new line of a completion's text as the line the + /// completion is accepted on: a completion that asks it to, where it can be asked, and + /// otherwise every one, unless the client says it inserts the text as it is. + /// + /// a client that says neither is taken to adjust: VS Code says it does, and the client of + /// the `JetBrains` IDEs says nothing about it and adjusts + pub(crate) const fn completion_adjusts_indentation(self) -> bool { + self.supports_completion_item_adjust_indentation() + || !self.contains(Self::COMPLETION_INSERTS_AS_IS) + } + /// Returns `true` if the client prefers Markdown over plain text in completion items. pub(crate) const fn prefers_markdown_in_completion(self) -> bool { self.contains(Self::PREFER_MARKDOWN_IN_COMPLETION) @@ -529,6 +548,24 @@ impl ResolvedClientCapabilities { flags |= Self::COMPLETION_ITEM_SNIPPET_SUPPORT; } + let completion = text_document.and_then(|text_document| text_document.completion.as_ref()); + if completion + .and_then(|completion| completion.completion_item.as_ref()) + .and_then(|completion_item| completion_item.insert_text_mode_support.as_ref()) + .is_some_and(|support| { + support + .value_set + .contains(&lsp_types::InsertTextMode::AdjustIndentation) + }) + { + flags |= Self::COMPLETION_ITEM_ADJUST_INDENTATION; + } + if completion.and_then(|completion| completion.insert_text_mode) + == Some(lsp_types::InsertTextMode::AsIs) + { + flags |= Self::COMPLETION_INSERTS_AS_IS; + } + let folding_range = text_document.and_then(|text_document| text_document.folding_range.as_ref()); if folding_range diff --git a/crates/ty_server/src/server/api/requests/completion.rs b/crates/ty_server/src/server/api/requests/completion.rs index da41a8c4ba..f8ae6f8671 100644 --- a/crates/ty_server/src/server/api/requests/completion.rs +++ b/crates/ty_server/src/server/api/requests/completion.rs @@ -81,8 +81,7 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { db, &env, snapshot.workspace_settings().completions(), - CompletionCapabilities::default() - .snippets(client_capabilities.supports_completion_item_snippets()), + completion_capabilities(client_capabilities), program_file, offset, ); @@ -152,6 +151,12 @@ pub(super) fn completion_list<'db>( .insert .as_ref() .map_or_else(|| label.clone(), ToString::to_string); + // a text over several lines is written for a client that indents its new lines + // as the line it is accepted on; one that can be told to is + let insert_text_mode = (client_capabilities + .supports_completion_item_adjust_indentation() + && new_text.contains('\n')) + .then_some(lsp_types::InsertTextMode::AdjustIndentation); let import_suffix = comp .module_name .and_then(|name| import_edit.is_some().then(|| format!(" (import {name})"))); @@ -210,6 +215,7 @@ pub(super) fn completion_list<'db>( CompletionItem { label, kind, + insert_text_mode, sort_text: Some(format!("{i:-max_index_len$}")), detail: type_display, label_details, @@ -236,6 +242,16 @@ pub(super) fn completion_list<'db>( } } +/// what a completion's text can be written for: snippets, and how the client indents the new +/// lines of a text over several +pub(super) fn completion_capabilities( + client_capabilities: ResolvedClientCapabilities, +) -> CompletionCapabilities { + CompletionCapabilities::default() + .snippets(client_capabilities.supports_completion_item_snippets()) + .adjusts_indentation(client_capabilities.completion_adjusts_indentation()) +} + impl RetriableRequestHandler for CompletionRequestHandler { const RETRY_ON_CANCELLATION: bool = true; } diff --git a/crates/ty_server/src/server/api/requests/smart_completion.rs b/crates/ty_server/src/server/api/requests/smart_completion.rs index ad22cba35a..0c93f04d98 100644 --- a/crates/ty_server/src/server/api/requests/smart_completion.rs +++ b/crates/ty_server/src/server/api/requests/smart_completion.rs @@ -11,12 +11,12 @@ use std::borrow::Cow; use lsp_types::{ CompletionList, CompletionParams, LspRequestMethod, MessageDirection, Request, Uri, }; -use ty_ide::{CompletionCapabilities, smart_completion}; +use ty_ide::smart_completion; use ty_project::{ProjectDatabase, SemanticDb as _}; use ty_python_semantic::ProgramEnvironment; use crate::document::PositionExt; -use crate::server::api::requests::completion::completion_list; +use crate::server::api::requests::completion::{completion_capabilities, completion_list}; use crate::server::api::traits::{ BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, }; @@ -76,11 +76,7 @@ impl BackgroundDocumentRequestHandler for SmartCompletionRequestHandler { db, &env, snapshot.workspace_settings().completions(), - CompletionCapabilities::default().snippets( - snapshot - .resolved_client_capabilities() - .supports_completion_item_snippets(), - ), + completion_capabilities(snapshot.resolved_client_capabilities()), program_file, offset, ) else { diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 18d2302afc..df33156156 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -1760,3 +1760,77 @@ x = assert_eq!(ask(&mut server, Position::new(8, 4)), None); Ok(()) } + +/// the text a postfix `for` written at `x.` in `if x:` writes, and the mode it asks the client to +/// insert it in, for a client with `capabilities` +fn postfix_for( + capabilities: lsp_types::CompletionClientCapabilities, +) -> Result<(String, Option)> { + let workspace_root = SystemPath::new("src"); + let main = SystemPath::new("src/main.by"); + let content = "def f(x: list[int]):\n if x:\n x.\n"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default().with_auto_import(false)) + .with_workspace(workspace_root, None)? + .with_file(main, content)? + .with_completion_capabilities(capabilities) + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(main, content, 1); + + let completions = server.completion_request(&server.file_uri(main), Position::new(2, 10)); + let item = completions + .into_iter() + .find(|item| item.label == "for") + .expect("a postfix `for`"); + let text = match item.text_edit { + Some(lsp_types::CompletionItemTextEdit::TextEdit(edit)) => edit.new_text, + other => panic!("expected a text edit, got {other:?}"), + }; + Ok((text, item.insert_text_mode)) +} + +/// a statement a postfix template writes over several lines is indented for the way the client +/// inserts it: one level past the line it is accepted on for a client that indents each new line +/// as that line, which a client is taken to do unless it says it inserts the text as it is +#[test] +fn postfix_statement_is_indented_for_how_the_client_inserts_it() -> Result<()> { + // says nothing, as the JetBrains IDEs' client does, which indents each new line as the line + assert_eq!( + postfix_for(lsp_types::CompletionClientCapabilities::default())?, + ("for item in x:\n ".to_string(), None) + ); + + // inserts every completion as it is + assert_eq!( + postfix_for(lsp_types::CompletionClientCapabilities { + insert_text_mode: Some(lsp_types::InsertTextMode::AsIs), + ..Default::default() + })?, + ("for item in x:\n ".to_string(), None) + ); + + // inserts as it is unless an item asks otherwise, and this one does + assert_eq!( + postfix_for(lsp_types::CompletionClientCapabilities { + insert_text_mode: Some(lsp_types::InsertTextMode::AsIs), + completion_item: Some(lsp_types::ClientCompletionItemOptions { + insert_text_mode_support: Some( + lsp_types::ClientCompletionItemInsertTextModeOptions::new(vec![ + lsp_types::InsertTextMode::AsIs, + lsp_types::InsertTextMode::AdjustIndentation, + ]) + ), + ..Default::default() + }), + ..Default::default() + })?, + ( + "for item in x:\n ".to_string(), + Some(lsp_types::InsertTextMode::AdjustIndentation) + ) + ); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index cd2fe7bd3f..1fb27d4d6f 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -1627,6 +1627,18 @@ impl TestServerBuilder { } /// Enable or disable the completion snippet capability. + /// Set what the client says of completions, in place of anything said before. + pub(crate) fn with_completion_capabilities( + mut self, + completion: lsp_types::CompletionClientCapabilities, + ) -> Self { + self.client_capabilities + .text_document + .get_or_insert_default() + .completion = Some(completion); + self + } + pub(crate) fn enable_completion_snippets(mut self, enabled: bool) -> Self { self.client_capabilities .text_document diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 8763f1ddbc..1d1e50a811 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -44,6 +44,14 @@ editor supports snippets each is a completion labelled with its name, `for`, and the detail `postfix`, whose `textEdit` replaces the whole `x.for` and whose `filterText` is that text +a statement that opens a suite continues on a line indented one level past the +line it is written on. an editor indents each new line of a completion as the +line it is accepted on, so the text says only the one level; an editor whose +client says it inserts a completion's text as it is +(`completion.insertTextMode: asIs`) gets the line's own indentation written in +too, and one that can be told per completion is told +(`insertTextMode: adjustIndentation`) + `.let` and `.var` spell basedpython bindings; the rest are valid python and are offered in `.py` files too From dc3873937f0d874f6b06a97aa33bd4f6d5473800 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:44:59 +1000 Subject: [PATCH 088/110] mark each by/injections fragment checked that invalid-regex compiles as re would, an re pattern whose text and flags the checker reads, whatever decided its language and only while the rule is on, so an editor can leave regular expression errors to by there, with the verbose-flag reading the checker does shared with the question --- crates/ty_ide/src/injection.rs | 100 ++++++++++++++-- crates/ty_ide/src/refactor.rs | 16 +-- crates/ty_ide/src/refactor/evaluation.rs | 18 +-- .../ty_ide/src/refactor/extract_variable.rs | 46 ++++---- .../src/types/ide_support.rs | 111 ++++++++++++++++-- .../src/types/infer/builder.rs | 24 +--- crates/ty_python_semantic/src/types/regex.rs | 27 ++++- .../src/server/api/requests/code_action.rs | 6 +- .../src/server/api/requests/injections.rs | 9 +- crates/ty_server/tests/e2e/refactors.rs | 4 +- .../features/language-injection.md | 9 ++ 11 files changed, 288 insertions(+), 82 deletions(-) diff --git a/crates/ty_ide/src/injection.rs b/crates/ty_ide/src/injection.rs index 1d2877fc67..b898e2e10b 100644 --- a/crates/ty_ide/src/injection.rs +++ b/crates/ty_ide/src/injection.rs @@ -57,11 +57,13 @@ use ruff_python_ast::visitor::source_order::{ }; use ruff_python_ast::{self as ast, AnyNodeRef}; use ruff_text_size::{Ranged, TextRange}; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use ty_python_core::ProgramFile; use ty_python_core::definition::{Definition, DefinitionKind}; use ty_python_semantic::SemanticModel; -use ty_python_semantic::types::ide_support::{call_signature_details, is_re_pattern_parameter}; +use ty_python_semantic::types::ide_support::{ + call_signature_details, invalid_regex_checks_pattern, is_re_pattern_parameter, +}; use crate::Db; @@ -104,6 +106,12 @@ pub struct Injection { /// What decided the language, which is what an editor shows a user who asks /// why a string is being treated as another language. pub origin: InjectionOrigin, + + /// whether `by` checks the fragment itself: the pattern of a call to one of + /// python's `re` functions that `invalid-regex` compiles as `re` would. an + /// editor whose own support for the language reports errors too can leave + /// them to `by` here, where reporting them as well would say each twice + pub checked: bool, } /// What decided that a fragment is in a particular language. @@ -175,10 +183,16 @@ pub fn injections<'db>(db: &'db dyn Db, file: ProgramFile<'db>) -> Injections { markers: &markers, expectations: Expectations::default(), found: Vec::new(), + checked: FxHashSet::default(), }; walk_body(&mut finder, module.suite()); let mut found = finder.found; + // a string is checked however its language was decided: a marker over an `re` call says + // what the pattern already is + for injection in &mut found { + injection.checked = finder.checked.contains(&injection.ranges); + } // One string can be reached both ways — a marked statement whose string is // also a call argument. Sorting by origin puts the marker the reader wrote // closest to the string first, and the duplicate behind it is dropped. @@ -234,6 +248,9 @@ struct InjectionFinder<'a, 'db> { markers: &'a [(TextRange, String)], expectations: Expectations<'db>, found: Vec, + /// the text ranges of the string literals `invalid-regex` checks, as + /// [`Injection::ranges`] gives them + checked: FxHashSet>, } impl InjectionFinder<'_, '_> { @@ -305,6 +322,7 @@ impl InjectionFinder<'_, '_> { language: language.to_string(), ranges: content_ranges(string, self.source, self.dedents), origin: InjectionOrigin::Comment, + checked: false, }); } } @@ -322,6 +340,7 @@ impl InjectionFinder<'_, '_> { language, ranges: content_ranges(string, self.source, self.dedents), origin, + checked: false, }); } } @@ -385,6 +404,13 @@ impl<'db> SourceOrderVisitor<'db> for InjectionFinder<'_, 'db> { fn visit_expr(&mut self, expr: &'db ast::Expr) { if let ast::Expr::Call(call) = expr { self.mark_call_arguments(call); + if let Some(ast::Expr::StringLiteral(pattern)) = + call.arguments.find_argument_value("pattern", 0) + && invalid_regex_checks_pattern(self.model, call) + { + self.checked + .insert(content_ranges(pattern, self.source, self.dedents)); + } } walk_expr(self, expr); } @@ -694,9 +720,14 @@ mod tests { DiagnosticId::Lint(LintName::of("injection")), Severity::Info, format!( - "{} ({})", + "{} ({}{})", self.injection.language, - self.injection.origin.as_str() + self.injection.origin.as_str(), + if self.injection.checked { + ", checked" + } else { + "" + } ), ); let first = ranges.next().expect("an injection to cover something"); @@ -1151,19 +1182,19 @@ re.split(\",\", \"a,b\") ", ); assert_snapshot!(test.injections(), @r#" - info[injection]: PythonRegExp (declared) + info[injection]: PythonRegExp (declared, checked) --> main.by:4:20 | 4 | word = re.compile("(?P\\w+)") | ^^^^^^^^^^^^^^ - info[injection]: PythonRegExp (declared) + info[injection]: PythonRegExp (declared, checked) --> main.by:5:10 | 5 | replace("a+", "b", "aaa") | ^^ - info[injection]: PythonRegExp (declared) + info[injection]: PythonRegExp (declared, checked) --> main.by:6:11 | 6 | re.split(",", "a,b") @@ -1171,6 +1202,61 @@ re.split(\",\", \"a,b\") "#); } + /// `invalid-regex` checks a pattern whose text and flags it can read, however the language + /// was decided, and leaves the rest to the editor: one whose flags it cannot read, and one + /// handed on through a parameter, whose text it does not know at the `re` call + #[test] + fn a_re_pattern_by_checks_is_marked_checked() { + let test = injection_test( + "\ +import re + +def flags() -> int: ... + +re.compile(\"a+\", re.IGNORECASE | re.VERBOSE) +re.compile(\"a+\", flags()) +# language=RegExp +re.match(\"b+\", \"bbb\") + +def find(pattern: str, text: str): + return re.search(pattern, text) + +find(\"[0-9]+\", \"abc 123\") +", + ); + assert_snapshot!(test.injections(), @r#" + info[injection]: PythonRegExp (declared, checked) + --> main.by:5:13 + | + 5 | re.compile("a+", re.IGNORECASE | re.VERBOSE) + | ^^ + + info[injection]: PythonRegExp (declared) + --> main.by:6:13 + | + 6 | re.compile("a+", flags()) + | ^^ + + info[injection]: RegExp (comment, checked) + --> main.by:8:11 + | + 8 | re.match("b+", "bbb") + | ^^ + + info[injection]: RegExp (comment) + --> main.by:8:17 + | + 8 | re.match("b+", "bbb") + | ^^^ + + info[injection]: PythonRegExp (propagated) + --> main.by:13:7 + | + 13 | find("[0-9]+", "abc 123") + | ^^^^^^ + "#); + } + #[test] fn a_re_pattern_travels_through_a_parameter_passed_straight_on() { let test = injection_test( diff --git a/crates/ty_ide/src/refactor.rs b/crates/ty_ide/src/refactor.rs index 6e93352cb3..5442819e5a 100644 --- a/crates/ty_ide/src/refactor.rs +++ b/crates/ty_ide/src/refactor.rs @@ -145,8 +145,8 @@ impl RefactorKind { pub struct RefactorOffer { pub kind: RefactorKind, pub title: String, - /// The range to ask about to have this rewrite again: the range the offer was asked for, - /// or, for one of the expressions offered around a caret, that expression. + /// the range to ask about to have this rewrite again: the range the offer was asked for, + /// or, for one of the expressions offered around a caret, that expression pub range: TextRange, /// `Ok` holds the rewrite and `Err` why there is none. /// @@ -318,11 +318,11 @@ impl<'db> RefactorContext<'db> { /// out, so a caller that will not offer a refactoring should not name it — a client narrowing a /// `textDocument/codeAction` with `only` then pays for nothing it asked no question about. /// -/// With `around_caret`, an empty `range` is answered for extracting a variable or a constant +/// with `around_caret`, an empty `range` is answered for extracting a variable or a constant /// with one offer for each expression around the caret, innermost first, each carrying the /// expression's range: what an editor offers to choose from when extract is asked for with -/// nothing selected. Without it a caret extracts nothing, which is what a client asking while -/// the caret moves wants, rather than an offer for every expression the caret is in. +/// nothing selected. without it a caret extracts nothing, which is what a client asking while +/// the caret moves wants, rather than an offer for every expression the caret is in pub fn refactors( db: &dyn Db, file: ProgramFile<'_>, @@ -467,9 +467,9 @@ pub(crate) mod test_support { Self { db, file, range } } - /// What `kind` offers around the test's caret, one line for each offer: the text it - /// is about, its title, and why it is refused when it is. Each offer that applies is - /// checked to be the rewrite asking about its own range gives. + /// what `kind` offers around the test's caret, one line for each offer: the text it + /// is about, its title, and why it is refused when it is. each offer that applies is + /// checked to be the rewrite asking about its own range gives pub(crate) fn around_caret(&self, kind: RefactorKind) -> String { let program_file = self.db.program_file(self.file); let source = ruff_db::source::source_text(&self.db, self.file); diff --git a/crates/ty_ide/src/refactor/evaluation.rs b/crates/ty_ide/src/refactor/evaluation.rs index a5594f222a..cf3a2f7d0d 100644 --- a/crates/ty_ide/src/refactor/evaluation.rs +++ b/crates/ty_ide/src/refactor/evaluation.rs @@ -104,14 +104,14 @@ pub(crate) fn placement_in_statement( .unwrap_or(Placement::NotOnce(NotOnce::Opaque)) } -/// Whether evaluating `expr` has no effect at all: it runs no code of the program's, +/// whether evaluating `expr` has no effect at all: it runs no code of the program's, /// binds nothing and raises nothing, so evaluating something else before it or after it -/// cannot tell the difference. +/// cannot tell the difference /// -/// Wider than [`is_pure`], because it asks the checker what the operands are: `a + b` +/// wider than [`is_pure`], because it asks the checker what the operands are: `a + b` /// over two `int`s is python's own addition, and `math.pi` looks a name up in a /// module, where the same spelling over values of a class the program wrote could -/// run any `__add__` or property. +/// run any `__add__` or property pub(crate) fn is_effect_free(model: &SemanticModel<'_>, expr: &Expr) -> bool { if is_pure(expr) { return true; @@ -181,10 +181,10 @@ pub(crate) fn is_stable_value(model: &SemanticModel<'_>, expr: &Expr) -> bool { !attributes.0 } -/// Whether the operation `expr` performs on its operands — not the evaluation of the -/// operands themselves — runs no code of the program's and cannot raise. +/// whether the operation `expr` performs on its operands — not the evaluation of the +/// operands themselves — runs no code of the program's and cannot raise /// -/// That is an operator of python's own over builtin scalars that is defined for every +/// that is an operator of python's own over builtin scalars that is defined for every /// pair of values it is given: `+`, `-` and `*` over numbers that python does not convert /// to compute them, `+` joining two strings or two byte strings, the bitwise operators /// over integers, equality, and ordering among numbers that are not complex or among @@ -341,8 +341,8 @@ impl Walk<'_, '_> { self.effects = true; } - /// The operation `expr` performs once its operands are evaluated, which is an - /// effect unless the checker shows it to be one of python's own quiet ones. + /// the operation `expr` performs once its operands are evaluated, which is an + /// effect unless the checker shows it to be one of python's own quiet ones fn operation(&mut self, expr: &Expr) { if !operation_is_quiet(self.model, expr) { self.effect(); diff --git a/crates/ty_ide/src/refactor/extract_variable.rs b/crates/ty_ide/src/refactor/extract_variable.rs index f98fcefdf1..88bc3be6cb 100644 --- a/crates/ty_ide/src/refactor/extract_variable.rs +++ b/crates/ty_ide/src/refactor/extract_variable.rs @@ -52,11 +52,11 @@ pub(super) fn plan( plan_selection(context, &selection, target, Title::Plain) } -/// What extracting does at a caret rather than a selection: each expression of the statement +/// what extracting does at a caret rather than a selection: each expression of the statement /// around `offset`, innermost first, with the range a client selects to extract it and the /// rewrite — the expressions `by/expressionTypes` answers at the same caret, less a bare name -/// and those that cannot be bound to a name. When none of them can be extracted, the innermost one's refusal is -/// the answer, so an editor asking about a caret is told why rather than nothing. +/// and those that cannot be bound to a name. when none of them can be extracted, the innermost one's refusal is +/// the answer, so an editor asking about a caret is told why rather than nothing pub(super) fn around_caret( context: &RefactorContext<'_>, offset: TextSize, @@ -99,8 +99,8 @@ pub(super) fn around_caret( available } -/// Whether a title names the expression it extracts, which it must when several are offered at -/// once. +/// whether a title names the expression it extracts, which it must when several are offered at +/// once #[derive(Clone, Copy, PartialEq, Eq)] enum Title { Plain, @@ -390,7 +390,7 @@ fn parameter_insertion(parameters: &ast::Parameters, name: &str, value: &str) -> } } -/// `text` on one line and at most a few dozen characters, for a title to name it by. +/// `text` on one line and at most a few dozen characters, for a title to name it by fn abbreviated(text: &str) -> String { const MOST: usize = 40; let one_line = text.split_whitespace().collect::>().join(" "); @@ -402,12 +402,12 @@ fn abbreviated(text: &str) -> String { shortened } -/// What a selection extracts. +/// what a selection extracts enum Extracted<'a> { - /// An expression of the tree. + /// an expression of the tree Expr(&'a Expr), - /// Consecutive operands of a chain of one associative operator, `b + c` in `a + b + c`, - /// which python groups as `(a + b) + c` and so has no node of its own. + /// consecutive operands of a chain of one associative operator, `b + c` in `a + b + c`, + /// which python groups as `(a + b) + c` and so has no node of its own Run { operands: Vec<&'a Expr> }, } @@ -419,7 +419,7 @@ struct Selection<'a> { } impl<'a> Selection<'a> { - /// The expressions the selection is made of, evaluated in this order. + /// the expressions the selection is made of, evaluated in this order fn operands(&self) -> Vec<&'a Expr> { match &self.extracted { Extracted::Expr(expr) => vec![*expr], @@ -427,7 +427,7 @@ impl<'a> Selection<'a> { } } - /// The first expression of the selection to be evaluated. + /// the first expression of the selection to be evaluated fn first(&self) -> &'a Expr { match &self.extracted { Extracted::Expr(expr) => expr, @@ -435,7 +435,7 @@ impl<'a> Selection<'a> { } } - /// A name that says what the selection evaluates to. + /// a name that says what the selection evaluates to fn suggested_name(&self, python_version_minor: u8) -> String { match &self.extracted { Extracted::Expr(expr) => suggest_for_expression(expr, python_version_minor), @@ -446,7 +446,7 @@ impl<'a> Selection<'a> { /// The expression the selection covers exactly, ignoring surrounding /// whitespace and allowing parentheses around it — or, when the selection -/// covers consecutive operands of an associative chain, those operands. +/// covers consecutive operands of an associative chain, those operands fn selected_expression<'a>( context: &'a RefactorContext<'_>, range: TextRange, @@ -517,14 +517,14 @@ fn selected_expression<'a>( selected_run(context, trimmed) } -/// Consecutive operands of an associative chain that `range` covers exactly, from the first -/// one's opening parenthesis, if it has one, to the last one's closing parenthesis. +/// consecutive operands of an associative chain that `range` covers exactly, from the first +/// one's opening parenthesis, if it has one, to the last one's closing parenthesis /// -/// `a + b + c` is `(a + b) + c`, so `b + c` is no expression of it. Binding `b + c` to a name +/// `a + b + c` is `(a + b) + c`, so `b + c` is no expression of it. binding `b + c` to a name /// and reading `a + value` instead evaluates `a + (b + c)`, which is the same value exactly when /// the operator is associative over the operands' types — which the checker is asked, not the /// spelling: `+` over integers or over strings is, `+` over floats rounds differently grouped -/// the other way, and an operator a class defines is anything it likes. +/// the other way, and an operator a class defines is anything it likes fn selected_run<'a>(context: &'a RefactorContext<'_>, range: TextRange) -> Option> { let covering = covering_node(context.parsed.syntax().into(), range); let chain = covering.ancestors().find_map(|node| match node { @@ -574,9 +574,9 @@ fn selected_run<'a>(context: &'a RefactorContext<'_>, range: TextRange) -> Optio }) } -/// Whether `op` is associative over the values `operands` evaluate to, as the checker reads +/// whether `op` is associative over the values `operands` evaluate to, as the checker reads /// them: integers under `+`, `*` and the bitwise operators, and strings or byte strings — all of -/// one kind — under `+`. +/// one kind — under `+` fn is_associative(context: &RefactorContext<'_>, op: ast::Operator, operands: &[&Expr]) -> bool { let Some(kinds) = operands .iter() @@ -623,8 +623,8 @@ fn is_extractable(expr: &Expr) -> bool { } } -/// The source of the selection as the value of an assignment, parenthesized where its text -/// would not parse there on its own. +/// the source of the selection as the value of an assignment, parenthesized where its text +/// would not parse there on its own fn expression_source(context: &RefactorContext<'_>, selection: &Selection<'_>) -> String { let Extracted::Expr(expr) = selection.extracted else { let text = &context.source()[selection.replaced]; @@ -887,7 +887,7 @@ mod tests { } /// `a + b + c` is `(a + b) + c`; extracting `b + c` evaluates `a + (b + c)`, which is - /// the same value because adding integers is associative. + /// the same value because adding integers is associative #[test] fn operands_of_an_associative_chain() { assert_snapshot!(RefactorTest::basedpython( diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 7187729d46..56bce18924 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -4,6 +4,7 @@ use std::sync::LazyLock; use crate::FxIndexSet; use std::ops::ControlFlow; +use crate::lint::LintId; use crate::place::{ ConsideredDefinitions, Place, RequiresExplicitReExport, implicit_builtins_symbol_scope, imported_symbol, place_by_id, place_from_bindings, @@ -3096,8 +3097,8 @@ pub fn string_literal_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> Opti } } -/// One of the builtin types whose operators are python's own: none of them runs code of the -/// program's. +/// one of the builtin types whose operators are python's own: none of them runs code of the +/// program's #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BuiltinScalar { Bool, @@ -3108,15 +3109,15 @@ pub enum BuiltinScalar { Bytes, } -/// The builtin scalar types `expr` evaluates to, one for each kind a union of them holds, or -/// `None` when it can evaluate to anything else. +/// the builtin scalar types `expr` evaluates to, one for each kind a union of them holds, or +/// `None` when it can evaluate to anything else /// -/// A literal is of its own type exactly, and so is an instance of `final int`, or of a class no -/// class can subclass, as `bool`. A plain `int` may be an instance of a subclass, whose operators -/// are the program's own, so it is not one. An operator python applies to values of exactly +/// a literal is of its own type exactly, and so is an instance of `final int`, or of a class no +/// class can subclass, as `bool`. a plain `int` may be an instance of a subclass, whose operators +/// are the program's own, so it is not one. an operator python applies to values of exactly /// builtin scalar types is python's own, and makes a value of exactly the builtin type it is /// declared to give, as `a + b` of two `final int`s is exactly an `int`, and `not x` is always -/// exactly a `bool`. +/// exactly a `bool` pub fn builtin_scalar_kinds( model: &SemanticModel<'_>, expr: &ast::Expr, @@ -3232,8 +3233,65 @@ pub fn is_re_pattern_parameter<'db>( == Some(ty_module_resolver::KnownModule::Re) } -/// Whether `expr` evaluates to a module, so that reading an attribute of it looks the name up -/// in the module's namespace and runs nothing. +/// whether `invalid-regex` checks the pattern `call` hands one of python's `re` functions, which +/// it does where the rule is on for the file and not suppressed for the pattern, the pattern's +/// text is a literal the checker reads, and the call's flags say whether it is verbose: it then +/// compiles the pattern as `re` would, and reports what `re` would refuse +/// +/// an editor that injects its own regular expression support into the pattern can leave the +/// errors to `by` there, and report its own anywhere else +pub fn invalid_regex_checks_pattern(model: &SemanticModel<'_>, call: &ast::ExprCall) -> bool { + let db = model.db(); + if !db + .rule_selection(model.file()) + .is_enabled(LintId::of(&crate::types::diagnostic::INVALID_REGEX)) + { + return false; + } + let Some(Type::FunctionLiteral(function)) = call.func.inferred_type(model) else { + return false; + }; + if crate::types::regex::RegexCall::from_name(function.name(db).as_str()).is_none() + || ty_module_resolver::file_to_module(db, function.program_file(db).resolver_file(db)) + .and_then(|module| module.known(db)) + != Some(ty_module_resolver::KnownModule::Re) + { + return false; + } + let Some(pattern) = call.arguments.find_argument_value("pattern", 0) else { + return false; + }; + // a suppression anywhere in the pattern hides what the rule reports there + if crate::suppression::suppressions(db, model.python_file()) + .find_suppression( + pattern.range(), + LintId::of(&crate::types::diagnostic::INVALID_REGEX), + ) + .is_some() + { + return false; + } + let Some(pattern_ty) = pattern.inferred_type(model) else { + return false; + }; + let env = model.program_environment(); + if crate::types::regex::pattern_source(db, &env, pattern_ty).is_none() { + return false; + } + let flags = function + .signature(db) + .overloads + .iter() + .find_map(|signature| signature.parameters().keyword_by_name("flags")) + .and_then(|(index, _)| call.arguments.find_argument_value("flags", index)); + crate::types::regex::verbose_flag(db, flags, &mut |operand| { + operand.inferred_type(model).unwrap_or(Type::unknown()) + }) + .is_some() +} + +/// whether `expr` evaluates to a module, so that reading an attribute of it looks the name up +/// in the module's namespace and runs nothing pub fn is_module_value(model: &SemanticModel<'_>, expr: &ast::Expr) -> bool { matches!(expr.inferred_type(model), Some(Type::ModuleLiteral(_))) } @@ -5745,6 +5803,39 @@ def f( Ok(()) } + #[test] + fn invalid_regex_leaves_a_pattern_whose_report_is_suppressed_unchecked() -> anyhow::Result<()> { + let db = TestDbBuilder::new() + .with_file( + "/src/foo.py", + r#" +import re + +re.compile("a(") +re.compile("a(") # ty: ignore[invalid-regex] +re.compile("a(") # ty: ignore +re.compile("a(") # ty: ignore[invalid-argument-type] +re.compile( + "a(" # ty: ignore[invalid-regex] +) +"#, + ) + .build()?; + let file = system_path_to_file(&db, "/src/foo.py")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + let parsed = parsed_module(&db, file.python_file(&db)).load(&db); + let model = SemanticModel::new(&db, file); + let checked: Vec = parsed + .suite() + .iter() + .filter_map(ast::Stmt::as_expr_stmt) + .filter_map(|statement| statement.value.as_call_expr()) + .map(|call| super::invalid_regex_checks_pattern(&model, call)) + .collect(); + assert_eq!(checked, [true, false, false, true, false]); + Ok(()) + } + #[test] fn builtin_definition_lookup_does_not_infer_scope() -> anyhow::Result<()> { let mut db = TestDbBuilder::new() diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 0dfbc61bc7..a909006a46 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -12625,27 +12625,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// verbose mode changes how the pattern itself tokenizes, so a guess here /// would produce confidently wrong group types. fn regex_verbose_flag(&self, flags: Option<&ast::Expr>) -> Option { - let Some(flags) = flags else { - return Some(false); - }; - // flags are combined with `|`, and the combination is verbose if any - // operand is. any other operator leaves us unable to say - if let ast::Expr::BinOp(binop) = flags { - if binop.op != ast::Operator::BitOr { - return None; - } - let left = self.regex_verbose_flag(Some(&binop.left))?; - let right = self.regex_verbose_flag(Some(&binop.right))?; - return Some(left || right); - } // the argument may not have been inferred yet — the repl callable of // `re.sub` is refined before the later arguments are visited — so fall // back to a speculative builder when there is no stored type - let ty = self.try_expression_type(flags).unwrap_or_else(|| { - self.speculate_without_diagnostics() - .infer_expression(flags, TypeContext::default()) - }); - regex::flag_is_verbose(self.db(), ty) + regex::verbose_flag(self.db(), flags, &mut |operand| { + self.try_expression_type(operand).unwrap_or_else(|| { + self.speculate_without_diagnostics() + .infer_expression(operand, TypeContext::default()) + }) + }) } fn infer_call_expression_impl( diff --git a/crates/ty_python_semantic/src/types/regex.rs b/crates/ty_python_semantic/src/types/regex.rs index edd2b2ddf6..2cea6f5bda 100644 --- a/crates/ty_python_semantic/src/types/regex.rs +++ b/crates/ty_python_semantic/src/types/regex.rs @@ -16,6 +16,7 @@ mod source; use std::fmt; +use ruff_python_ast as ast; use ruff_python_ast::name::Name; use ty_module_resolver::{KnownModule, file_to_module}; @@ -389,8 +390,32 @@ const VERBOSE_FLAG_NAMES: [&str; 2] = ["X", "VERBOSE"]; /// `re.VERBOSE`'s bit, for a `flags` argument spelled as a plain integer const VERBOSE_FLAG_BIT: i64 = 64; +/// whether the `flags` argument of an `re` call turns on verbose mode, where we can tell, with +/// `type_of` reading an operand's type. no argument is no flag +/// +/// flags are combined with `|`, and the combination is verbose if any operand is. any other +/// operator leaves us unable to say +pub(crate) fn verbose_flag<'db>( + db: &'db dyn Db, + flags: Option<&ast::Expr>, + type_of: &mut dyn FnMut(&ast::Expr) -> Type<'db>, +) -> Option { + let Some(flags) = flags else { + return Some(false); + }; + if let ast::Expr::BinOp(binop) = flags { + if binop.op != ast::Operator::BitOr { + return None; + } + let left = verbose_flag(db, Some(&binop.left), type_of)?; + let right = verbose_flag(db, Some(&binop.right), type_of)?; + return Some(left || right); + } + flag_is_verbose(db, type_of(flags)) +} + /// whether one `flags` operand turns on verbose mode, where we can tell -pub(crate) fn flag_is_verbose<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { +fn flag_is_verbose<'db>(db: &'db dyn Db, ty: Type<'db>) -> Option { if let Some(literal) = ty.as_enum_literal() { let class = literal.enum_class(db); return is_regex_flag_class(db, class) diff --git a/crates/ty_server/src/server/api/requests/code_action.rs b/crates/ty_server/src/server/api/requests/code_action.rs index cc236f6e5c..db81bd2d93 100644 --- a/crates/ty_server/src/server/api/requests/code_action.rs +++ b/crates/ty_server/src/server/api/requests/code_action.rs @@ -286,9 +286,9 @@ pub(crate) fn to_lsp_edits( /// A client that can resolve an action's edit is sent the action without one, carrying what /// `codeAction/resolve` needs to spell it again; any other client is sent `edits` up front. That /// saves serialising a rewrite nobody chose, not working it out — the rewrite in hand is how this -/// refactoring was known to apply at all. An action about another range than the one asked +/// refactoring was known to apply at all. an action about another range than the one asked /// about — an extraction offered around a caret — carries its [`RefactorData`] either way, for -/// its `range` to tell a client which expression it extracts. +/// its `range` to tell a client which expression it extracts fn refactor_action( db: &ProjectDatabase, snapshot: &DocumentSnapshot, @@ -319,7 +319,7 @@ fn refactor_action( /// its edit: the refactoring, and the document and range it was offered for. /// /// `range` is the range the refactoring is about: the range the client asked about, or, for -/// an extraction offered around a caret, the expression it extracts. +/// an extraction offered around a caret, the expression it extracts #[derive(serde::Serialize, serde::Deserialize)] pub(crate) struct RefactorData { pub(crate) uri: Uri, diff --git a/crates/ty_server/src/server/api/requests/injections.rs b/crates/ty_server/src/server/api/requests/injections.rs index 38a65c6630..155177a347 100644 --- a/crates/ty_server/src/server/api/requests/injections.rs +++ b/crates/ty_server/src/server/api/requests/injections.rs @@ -68,6 +68,11 @@ pub(crate) struct InjectionFragment { /// A client shows this when a reader asks why a string is being treated as another language — /// `propagated` is the one whose reason is not visible at the string itself. origin: String, + + /// whether `by` checks the fragment itself and reports its errors: an `re` pattern that + /// `invalid-regex` compiles as `re` would. a client whose own support for the language reports + /// errors as well leaves them out here, where they would be said twice + checked: bool, } pub(crate) struct InjectionsRequestHandler; @@ -121,6 +126,7 @@ impl BackgroundDocumentRequestHandler for InjectionsRequestHandler { language: injection.language.clone(), ranges, origin: injection.origin.as_str().to_string(), + checked: injection.checked, }) }) .collect(); @@ -163,12 +169,13 @@ mod tests { ), ], origin: "comment".to_string(), + checked: false, }], }; assert_eq!( serde_json::to_string(&response).expect("a response to serialise"), - r#"{"injections":[{"language":"sql","ranges":[{"start":{"line":1,"character":9},"end":{"line":1,"character":17}},{"start":{"line":1,"character":20},"end":{"line":1,"character":27}}],"origin":"comment"}]}"# + r#"{"injections":[{"language":"sql","ranges":[{"start":{"line":1,"character":9},"end":{"line":1,"character":17}},{"start":{"line":1,"character":20},"end":{"line":1,"character":27}}],"origin":"comment","checked":false}]}"# ); } } diff --git a/crates/ty_server/tests/e2e/refactors.rs b/crates/ty_server/tests/e2e/refactors.rs index d98ecd55f9..06e11155e0 100644 --- a/crates/ty_server/tests/e2e/refactors.rs +++ b/crates/ty_server/tests/e2e/refactors.rs @@ -198,9 +198,9 @@ fn resolve_after_an_edit_is_content_modified() -> Result<()> { Ok(()) } -/// Asked explicitly for extractions with nothing selected, the server offers one for each +/// asked explicitly for extractions with nothing selected, the server offers one for each /// expression around the caret, each naming the expression and carrying its range; asked as -/// the caret moves, it offers none. +/// the caret moves, it offers none #[test] fn extraction_at_a_caret_offers_the_expressions_around_it() -> Result<()> { let (mut server, file) = server(true)?; diff --git a/docs/basedpython/features/language-injection.md b/docs/basedpython/features/language-injection.md index 7108474a26..db665cebee 100644 --- a/docs/basedpython/features/language-injection.md +++ b/docs/basedpython/features/language-injection.md @@ -97,6 +97,15 @@ snippet = """ a fragment is checked on its own, so it sees the standard library but not the names around the string it sits in +a pattern handed to one of python's `re` functions is checked by basedpython +itself: its `invalid-regex` rule compiles it as `re` would, where the pattern is +a literal and the call's flags say whether it is verbose, and reports what `re` +would refuse. `by/injections` marks each such fragment +`"checked": true`, whatever decided its language, so an editor whose own regular +expression support reports errors too can leave them to basedpython there, and +report its own everywhere else. with `invalid-regex` turned off, nothing is +marked + the indentation a triple-quoted string [strips](dedent-strings.md) is stripped from the fragment too, which is what makes an indented block of basedpython or python a program rather than a syntax From e2f0713eda1a549a531297c7580c25ce5b7f95c6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:50:52 +1000 Subject: [PATCH 089/110] answer by/typeDeclaration for a class whose module's source only imports it from where it is implemented, as collections does deque and defaultdict from _collections, at its stub, and for one the source declares as well as imports, as OrderedDict, at the class it declares, rather than answering null because the import is not a declaration --- crates/ty_ide/src/type_name.rs | 41 ++++++++++++++++++++++++++--- docs/basedpython/features/editor.md | 4 ++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/ty_ide/src/type_name.rs b/crates/ty_ide/src/type_name.rs index 12e1b59447..83a6cd5001 100644 --- a/crates/ty_ide/src/type_name.rs +++ b/crates/ty_ide/src/type_name.rs @@ -13,7 +13,7 @@ use ty_python_semantic::SemanticModel; use ty_python_semantic::types::Type; use crate::Db; -use crate::declaration::{DeclarationAt, single_declaration}; +use crate::declaration::{DeclarationAt, DeclaredKind, declaration, single_declaration}; use crate::goto::Definitions; /// the class `type_name` names, as a module of the program `context` is in would import it, with @@ -31,9 +31,29 @@ pub fn type_declaration( if !matches!(ty, Type::ClassLiteral(_)) { return None; } - let definitions = Definitions::from_ty(db, &model.program_environment(), ty)?.map_stubs(db); - let declaration = single_declaration(db, &definitions)?; - let target = definitions.into_navigation_targets(db).into_iter().next()?; + let stub = Definitions::from_ty(db, &model.program_environment(), ty)?; + let source = Definitions::new(stub.iter().cloned().collect()).map_stubs(db); + // a module's source can import the class from where it is implemented rather than declare it, + // as `collections` does `deque` from `_collections`, and that import is not the class's + // declaration: its stub declares it then + class_declaration(db, &source).or_else(|| class_declaration(db, &stub)) +} + +/// the class `definitions` declare, and where: the definitions that declare a class, an import +/// of one left out, when there is exactly one class among them +fn class_declaration(db: &dyn Db, definitions: &Definitions<'_>) -> Option { + let classes = Definitions::new( + definitions + .iter() + .filter(|resolved| { + declaration(db, resolved) + .is_some_and(|declaration| declaration.kind == DeclaredKind::Class) + }) + .cloned() + .collect(), + ); + let declaration = single_declaration(db, &classes)?; + let target = classes.into_navigation_targets(db).into_iter().next()?; Some(DeclarationAt { declaration, target, @@ -106,6 +126,19 @@ def make() -> None: assert_snapshot!(test().type_declaration("builtins.list"), @"class builtins.list at stdlib/builtins.byi:2659"); } + /// a module whose source imports a class from where it is implemented, as `collections` does + /// `deque` from the C module `_collections`, does not declare it: its stub does + #[test] + fn a_class_whose_source_only_imports_it_is_its_stub() { + let test = CursorTest::builder() + .source("shelf.pyi", "class Stack: ...\n\nclass Queue: ...\n") + .source("shelf.py", "from _shelf import Stack\n\nclass Queue: ...\n") + .source("main.py", "") + .build(); + assert_snapshot!(test.type_declaration("shelf.Stack"), @"class shelf.Stack at /shelf.pyi:1"); + assert_snapshot!(test.type_declaration("shelf.Queue"), @"class shelf.Queue at /shelf.py:3"); + } + #[test] fn what_is_not_a_class_is_not_answered() { let test = test(); diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 1d1e50a811..68dd11573e 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1952,7 +1952,9 @@ as a module of the project's environment, and each part after it as a member of what came before, through submodules and re-exports, so `shapes.Circle`, where `shapes/__init__` re-exports it, answers `shapes.base.Circle`. a class with a source is answered at its source, and one with only a stub, as `builtins.list`, -at its stub. the answer is `null` for a name that reaches anything other than a +at its stub, as is one whose module's source only imports it from where it is +implemented, as `collections` does `deque` from the C module `_collections`. the +answer is `null` for a name that reaches anything other than a class, or nothing, as `make..Local` does. a session of several projects is answered from the first whose environment resolves it From afd1c7dee19dff89742c2a2a01a549970734a57b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:54:20 +1000 Subject: [PATCH 090/110] switch the postfix templates off with completions.postfixTemplates, on unless set to false, leaving .await, which is syntax, offered either way --- crates/ty_ide/src/completion.rs | 46 ++++++++++++++++--- crates/ty_server/src/capabilities.rs | 8 ++-- crates/ty_server/src/session/options.rs | 12 +++++ crates/ty_server/tests/e2e/completions.rs | 32 +++++++++++++ crates/ty_server/tests/e2e/main.rs | 2 +- .../e2e__commands__debug_command.snap | 1 + docs/basedpython/features/editor.md | 5 ++ 7 files changed, 95 insertions(+), 11 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index ebdc1ce1aa..58f4c3b652 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -166,6 +166,7 @@ pub fn completion<'db>( source_type, &source, capabilities, + settings.postfix_templates, &mut completions, ); if settings.auto_import { @@ -382,10 +383,10 @@ fn replacing(completions: Vec>, typed: TextRange) -> Vec CompletionSettings { CompletionSettings { auto_import: true, + postfix_templates: true, // basedpython defaults this on where upstream ty leaves it off. A // callable completion exists to be called, and typing the parentheses // back is work the editor already knows how to do — the machinery @@ -4045,6 +4049,7 @@ fn add_postfix_completions<'db>( source_type: PySourceType, source: &SourceText, capabilities: CompletionCapabilities, + templates: bool, completions: &mut Completions<'db>, ) { let env = &model.program_environment(); @@ -4068,8 +4073,9 @@ fn add_postfix_completions<'db>( ); } + // the templates can be switched off, while `.await` is syntax and stays let value = expr.value.range(); - if value.is_empty() { + if !templates || value.is_empty() { return; } let Some(text) = source.get(value.start().to_usize()..value.end().to_usize()) else { @@ -14515,9 +14521,32 @@ count = 3 ); } - /// A client that indents each new line as the line the completion is + /// the templates can be switched off; `.await`, which is syntax, stays + #[test] + fn basedpython_postfix_templates_switched_off() { + CursorTest::builder() + .source("main.by", "def f(x: list[int]):\n x.") + .completion_test_builder() + .skip_postfix_templates() + .build() + .not_contains("for") + .not_contains("print") + .contains("append"); + CursorTest::builder() + .source( + "main.by", + "from collections.abc import Coroutine\nasync def f(c: Coroutine[None, None, int]):\n c.", + ) + .completion_test_builder() + .skip_postfix_templates() + .build() + .contains("await") + .not_contains("print"); + } + + /// a client that indents each new line as the line the completion is /// accepted on gets the new line indented one level past that line, and - /// adds the statement's own indentation itself. + /// adds the statement's own indentation itself #[test] fn basedpython_postfix_statement_indentation_for_a_client_that_adjusts_it() { let builder = CursorTest::builder() @@ -17043,6 +17072,11 @@ raise self } + fn skip_postfix_templates(mut self) -> CompletionTestBuilder { + self.settings.postfix_templates = false; + self + } + /// When set, completions insert parentheses for callable items. fn complete_function_parentheses(mut self) -> CompletionTestBuilder { self.settings.complete_function_parentheses = true; diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index 0ebfcb0f5d..d852c5d29d 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -234,15 +234,15 @@ impl ResolvedClientCapabilities { self.contains(Self::COMPLETION_ITEM_SNIPPET_SUPPORT) } - /// Returns `true` if a completion item can ask the client to indent each new line of its - /// text as the line the item is accepted on (`insertTextMode: adjustIndentation`). + /// returns `true` if a completion item can ask the client to indent each new line of its + /// text as the line the item is accepted on (`insertTextMode: adjustIndentation`) pub(crate) const fn supports_completion_item_adjust_indentation(self) -> bool { self.contains(Self::COMPLETION_ITEM_ADJUST_INDENTATION) } - /// Returns `true` if the client indents each new line of a completion's text as the line the + /// returns `true` if the client indents each new line of a completion's text as the line the /// completion is accepted on: a completion that asks it to, where it can be asked, and - /// otherwise every one, unless the client says it inserts the text as it is. + /// otherwise every one, unless the client says it inserts the text as it is /// /// a client that says neither is taken to adjust: VS Code says it does, and the client of /// the `JetBrains` IDEs says nothing about it and adjusts diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index 19f05f84ae..c2d5137929 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -170,6 +170,15 @@ impl ClientOptions { self } + #[must_use] + pub fn with_postfix_templates(mut self, enabled: bool) -> Self { + self.workspace + .completions + .get_or_insert_default() + .postfix_templates = Some(enabled); + self + } + #[must_use] pub fn with_import_style(mut self, style: ImportStyleOption) -> Self { self.workspace @@ -478,6 +487,8 @@ impl InlayHintOptions { #[serde(rename_all = "camelCase")] pub struct CompletionOptions { auto_import: Option, + /// whether `.` after an expression offers the postfix templates that rewrite it + postfix_templates: Option, /// Whether callable completions should insert parentheses. complete_function_parentheses: Option, /// how an auto-import brings in a member of a module the file imports nothing from yet @@ -507,6 +518,7 @@ impl CompletionOptions { fn into_settings(self) -> CompletionSettings { CompletionSettings { auto_import: self.auto_import.unwrap_or(true), + postfix_templates: self.postfix_templates.unwrap_or(true), // basedpython defaults this on — see `CompletionSettings::default` complete_function_parentheses: self.complete_function_parentheses.unwrap_or(true), import_style: match self.import_style { diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index df33156156..8a10df354b 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -1834,3 +1834,35 @@ fn postfix_statement_is_indented_for_how_the_client_inserts_it() -> Result<()> { Ok(()) } + +/// `completions.postfixTemplates: false` leaves the postfix templates out +#[test] +fn postfix_templates_can_be_switched_off() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let main = SystemPath::new("src/main.by"); + let content = "def f(x: list[int]):\n x.\n"; + let labels = |enabled: bool| -> Result> { + let mut server = TestServerBuilder::new()? + .with_initialization_options( + &ClientOptions::default() + .with_auto_import(false) + .with_postfix_templates(enabled), + ) + .with_workspace(workspace_root, None)? + .with_file(main, content)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(main, content, 1); + Ok(server + .completion_request(&server.file_uri(main), Position::new(1, 6)) + .into_iter() + .map(|item| item.label) + .collect()) + }; + + assert!(labels(true)?.iter().any(|label| label == "for")); + let without = labels(false)?; + assert!(!without.iter().any(|label| label == "for"), "{without:?}"); + assert!(without.iter().any(|label| label == "append"), "{without:?}"); + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 1fb27d4d6f..4099831ade 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -1627,7 +1627,7 @@ impl TestServerBuilder { } /// Enable or disable the completion snippet capability. - /// Set what the client says of completions, in place of anything said before. + /// set what the client says of completions, in place of anything said before pub(crate) fn with_completion_capabilities( mut self, completion: lsp_types::CompletionClientCapabilities, diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index b50f2df72f..f87f0c98db 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -47,6 +47,7 @@ Settings: WorkspaceSettings { }, completions: CompletionSettings { auto_import: true, + postfix_templates: true, complete_function_parentheses: true, import_style: FromImport, }, diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 68dd11573e..be5dc5da39 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -55,6 +55,11 @@ too, and one that can be told per completion is told `.let` and `.var` spell basedpython bindings; the rest are valid python and are offered in `.py` files too +set `ty.completions.postfixTemplates` to `false` (or +`completions.postfixTemplates` in the initialization options, for a client that +does not answer `workspace/configuration` with it) to be offered none of them. +`.await` is not a template but syntax, and is offered either way + ### postfix `await` [`.await`](await-attribute.md) is not a template — it is real syntax, so it From bf43e02fb794a548f69a2bfd05b2a22c2509fe0c Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:52:40 +1000 Subject: [PATCH 091/110] parse a document as basedpython by its path's extension whatever the uri's scheme, and by the client's `basedpython` or `by` language id when the path has no extension that says, rather than by an extension read only off a file path, which left `untitled:console-1.by` reporting its `let` as not valid in .py files --- crates/ruff_db/src/files.rs | 16 ++++- crates/ruff_db/src/parsed.rs | 8 +-- crates/ty_ide/src/pasted_imports.rs | 7 +-- .../src/types/class/static_literal.rs | 2 +- .../ty_server/src/document/text_document.rs | 23 +++++-- crates/ty_server/src/session.rs | 2 +- crates/ty_server/src/system.rs | 10 ++- .../tests/e2e/publish_diagnostics.rs | 63 +++++++++++++++++++ docs/basedpython/features/editor.md | 9 +++ 9 files changed, 117 insertions(+), 23 deletions(-) diff --git a/crates/ruff_db/src/files.rs b/crates/ruff_db/src/files.rs index c643eb0e06..a06c2ff0f0 100644 --- a/crates/ruff_db/src/files.rs +++ b/crates/ruff_db/src/files.rs @@ -625,17 +625,29 @@ impl File { || path.ends_with("__init__.by") } + /// the kind of source the file holds, which is also the dialect it is parsed as: python or + /// basedpython, a module or a stub, or a notebook + /// + /// an extension that names one decides it, whatever the path's scheme: `untitled:console-1.by` + /// is basedpython as `app/main.by` is. a path with no such extension — an editor's + /// `untitled:Untitled-1`, a script with none — is what the system says it is, which for a + /// language server is the language the editor opened the document in, and python when the + /// system does not know either pub fn source_type(self, db: &dyn Db) -> PySourceType { match self.path(db) { FilePath::System(path) => path .extension() - .map_or(PySourceType::Python, PySourceType::from_extension), + .and_then(PySourceType::try_from_extension) + .or_else(|| db.system().source_type(path)) + .unwrap_or(PySourceType::Python), FilePath::Vendored(path) => path .extension() .map_or(PySourceType::Stub, PySourceType::from_extension), FilePath::SystemVirtual(path) => path .extension() - .map_or(PySourceType::Python, PySourceType::from_extension), + .and_then(PySourceType::try_from_extension) + .or_else(|| db.system().virtual_path_source_type(path)) + .unwrap_or(PySourceType::Python), } } } diff --git a/crates/ruff_db/src/parsed.rs b/crates/ruff_db/src/parsed.rs index 0dc0de5167..a79fb24144 100644 --- a/crates/ruff_db/src/parsed.rs +++ b/crates/ruff_db/src/parsed.rs @@ -47,12 +47,8 @@ pub(super) fn disable_lru(db: &mut dyn Db) { } fn parsed_module_impl(db: &dyn Db, file: File, target_version: PythonVersion) -> Parsed { - let ty = file.source_type(db); - let is_basedpython = matches!(file.path(db).extension(), Some("by" | "byi")); - - let options = ParseOptions::from(ty) - .with_target_version(target_version) - .with_basedpython(is_basedpython); + // the source type says which dialect the file is, basedpython syntax included + let options = ParseOptions::from(file.source_type(db)).with_target_version(target_version); let source = source_text(db, file); diff --git a/crates/ty_ide/src/pasted_imports.rs b/crates/ty_ide/src/pasted_imports.rs index 8e8f0ba028..6809aede95 100644 --- a/crates/ty_ide/src/pasted_imports.rs +++ b/crates/ty_ide/src/pasted_imports.rs @@ -50,7 +50,7 @@ pub struct CopiedImport { } impl CopiedImport { - /// How [`Importer::import`] is asked for this import. + /// How [`Importer::import`](ty_python_semantic::importer::Importer::import) is asked for this import. fn request(&self) -> ImportRequest<'_> { match &self.member { Some(member) if *member == self.name => { @@ -99,10 +99,7 @@ pub fn copied_imports( let importing_file = ImportingFile::File(file.file(db), file.resolver_environment(db)); let own_module = file_to_module(db, file.resolver_file(db)) .map(|module| module.name(db).as_str().to_string()); - let options = ParseOptions::from(file.file(db).source_type(db)).with_basedpython(matches!( - file.file(db).path(db).extension(), - Some("by" | "byi") - )); + let options = ParseOptions::from(file.file(db).source_type(db)); let mut seen = FxHashSet::default(); let mut imports = Vec::new(); diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index c0968c9a9b..b26e1198b8 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -948,7 +948,7 @@ impl<'db> StaticClassLiteral<'db> { // basedpython classes can gain injected bases (synthetic enum/protocol // markers, enum variants) even with no explicit bases in the header, so // the fast path only applies to standard python files - let is_by = matches!(self.file(db).path(db).extension(), Some("by" | "byi")); + let is_by = self.file(db).source_type(db).is_basedpython(); if !is_by && !self.has_explicit_bases(db) { return &[]; } diff --git a/crates/ty_server/src/document/text_document.rs b/crates/ty_server/src/document/text_document.rs index 0d027b8e99..712ad1c646 100644 --- a/crates/ty_server/src/document/text_document.rs +++ b/crates/ty_server/src/document/text_document.rs @@ -38,6 +38,10 @@ pub struct TextDocument { #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum LanguageId { Python, + /// what the client calls basedpython, `basedpython` or `by`. a document whose path has no + /// extension saying which dialect it is — an editor's `untitled:Untitled-1` — is basedpython + /// for it, where one called python is python + BasedPython, /// A django template. Not python, but the server still has language services /// for it — see [`ty_ide::django_template_completions`]. DjangoTemplate, @@ -62,15 +66,16 @@ impl LanguageId { /// outright, ahead of the path fallback, because a client that says `jinja` /// has settled the question that fallback exists to guess at. /// - /// Editors disagree about what to call python too — `python3`, `py` and - /// `basedpython` are all in use — and [`Self::Other`] is a document the + /// editors disagree about what to call python too — `python3` and `py` are + /// both in use — and [`Self::Other`] is a document the /// server never adds to the project, so an id we do not know would leave a /// `.py` file with no diagnostics and every service reading the file as last /// saved rather than as the client has it. So a path python owns is python, /// exactly as a path django owns is a template. fn new(language_id: &LanguageKind, path: &AnySystemPath) -> Self { match language_id.as_str() { - "python" | "by" | "basedpython" => Self::Python, + "python" => Self::Python, + "by" | "basedpython" => Self::BasedPython, "django-html" | "django-txt" | "htmldjango" | "django" => Self::DjangoTemplate, "jinja" | "jinja-html" | "jinja2" => Self::Other, _ => Self::of_path(path), @@ -90,6 +95,11 @@ impl LanguageId { pub(crate) const fn is_django_template(self) -> bool { matches!(self, Self::DjangoTemplate) } + + /// whether the document is python of either dialect, which the server adds to the project + pub(crate) const fn is_python(self) -> bool { + matches!(self, Self::Python | Self::BasedPython) + } } impl TextDocument { @@ -264,11 +274,12 @@ mod tests { #[test] fn the_editors_python_language_ids_are_python() { - for id in ["python", "by", "basedpython"] { + assert_eq!(opened("/app/mod.py", "python"), LanguageId::Python); + for id in ["by", "basedpython"] { assert_eq!( opened("/app/mod.by", id), - LanguageId::Python, - "`{id}` is python" + LanguageId::BasedPython, + "`{id}` is basedpython" ); } } diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index bf6b3d4daa..d78840156c 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -1575,7 +1575,7 @@ impl Session { // script from disk. This ensures both use the saved metadata, so saving changed metadata // requests another synchronization. if self.use_uv != UseUv::Off - && language_id == LanguageId::Python + && language_id.is_python() && document.notebook().is_none() && let DocumentKey::File(system_path) = DocumentKey::from_uri(document.uri()) { diff --git a/crates/ty_server/src/system.rs b/crates/ty_server/src/system.rs index 6c694d90d0..94e76a5926 100644 --- a/crates/ty_server/src/system.rs +++ b/crates/ty_server/src/system.rs @@ -131,10 +131,16 @@ impl LSPSystem { ) -> Option { match document { Document::Text(text) => match text.language_id() { - LanguageId::Python => { + language @ (LanguageId::Python | LanguageId::BasedPython) => { + // the extension says which dialect, when it names one; otherwise the + // language the client opened the document in does let source_type = extension .and_then(PySourceType::try_from_extension) - .unwrap_or(PySourceType::Python); + .unwrap_or(if language == LanguageId::BasedPython { + PySourceType::BasedPython + } else { + PySourceType::Python + }); // JupyterLab presents notebook virtual documents to language servers as // simple text files rather than serialized `.ipynb` contents. See: diff --git a/crates/ty_server/tests/e2e/publish_diagnostics.rs b/crates/ty_server/tests/e2e/publish_diagnostics.rs index 676312e7ad..b09d352e53 100644 --- a/crates/ty_server/tests/e2e/publish_diagnostics.rs +++ b/crates/ty_server/tests/e2e/publish_diagnostics.rs @@ -166,6 +166,69 @@ def foo() -> str: Ok(()) } +/// a document is basedpython by the extension of its path whatever the uri's scheme, and by the +/// language the client opened it in when the path has no extension that says: an editor's +/// console, `untitled:console-1.by`, and its `untitled:Untitled-1`, which it calls `basedpython`, +/// both accept a `let`. a document the client calls python is python +#[test] +fn a_document_s_dialect_is_its_extension_or_its_language() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .enable_pull_diagnostics(false) + .build() + .wait_until_workspaces_are_initialized(); + + let source = "let rate = 0.2\nprint(rate)\n"; + let mut by_document = BTreeMap::new(); + for (uri, language) in [ + ("untitled:console-1.by", "basedpython"), + ("untitled:console-2.by", "python"), + ("vscode-vfs://github/app/main.by", "python"), + ("untitled:Untitled-3", "basedpython"), + ("inmemory://model/4", "by"), + ("untitled:Untitled-5", "python"), + ("untitled:script-6.py", "basedpython"), + ] { + server.send_notification::(DidOpenTextDocumentParams { + text_document: TextDocumentItem { + uri: Uri::parse(uri)?, + language_id: LanguageKind::from(language.to_string()), + version: 1, + text: source.to_string(), + }, + }); + let diagnostics = server.await_notification::(); + let messages: Vec = diagnostics + .diagnostics + .iter() + .map(|diagnostic| match &diagnostic.message { + Message::String(message) => message.clone(), + Message::MarkupContent(content) => content.value.clone(), + }) + .collect(); + by_document.insert(format!("{uri} ({language})"), messages); + } + + insta::assert_debug_snapshot!(by_document, @r#" + { + "inmemory://model/4 (by)": [], + "untitled:Untitled-3 (basedpython)": [], + "untitled:Untitled-5 (python)": [ + "`let` declarations are not valid in .py files", + ], + "untitled:console-1.by (basedpython)": [], + "untitled:console-2.by (python)": [], + "untitled:script-6.py (basedpython)": [ + "`let` declarations are not valid in .py files", + ], + "vscode-vfs://github/app/main.by (python)": [], + } + "#); + + Ok(()) +} + #[test] fn on_did_open_diagnostics_off() -> Result<()> { let workspace_root = SystemPath::new("src"); diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index be5dc5da39..2fb0668911 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -5,6 +5,15 @@ hints, an outline, and the rest of the language-server protocol. this page covers the parts that are basedpython's own — the ones you would not find by guessing from python +## which dialect a document is + +a document is basedpython or python by its path's extension, whatever the uri's +scheme: `untitled:console-1.by` and `vscode-vfs://github/app/main.by` are +basedpython, as `app/main.by` is, and `untitled:script.py` is python. a path +with no extension that says — an editor's `untitled:Untitled-1` — is the +language the editor opened it in: `basedpython` (or `by`) is basedpython, and +`python` is python + ## postfix templates writing `.` after an expression offers a set of templates that rewrite the From cedb429937b0ebc94eff69ea431892abdd1b13bf Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:02:05 +1000 Subject: [PATCH 092/110] tell an editor's console with by repl --events PORT, on a loopback connection of its own that program output cannot mix with, when the session waits for an input or the next line of one, when the program reads standard input through input(), sys.stdin or sys.stdin.buffer, and when each input finished and whether it ran, so over pipes it can send an input framed and a line for the program unframed rather than framing the program's line as an input --- crates/ty/docs/cli.md | 4 +- crates/ty/docs/environment.md | 8 ++ crates/ty/src/args.rs | 9 ++ crates/ty/src/by_commands.rs | 210 ++++++++++++++++++++++++++---- crates/ty/src/by_repl.rs | 79 +++++++++-- crates/ty/src/lib.rs | 8 +- crates/ty/tests/by_e2e.rs | 183 +++++++++++++++++++++++++- crates/ty_static/src/env_vars.rs | 7 + docs/basedpython/cli-reference.md | 3 +- docs/basedpython/features/repl.md | 27 +++- 10 files changed, 491 insertions(+), 47 deletions(-) diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index b8c06e1388..e1f5c35661 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -288,7 +288,9 @@ by repl [OPTIONS]

Options

-
--help, -h

Print help (see a summary with '-h')

+
--events port

Tell an editor what the session reads standard input for, on a connection of its own.

+

by repl connects to PORT on the loopback interface and writes a line of JSON there each time it waits for an input, the program reads standard input, or an input finishes: a console that talks to it through pipes cannot tell otherwise whether a line it sends is an input or a line for the program. The first line carries the token BY_REPL_EVENTS_TOKEN holds, for the editor to know the connection is the session's.

+
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the session's python must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

--python, --venv path

the interpreter to run on, or the environment holding it, as by run takes it

diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 4d072e8391..9088c47bb0 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -13,6 +13,14 @@ not open the socket that makes it reachable. Equivalent to the `--no-server` command-line argument. +### `BY_REPL_EVENTS_TOKEN` + +The token `by repl --events` sends first on the connection it makes to an editor. + +An editor listening for a session's events sets this to a value nothing else can guess, +and takes the connection whose first line carries it as the session's. The program the +session runs does not see it. + ### `BY_TEST_REPORTER` report the tests pytest runs under `by run` as service messages, `TeamCity`'s protocol diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 36cb2628f3..c2b02a2aff 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -167,6 +167,15 @@ pub(crate) enum Command { /// followed by the input being typed. #[arg(long, value_name = "PATH")] session_file: Option, + /// Tell an editor what the session reads standard input for, on a connection of its own. + /// + /// `by repl` connects to PORT on the loopback interface and writes a line of JSON there + /// each time it waits for an input, the program reads standard input, or an input + /// finishes: a console that talks to it through pipes cannot tell otherwise whether a + /// line it sends is an input or a line for the program. The first line carries the token + /// `BY_REPL_EVENTS_TOKEN` holds, for the editor to know the connection is the session's. + #[arg(long, value_name = "PORT")] + events: Option, }, /// Start a new project. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 09d07c7fc8..fc383970d3 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -2005,12 +2005,15 @@ pub(crate) const BY_REPL_FILENAME: &str = "_by_repl.py"; const BY_REPL_SRC: &str = r#"# generated by `by repl` — the python half of a session, see `by_repl.rs` import ast +import contextlib +import io import json import linecache import os import signal import socket import sys +import threading import types import _by_runner @@ -2021,6 +2024,35 @@ _PASTE_END = "\x1b[201~" _INTERACTIVE = sys.stdin.isatty() +# standard input as this file reads it, for the inputs: the program's `sys.stdin` can be +# another object, which says when the program reads +_STDIN = sys.stdin + + +@contextlib.contextmanager +def _holding_ctrl_c(): + """A ctrl-c that arrives inside the block is raised once it has finished, so a message is + never cut short. Only the main thread receives signals, so only it has any to hold.""" + if threading.current_thread() is not threading.main_thread(): + yield + return + interrupted = [] + previous = signal.signal(signal.SIGINT, lambda *_: interrupted.append(True)) + try: + yield + finally: + signal.signal(signal.SIGINT, previous) + if interrupted: + raise KeyboardInterrupt + + +class _Interrupted(KeyboardInterrupt): + """A ctrl-c that arrived while `by repl` answered, with the answer it gave.""" + + def __init__(self, reply): + super().__init__() + self.reply = reply + class _Session: """The connection to `by repl`, which says whether an input is whole and what it runs as.""" @@ -2028,29 +2060,147 @@ class _Session: def __init__(self): port = int(os.environ.pop("BY_REPL_PORT")) token = os.environ.pop("BY_REPL_TOKEN") + # whether an editor listens for events: what standard input is read for, and when an + # input has finished + self.events = os.environ.pop("BY_REPL_EVENTS", None) == "1" self._socket = socket.create_connection(("127.0.0.1", port)) self._reader = self._socket.makefile("r", encoding="utf-8", newline="\n") + # a thread of the program's can read standard input while the main thread talks to + # `by repl`, and each message goes whole + self._lock = threading.Lock() self.tell({"token": token}) + def _send(self, message): + data = (json.dumps(message) + "\n").encode("utf-8") + with self._lock: + self._socket.sendall(data) + def tell(self, message): - self._socket.sendall((json.dumps(message) + "\n").encode("utf-8")) + with _holding_ctrl_c(): + self._send(message) def ask(self, message): # a ctrl-c that arrives while the answer is on its way is held until it has arrived, # so the next question is not answered with this one's answer - interrupted = [] - previous = signal.signal(signal.SIGINT, lambda *_: interrupted.append(True)) + line = "" try: - self.tell(message) - line = self._reader.readline() - finally: - signal.signal(signal.SIGINT, previous) + with _holding_ctrl_c(): + self._send(message) + line = self._reader.readline() + except KeyboardInterrupt: + if not line: + raise EOFError from None + raise _Interrupted(json.loads(line)) from None if not line: raise EOFError - if interrupted: - raise KeyboardInterrupt return json.loads(line) + def event(self, event): + """Tell the editor `event`, after what has been printed so far: it is in the pipes + before the event is sent.""" + if not self.events: + return + sys.stdout.flush() + sys.stderr.flush() + self.tell({"event": event}) + + +class _ProgramStdin(io.TextIOBase): + """`sys.stdin` for the program while an editor listens for events: each read says first + that the program is reading standard input, so the editor sends the next line as the + program's rather than as an input.""" + + def __init__(self, stdin, session): + self._stdin = stdin + self._session = session + self._buffer = _ProgramStdinBuffer(stdin.buffer, session) + + def _reading(self): + self._session.event({"reading": "program"}) + + def read(self, size=-1): + self._reading() + return self._stdin.read(size) + + def readline(self, size=-1): + self._reading() + return self._stdin.readline(size) + + def readlines(self, hint=-1): + self._reading() + return self._stdin.readlines(hint) + + def readable(self): + return True + + def fileno(self): + return self._stdin.fileno() + + def isatty(self): + return self._stdin.isatty() + + @property + def buffer(self): + return self._buffer + + @property + def encoding(self): + return self._stdin.encoding + + @property + def errors(self): + return self._stdin.errors + + @property + def newlines(self): + return self._stdin.newlines + + def __getattr__(self, name): + return getattr(self._stdin, name) + + +class _ProgramStdinBuffer(io.BufferedIOBase): + """`sys.stdin.buffer` for the program, saying it reads as `_ProgramStdin` does.""" + + def __init__(self, buffer, session): + self._inner = buffer + self._session = session + + def _reading(self): + self._session.event({"reading": "program"}) + + def read(self, size=-1): + self._reading() + return self._inner.read(size) + + def read1(self, size=-1): + self._reading() + return self._inner.read1(size) + + def readinto(self, buffer): + self._reading() + return self._inner.readinto(buffer) + + def readinto1(self, buffer): + self._reading() + return self._inner.readinto1(buffer) + + def readline(self, size=-1): + self._reading() + return self._inner.readline(size) + + def readable(self): + return True + + def fileno(self): + return self._inner.fileno() + + def isatty(self): + return self._inner.isatty() + + def __getattr__(self, name): + return getattr(self._inner, name) + def _read(prompt): # what the last input printed is out before anything else is read, as it is on a terminal, @@ -2059,7 +2209,7 @@ def _read(prompt): sys.stderr.flush() if _INTERACTIVE: return input(prompt) + "\n" - line = sys.stdin.readline() + line = _STDIN.readline() if not line: raise EOFError return line @@ -2067,6 +2217,7 @@ def _read(prompt): def _read_input(session): """The next input: lines until `by repl` says they are whole, or a framed block.""" + session.event({"reading": "input"}) line = _read(">>> ") if line.startswith(_PASTE_START): text = line[len(_PASTE_START):] @@ -2075,16 +2226,24 @@ def _read_input(session): return text[: text.index(_PASTE_END)] text = line while session.ask({"status": text})["status"] == "incomplete": + session.event({"reading": "continuation"}) text += _read("... ") return text def _run(session, namespace, text): - reply = session.ask({"run": text}) + """Run `text`, an input: the number it was given, and whether it ran, which makes it part + of the session whatever it raised.""" + try: + reply = session.ask({"run": text}) + except _Interrupted as interrupted: + # as python's repl drops an input interrupted while it compiles + sys.stderr.write("\nKeyboardInterrupt\n") + return interrupted.reply["input"], False + number = reply["input"] if "python" not in reply: # refused: `by repl` has said why - return - number = reply["input"] + return number, False filename = f"" lines = text.splitlines(True) # what a traceback quotes for a line of the input, for as long as the session lasts @@ -2119,15 +2278,19 @@ def _run(session, namespace, text): except Exception: sys.stderr.write("error: the python this input was transpiled to does not compile:\n") sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) - return - session.tell({"ran": number}) + return number, False + except KeyboardInterrupt: + sys.stderr.write("\nKeyboardInterrupt\n") + return number, False try: + session.tell({"ran": number}) for code in codes: exec(code, namespace) except SystemExit: raise except BaseException: sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) + return number, True def main(): @@ -2138,6 +2301,8 @@ def main(): main_module = types.ModuleType("__main__") sys.modules["__main__"] = main_module namespace = main_module.__dict__ + if session.events: + sys.stdin = _ProgramStdin(_STDIN, session) if _INTERACTIVE: try: import readline # noqa: F401 — line editing for `input` @@ -2146,22 +2311,17 @@ def main(): while True: try: text = _read_input(session) + if not text.strip(): + continue + number, ran = _run(session, namespace, text) + session.event({"finished": number, "ran": ran}) except EOFError: + # the end of the session's input, or `by repl` has gone, and with it the session if _INTERACTIVE: sys.stdout.write("\n") return except KeyboardInterrupt: sys.stderr.write("\nKeyboardInterrupt\n") - continue - if not text.strip(): - continue - try: - _run(session, namespace, text) - except KeyboardInterrupt: - sys.stderr.write("\nKeyboardInterrupt\n") - except EOFError: - # `by repl` has gone, and with it the session - return if __name__ == "__main__": diff --git a/crates/ty/src/by_repl.rs b/crates/ty/src/by_repl.rs index e83a932140..a7cf6839bb 100644 --- a/crates/ty/src/by_repl.rs +++ b/crates/ty/src/by_repl.rs @@ -13,6 +13,10 @@ //! input, and the program's output is its own. The interpreter is told the port and a token in //! its environment, and the first thing it sends is the token, so nothing else on the machine //! that finds the port is answered. +//! +//! For the same reason an editor that asks with `--events` is told what standard input is +//! being read for — an input, or the program — on a connection of its own: the interpreter says +//! so to this process, which passes it on. use std::fs; use std::hash::{BuildHasher, RandomState}; @@ -42,6 +46,8 @@ use ruff_text_size::{TextRange, TextSize}; use serde_json::{Value, json}; use ty_project::{Db as _, ProjectDatabase}; +use ty_static::EnvVars; + use crate::ExitStatus; use crate::args::LoweringArgs; use crate::by_commands::{BY_REPL_FILENAME, RunTarget, StagedRun, stage_run}; @@ -50,17 +56,27 @@ use crate::by_commands::{BY_REPL_FILENAME, RunTarget, StagedRun, stage_run}; /// has connected: it does both before it runs anything of the session's. const CONNECT_TIMEOUT: Duration = Duration::from_secs(60); +/// What `by repl` is given beyond what it stages the project with. +pub(crate) struct ReplFlags<'a> { + /// `--python`: the interpreter, or the environment holding it + pub(crate) python: Option<&'a Path>, + /// `--session-file`: where an editor reads the session's source + pub(crate) session_file: Option<&'a Path>, + /// `--events`: the port on the loopback interface an editor listens for the session's + /// events on + pub(crate) events: Option, +} + #[allow(clippy::exit, clippy::print_stderr)] pub(crate) fn cmd_repl( min_version: Option<&str>, lowering: &LoweringArgs, - python_flag: Option<&Path>, - session_file: Option<&Path>, + flags: &ReplFlags<'_>, ) -> anyhow::Result { let staged = match stage_run( min_version, lowering, - python_flag, + flags.python, false, RunTarget::Session, )? { @@ -75,28 +91,34 @@ pub(crate) fn cmd_repl( rebuilder, .. } = staged; - let session_file = session_file.map(|path| cwd.join(path)); + let session_file = flags.session_file.map(|path| cwd.join(path)); if let Some(path) = &session_file { fs::write(path, "").with_context(|| format!("could not write {}", path.display()))?; } - let mut repl = Repl::new(&rebuilder, config, session_file)?; + let events = flags.events.map(connect_events).transpose()?; + let mut repl = Repl::new(&rebuilder, config, session_file, events)?; let listener = TcpListener::bind(("127.0.0.1", 0)).context("could not listen for the session")?; let port = listener.local_addr()?.port(); let token = token(); - let mut child = Command::new(&interpreter.path) + let mut command = Command::new(&interpreter.path); + command .arg(tree.path().join(BY_REPL_FILENAME)) .env("BY_REPL_PORT", port.to_string()) .env("BY_REPL_TOKEN", &token) - .current_dir(&cwd) - .spawn() - .with_context(|| { - format!( - "could not run `{}`, the interpreter from {}", - interpreter.path, interpreter.origin - ) - })?; + // the editor's, not the program's + .env_remove(EnvVars::BY_REPL_EVENTS_TOKEN) + .current_dir(&cwd); + if flags.events.is_some() { + command.env("BY_REPL_EVENTS", "1"); + } + let mut child = command.spawn().with_context(|| { + format!( + "could not run `{}`, the interpreter from {}", + interpreter.path, interpreter.origin + ) + })?; // ctrl-c is the interpreter's to answer, with a `KeyboardInterrupt` in the input it is // running; this process lives as long as the session does ctrlc::set_handler(|| {}).context("could not hand ctrl-c to the session")?; @@ -119,6 +141,17 @@ pub(crate) fn cmd_repl( std::process::exit(status.code().unwrap_or(1)); } +/// The editor's connection for the session's events, to `port` on the loopback interface, with +/// the token it set in `BY_REPL_EVENTS_TOKEN` sent on it. +fn connect_events(port: u16) -> anyhow::Result { + let mut stream = TcpStream::connect(("127.0.0.1", port)) + .with_context(|| format!("could not connect to the editor's events port, {port}"))?; + let token = std::env::var(EnvVars::BY_REPL_EVENTS_TOKEN).ok(); + writeln!(stream, "{}", json!({ "token": token }))?; + stream.flush()?; + Ok(stream) +} + /// A token nothing else can guess, from the process's own random hasher keys. fn token() -> String { let state = RandomState::new(); @@ -193,6 +226,9 @@ struct Repl { next: u32, /// `--session-file`: where an editor reads the session's source session_file: Option, + /// `--events`: the editor's connection, which each event the interpreter reports is + /// passed on to + events: Option, } struct RanInput { @@ -212,6 +248,7 @@ impl Repl { rebuilder: &Rebuilder, config: Config, session_file: Option, + events: Option, ) -> anyhow::Result { let module_dir = tempfile::TempDir::new().context("failed to create temp directory")?; // a module name nothing a project imports is called, since the directory is searched @@ -248,6 +285,7 @@ impl Repl { pending: None, next: 1, session_file, + events, }) } @@ -268,6 +306,9 @@ impl Repl { } else if let Some(number) = message["ran"].as_u64() { self.ran(number)?; None + } else if let Some(event) = message.get("event") { + self.event(event)?; + None } else { anyhow::bail!("an unknown message: {line}"); }; @@ -392,6 +433,16 @@ impl Repl { Ok(()) } + /// Pass `event`, which the interpreter reported, on to the editor listening for it, if one is. + fn event(&mut self, event: &Value) -> anyhow::Result<()> { + if let Some(events) = &mut self.events { + writeln!(events, "{event}") + .and_then(|()| events.flush()) + .context("the editor stopped listening for the session's events")?; + } + Ok(()) + } + /// `diagnostic` with every place it points to in the session's module moved to the input /// that place is in, `input` being the one at `start` that has not run yet. fn in_inputs( diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index d460d5c5b4..f1a4c796b6 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -148,11 +148,15 @@ fn run_command(command: Command) -> anyhow::Result { python, lowering, session_file, + events, } => by_repl::cmd_repl( min_version.as_deref(), &lowering, - python.as_deref(), - session_file.as_deref(), + &by_repl::ReplFlags { + python: python.as_deref(), + session_file: session_file.as_deref(), + events, + }, ), Command::Init { path, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 3617039c64..4b9a200574 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1,9 +1,11 @@ use by_ir::mangle::mangle; use std::fmt::Write as _; use std::fs; -use std::io::Write; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::time::{Duration, Instant}; use ty_static::EnvVars; /// a directory a test builds a project in, which goes when the test passes @@ -2071,6 +2073,183 @@ fn repl_session_file_holds_the_inputs_that_ran() { ); } +/// A `by repl` session in a project, driven as an editor's console drives one: through pipes, +/// listening for the session's events on a port of its own with `--events`. +struct EditorSession { + child: Child, + stdin: ChildStdin, + events: BufReader, +} + +impl EditorSession { + /// The session started in `dir` with `extra` arguments, its connection taken once it has + /// sent the token the editor gave it. + fn start(dir: &Path, extra: &[&str]) -> Self { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("a port to listen on"); + let port = listener.local_addr().expect("the port").port().to_string(); + let mut child = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .env(EnvVars::BY_REPL_EVENTS_TOKEN, "the editor's token") + .env("NO_COLOR", "1") + .args(["repl", "--python", &python, "--events", &port]) + .args(extra) + .current_dir(dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn by"); + listener + .set_nonblocking(true) + .expect("a listener that does not block"); + let started = Instant::now(); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the session") { + let output = child.wait_with_output().expect("the session's output"); + panic!( + "the session ended ({status}) without connecting: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + assert!( + started.elapsed() < Duration::from_secs(120), + "the session did not connect" + ); + std::thread::sleep(Duration::from_millis(20)); + } + Err(error) => panic!("could not accept the session: {error}"), + } + }; + stream.set_nonblocking(false).expect("a blocking stream"); + stream + .set_read_timeout(Some(Duration::from_secs(120))) + .expect("a read timeout"); + let stdin = child.stdin.take().expect("stdin"); + let mut session = Self { + child, + stdin, + events: BufReader::new(stream), + }; + assert_eq!( + session.next_event(), + serde_json::json!({ "token": "the editor's token" }) + ); + session + } + + /// The next event the session sent, `null` once it has closed the connection. + fn next_event(&mut self) -> serde_json::Value { + let mut line = String::new(); + self.events.read_line(&mut line).expect("an event"); + if line.is_empty() { + return serde_json::Value::Null; + } + serde_json::from_str(&line).expect("an event is a line of json") + } + + /// Wait for `expected`, the next event. + #[track_caller] + fn expect(&mut self, expected: &serde_json::Value) { + assert_eq!(&self.next_event(), expected); + } + + /// Send `text` to the session's standard input as it is. + fn send(&mut self, text: &str) { + self.stdin + .write_all(text.as_bytes()) + .expect("the session reads"); + self.stdin.flush().expect("the session reads"); + } + + /// Send `text` as an editor sends an input, framed as a paste. + fn send_input(&mut self, text: &str) { + self.send(&format!("\x1b[200~{text}\x1b[201~\n")); + } + + /// End the session's input, and what it printed and how it ended. + fn finish(self) -> (Option, String, String, Vec) { + let Self { + child, + stdin, + mut events, + } = self; + drop(stdin); + let output = child.wait_with_output().expect("the session ends"); + let mut rest = Vec::new(); + let mut line = String::new(); + while events.read_line(&mut line).is_ok_and(|read| read > 0) { + rest.push(serde_json::from_str(&line).expect("an event is a line of json")); + line.clear(); + } + // python writes its output in text mode, which ends a line with `\r\n` on windows + ( + output.status.code(), + String::from_utf8_lossy(&output.stdout).replace("\r\n", "\n"), + String::from_utf8_lossy(&output.stderr).replace("\r\n", "\n"), + rest, + ) + } +} + +/// with `--events`, an editor is told on a connection of its own what standard input is read +/// for: an input, the next line of one, or the program, whether it reads with `input()`, +/// `sys.stdin` or `sys.stdin.buffer` — so a line it sends for the program is sent as the +/// program's, unframed, and arrives as it was typed — and when each input has finished, and +/// whether it ran +#[test] +fn repl_tells_an_editor_what_standard_input_is_read_for() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut session = EditorSession::start(dir.path(), &[]); + session.expect(&serde_json::json!({ "reading": "input" })); + + session.send_input("name = input('who? ')"); + session.expect(&serde_json::json!({ "reading": "program" })); + session.send("bob\n"); + session.expect(&serde_json::json!({ "finished": 1, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + + // refused: it does not run, and is not part of the session + session.send_input("let count: int = name"); + session.expect(&serde_json::json!({ "finished": 2, "ran": false })); + session.expect(&serde_json::json!({ "reading": "input" })); + + // an input sent a line at a time waits for the line that completes it + session.send("if name:\n"); + session.expect(&serde_json::json!({ "reading": "continuation" })); + session.send(" print(repr(name))\n"); + session.expect(&serde_json::json!({ "reading": "continuation" })); + session.send("\n"); + session.expect(&serde_json::json!({ "finished": 3, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + + session.send_input("import sys\nlines = [sys.stdin.readline(), sys.stdin.buffer.readline()]"); + session.expect(&serde_json::json!({ "reading": "program" })); + session.send("text\n"); + session.expect(&serde_json::json!({ "reading": "program" })); + session.send("bytes\n"); + session.expect(&serde_json::json!({ "finished": 4, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + + // an input that raises has run all the same + session.send_input("lines[2]"); + session.expect(&serde_json::json!({ "finished": 5, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + session.send_input("lines"); + session.expect(&serde_json::json!({ "finished": 6, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + + let (status, stdout, stderr, rest) = session.finish(); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!(stdout, "who? 'bob'\n['text\\n', b'bytes\\n']\n", "{stderr}"); + assert!(stderr.contains("IndexError"), "{stderr}"); + // the connection closes with the session + assert_eq!(rest, Vec::::new()); +} + /// A project of `.by` tests for pytest to run, with a module they test, checked /// against the environment the tests run in: the `conftest` imports pytest, which /// `--python` does not configure. `None`, having said why, when `$PYTHON` (or diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 306e6cf7eb..220704940b 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -79,6 +79,13 @@ impl EnvVars { /// side was. Any other value is refused. pub const BY_TEST_REPORTER: &'static str = "BY_TEST_REPORTER"; + /// The token `by repl --events` sends first on the connection it makes to an editor. + /// + /// An editor listening for a session's events sets this to a value nothing else can guess, + /// and takes the connection whose first line carries it as the session's. The program the + /// session runs does not see it. + pub const BY_REPL_EVENTS_TOKEN: &'static str = "BY_REPL_EVENTS_TOKEN"; + /// Enable uv integration. /// /// When set to `"1"` or `"true"`, ty invokes `uv workspace metadata` to discover the workspace diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index e63e7cd501..390af0bb80 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -184,7 +184,8 @@ an interactive session in which you type basedpython, each input checked and tra continuation of the inputs before it and run in one python process. it starts where `by run` would, with the project staged as `by run` stages it, so the project's modules import as they do in a program. `--session-file PATH` keeps `PATH` holding the inputs that ran, for an editor's -console to complete against. see [the repl](features/repl.md) +console to complete against, and `--events PORT` tells the console, on a connection to `PORT`, +whether standard input is read for an input or by the program. see [the repl](features/repl.md) ## `by build` diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index 1b2e50c4e3..c282ae3046 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -100,14 +100,37 @@ inside a function body included, and runs when the closing marker arrives. this editor sends one while an input runs, the lines that arrive are the program's: `input()` reads them, as it reads -python's +python's, and an editor is told when that is with [`--events`](#in-an-editor) ## in an editor an editor's console runs `by repl` as a process and talks to it through its standard streams: - **sending.** each input is written to `by repl`'s standard input framed, as above, followed by - a newline. while the program reads `input()`, a line is written as it is + a newline. a line for the program — what `input()` reads — is written as it is, unframed + +- **events.** over a pipe a console cannot tell whether `by repl` is waiting for an input or the + program is reading standard input, and the two are sent differently. `--events PORT` has + `by repl` tell it, on a connection of its own that program output cannot mix with: it + connects to `PORT` on the loopback interface (`127.0.0.1`) before the session starts, and + writes one line of json there for each event. the first line is + `{"token": T}`, where `T` is the value of `BY_REPL_EVENTS_TOKEN` in `by repl`'s environment + (`null` when it is unset), for the console to know the connection is the session's; the + program does not see the variable. then: + + | event | meaning | + | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | + | `{"reading": "input"}` | `by repl` waits for the next input: send one, framed | + | `{"reading": "continuation"}` | an input sent unframed is not whole yet, and `by repl` waits for its next line | + | `{"reading": "program"}` | the program reads standard input — `input()`, `sys.stdin` or `sys.stdin.buffer` — and the next line is its own: send it unframed | + | `{"finished": N, "ran": R}` | input `N`, as it is numbered in ``, has finished. `R` is `true` when it ran, which makes it part of the session even when it raised, and `false` when it did not: it was refused, or interrupted before it started | + + `reading` is sent each time standard input is read for that purpose, and `finished` before + the `reading` that follows it. what the session printed before an event is in its pipes before + the event is sent. a line sent while nothing is reading waits for whatever reads next, as + typed-ahead text does in a terminal; a thread of the program's that reads standard input + while the session waits for an input reads lines meant for either. the connection closes + when the session ends - **output.** the program's output, the printed values and the tracebacks are the process's standard output and standard error, as they are python's. prompts are written only when From f7f96a32eb3dbfb2c0a8059d53848a50321ef456 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:10:27 +1000 Subject: [PATCH 093/110] name the runtime's helpers in a by run --profile snapshot , at the line of _by_runtime.py they start on and by their own names, whether the transpiler wrote them into a module or they ran from a package's copy of the runtime, so one helper is one function, two of one name stay two, and a viewer opens nothing rather than line 0 of the module a helper was written into, which PyCharm's viewer opened at the module's first line --- crates/by_transforms/src/runtime.rs | 33 +++++ crates/ty/docs/cli.md | 2 +- crates/ty/src/args.rs | 5 +- crates/ty/src/by_commands.rs | 181 +++++++++++++++++++++++++--- crates/ty/tests/by_e2e.rs | 99 +++++++++++++++ docs/basedpython/cli-reference.md | 14 ++- 6 files changed, 309 insertions(+), 25 deletions(-) diff --git a/crates/by_transforms/src/runtime.rs b/crates/by_transforms/src/runtime.rs index 12540b519b..707ed5b8bb 100644 --- a/crates/by_transforms/src/runtime.rs +++ b/crates/by_transforms/src/runtime.rs @@ -130,6 +130,8 @@ struct Definition { /// position in the file. a set of definitions renders in this order, which /// is the order they can be executed in order: usize, + /// the line of the file it starts on, 1-based, its decorators included + line: u32, /// the other definitions its body reads needs: Vec, /// each read of a name the runtime does not define, which python looks up in the @@ -174,6 +176,7 @@ fn build_index() -> BTreeMap { last += 1; } let span = suite[at].range().cover(suite[last].range()); + let line = SOURCE[..span.start().to_usize()].matches('\n').count() + 1; let mut source = SOURCE[span].to_owned(); if !source.ends_with('\n') { source.push('\n'); @@ -196,6 +199,7 @@ fn build_index() -> BTreeMap { definitions.entry(name).or_insert_with(|| Definition { source: source.clone(), order, + line: u32::try_from(line).unwrap_or(u32::MAX), needs: needs.clone(), builtin_reads: builtin_reads.clone(), }); @@ -1061,6 +1065,19 @@ pub(crate) fn defines(name: &str) -> bool { index().contains_key(name) } +/// the line of [`SOURCE`] each definition starts on, 1-based and its decorators included, by +/// each name it binds +/// +/// a definition pasted into a module is the same lines, so a line of it in the module is the +/// line of [`SOURCE`] as far into the definition: what a run names the runtime's code by, +/// wherever it ran from +pub fn definition_lines() -> BTreeMap<&'static str, u32> { + index() + .iter() + .map(|(name, definition)| (name.as_str(), definition.line)) + .collect() +} + /// the definitions `helpers` need, as preamble entries for a module that has nowhere to /// import them from, whose top-level statements are `module` and whose names `written` /// holds @@ -1173,6 +1190,22 @@ mod tests { } } + /// a definition's line is where its text starts in the file: a decorated one at its first + /// decorator + #[test] + fn a_definition_starts_on_its_line() { + let lines: Vec<&str> = SOURCE.lines().collect(); + for (name, line) in definition_lines() { + let definition = &index()[name]; + let first = definition.source.lines().next().unwrap_or_default(); + assert_eq!(lines[line as usize - 1], first, "`{name}` at line {line}"); + } + assert!( + lines[definition_lines()["_force_unwrap"] as usize - 1] + .starts_with("def _force_unwrap") + ); + } + /// the preamble is one entry per line, so an entry that did not end its own /// line would run into the next #[test] diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index e1f5c35661..c93dc420a6 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -266,7 +266,7 @@ by run [OPTIONS] [MODULE] [ARGS]...
--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

--profile path

Profile the program with cProfile and write the snapshot to PATH, as the pstats file cProfile writes (.pstat).

-

Every function of a transpiled module is named by its .by file and the .by line it starts at, and one of a .py the project carries over by the file in the project rather than the staged copy that ran, so a profile viewer's way to a function's source leads to the file it was written in. The profile starts at the program's first line and ends after its last atexit handler.

+

Every function of a transpiled module is named by its .by file and the .by line it starts at, and one of a .py the project carries over by the file in the project rather than the staged copy that ran, so a profile viewer's way to a function's source leads to the file it was written in. The runtime's helpers are named <by runtime>, at the line of _by_runtime.py they start on. The profile starts at the program's first line and ends after its last atexit handler.

--python, --venv path

The interpreter to run on, or the environment holding it. A bare name that is neither, like python3.14, is found on PATH.

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index c2b02a2aff..56ba3a1afe 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -139,8 +139,9 @@ pub(crate) enum Command { /// the `.by` line it starts at, and one of a `.py` the project carries over /// by the file in the project rather than the staged copy that ran, so a /// profile viewer's way to a function's source leads to the file it was - /// written in. The profile starts at the program's first line and ends - /// after its last `atexit` handler. + /// written in. The runtime's helpers are named ``, at the line + /// of `_by_runtime.py` they start on. The profile starts at the program's + /// first line and ends after its last `atexit` handler. #[arg(long, value_name = "PATH", conflicts_with = "coverage")] profile: Option, }, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index fc383970d3..39fd6a4396 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1978,7 +1978,7 @@ fn write_traceback_runtime( // through the staging, so that a project of its own with this name is // reported as the collision it is rather than silently overwritten by a shim // it knows nothing about - staging.write(Path::new(BY_RUNNER_FILENAME), None, BY_RUNNER_SRC)?; + staging.write(Path::new(BY_RUNNER_FILENAME), None, &runner_source()?)?; staging.write( Path::new(BY_COVERAGE_PLUGIN_FILENAME), None, @@ -2739,6 +2739,25 @@ fn write_project_module( staging.write(Path::new(BY_PROJECT_FILENAME), None, &text) } +/// the line [`BY_RUNNER_SRC`] holds for the runtime's definitions, which [`runner_source`] fills in +const RUNTIME_LINES_PLACEHOLDER: &str = "_RUNTIME_LINES = {}\n"; + +/// the runner shim, with the line of `_by_runtime.py` each of the runtime's definitions starts +/// on, by the names it binds: what a profile names the runtime's code by +fn runner_source() -> anyhow::Result { + anyhow::ensure!( + BY_RUNNER_SRC.contains(RUNTIME_LINES_PLACEHOLDER), + "the runner has no place for the runtime's lines" + ); + // a json object of strings and integers is a python dict literal + let lines = serde_json::to_string(&by_transforms::runtime::definition_lines())?; + Ok(BY_RUNNER_SRC.replacen( + RUNTIME_LINES_PLACEHOLDER, + &format!("_RUNTIME_LINES = {lines}\n"), + 1, + )) +} + const BY_RUNNER_SRC: &str = r#"# generated by `by run` — runs the target module with .by-aware tracebacks import atexit import dis @@ -2765,6 +2784,16 @@ _BY_MAP = {os.path.realpath(py): (py, info) for py, info in SOURCEMAP.items()} # with the key of the entry each was transpiled under _BY_SOURCES = {os.path.realpath(info[0]): py for py, info in SOURCEMAP.items()} +# the line of `_by_runtime.py`, as `by` writes it, each of the runtime's definitions starts +# on, by the names it binds. `by run` fills it in +_RUNTIME_LINES = {} +# what a profile names the runtime's code by, which is no file: `` is python's +# own name for code it holds rather than reads +_RUNTIME_FILE = "" +# while the program is profiled, each code object `_relined` gave no `.by` line, by its id, +# with the staged module and the line of it it was compiled from +_TRANSPILER_WRITTEN = None + # this file, whose frames are never the program's _RUNNER = os.path.realpath(__file__) # the start of the name `by repl` compiles an input under, `` @@ -2987,7 +3016,16 @@ def _svarint(out, value): def _relined(code, filename, by_line, width): """`code` and every code object it holds, named `filename`, with each location - moved to the line `by_line` gives, spanning the whole of that line. + moved to the line `by_line` gives, spanning the whole of that line.""" + relined = _relined_one(code, filename, by_line, width) + if _TRANSPILER_WRITTEN is not None and relined.co_firstlineno == 0: + # line 0 says only that the transpiler wrote it; a profile needs to know what + _TRANSPILER_WRITTEN[id(relined)] = (relined, code.co_filename, code.co_firstlineno) + return relined + + +def _relined_one(code, filename, by_line, width): + """`_relined`, for `code` itself. The line table's format is private to each python version: see `Objects/locations.md` (3.11 on) and `Objects/lnotab_notes.txt` (3.10, and 3.9 before it) in cpython. @@ -3521,6 +3559,8 @@ def _start_profile(snapshot): which python runs before the ones registered ahead of them.""" import cProfile + global _TRANSPILER_WRITTEN + _TRANSPILER_WRITTEN = {} profiler = cProfile.Profile() atexit.register(_write_profile, profiler, snapshot) return profiler @@ -3549,37 +3589,138 @@ def _write_profile(profiler, snapshot): lines, and moves to its `.by` here, as a staged copy of a hand-written `.py` moves to the project file. The two can then name one function, and are added up as one. A line the transpiler wrote on its own has no `.by` line, and is line 0. + + The runtime's own code is the exception: a helper the transpiler wrote into a + module, and one in a copy of the runtime written beside a package's modules, are + `` at the line of `_by_runtime.py` they are, under their own names. A + helper is then one function wherever it ran from, two helpers of one name are two, + and a viewer's way to a function's source finds no file to open, where line 0 of + the module it was written into was the module's first line. """ profiler.disable() - profiler.create_stats() located = {} - - def project_function(function): - filename, line, name = function - if function not in located: + parsed = {} + + def function(code): + # a builtin is its name, as `cProfile` keys it + if isinstance(code, str): + return ("~", 0, code) + if id(code) not in located: + filename, line, name = code.co_filename, code.co_firstlineno, code.co_name + written = _TRANSPILER_WRITTEN.get(id(code)) + if written is not None and written[0] is code: + # moved to the `.by`, where it has no line: the staged module's line says + # what it is + _, filename, line = written + runtime_line = _runtime_line(filename, line, name, parsed) real = os.path.realpath(filename) - if real in _BY_MAP or real in _CARRIED: + if runtime_line is not None: + located[id(code)] = (_RUNTIME_FILE, runtime_line, name) + elif real in _BY_MAP or real in _CARRIED: path, line = project_location(filename, line) - located[function] = (path, line or 0, name) + located[id(code)] = (path, line or 0, name) else: - located[function] = function - return located[function] + located[id(code)] = (filename, line, name) + return located[id(code)] + # what `cProfile.Profile.create_stats` makes of the entries, keyed as above rather than + # by each code object's own file, line and name, under which two functions can collide + entries = profiler.getstats() stats = {} - for function, (*counts, callers) in profiler.stats.items(): - by_caller = {} - for caller, timing in callers.items(): - caller = project_function(caller) - by_caller[caller] = ( - _summed(by_caller[caller], timing) if caller in by_caller else timing - ) - row = (*counts, by_caller) - key = project_function(function) + for entry in entries: + key = function(entry.code) + row = ( + entry.callcount - entry.reccallcount, + entry.callcount, + entry.inlinetime, + entry.totaltime, + {}, + ) stats[key] = _summed(stats[key], row) if key in stats else row + for entry in entries: + caller = function(entry.code) + for call in entry.calls or (): + callee = function(call.code) + if callee not in stats: + continue + timing = ( + call.callcount, + call.callcount - call.reccallcount, + call.inlinetime, + call.totaltime, + ) + callers = stats[callee][4] + callers[caller] = _summed(callers[caller], timing) if caller in callers else timing with open(snapshot, "wb") as handle: marshal.dump(stats, handle) +def _runtime_line(path, line, name, parsed): + """The line of `_by_runtime.py`, as `by` writes it, that the code named `name` starting + at `line` of the staged file at `path` is, when it is the runtime's: in a copy of the + runtime written beside a package's modules, or in one of the runtime's definitions the + transpiler wrote into a module. `None` for any other code. `parsed` keeps each module's + top-level statements once read.""" + import ast + + real = os.path.realpath(path) + if ( + os.path.basename(real) == "_by_runtime.py" + and os.path.commonpath([real, _TREE]) == _TREE + ): + # no project file can take this name in the tree, which the staging refuses + return line + entry = _BY_MAP.get(real) + # a module's own code starts at its first line, which can be a definition of the + # runtime's, and is the module's all the same + if entry is None or not line or name == "": + return None + _, (_, lines) = entry + if real not in parsed: + with open(real, "rb") as handle: + parsed[real] = ast.parse(handle.read()).body + statements = parsed[real] + + def start(stmt): + return min([stmt.lineno, *(d.lineno for d in getattr(stmt, "decorator_list", ()))]) + + def written(stmt): + first = start(stmt) + return not (0 < first <= len(lines) and lines[first - 1] is not None) + + def bound(stmt): + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return [stmt.name] + if isinstance(stmt, ast.Assign): + return [target.id for target in stmt.targets if isinstance(target, ast.Name)] + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + return [stmt.target.id] + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + return [alias.asname or alias.name.split(".")[0] for alias in stmt.names] + return [] + + index = next( + ( + index + for index, stmt in enumerate(statements) + if start(stmt) <= line <= stmt.end_lineno + ), + None, + ) + if index is None: + return None + # a statement that binds nothing is part of the definition before it, as `runtime.rs` + # slices the runtime into definitions + while index > 0 and not bound(statements[index]) and written(statements[index]): + index -= 1 + stmt = statements[index] + if not written(stmt): + return None + for name in bound(stmt): + if name in _RUNTIME_LINES: + return _RUNTIME_LINES[name] + line - start(stmt) + return None + def main(): # the plugins `by run` puts in the tree reach this module by name; it is running # as `__main__`, which the program's own module takes over below diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 4b9a200574..f5747322b4 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1852,6 +1852,105 @@ fn run_profile_names_functions_by_their_project_files() { assert!(staged.is_empty(), "a staged copy is named: {staged:?}"); } +/// the runtime's helpers are named ``, at the line of `_by_runtime.py` they start +/// on and by their own names, whether the transpiler wrote them into a module at a root or they +/// ran from the copy of the runtime beside a package's modules: one helper is one function +/// wherever it ran from, and none is named as line 0 of the module it was written into, which a +/// viewer opens at the module's first line +#[test] +fn run_profile_names_the_runtime_s_helpers_by_the_runtime() { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("pkg"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write( + package.join("util.by"), + "def twice(x: int | None) -> int:\n return x!! * 2\n", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "from pkg.util import twice\n\n\ndef first(xs: list[int] | None) -> int:\n return xs!![0]\n\n\nprint(twice(first([3])))\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + let main = project.join("main.by").display().to_string(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", &python, "--profile", "run.pstat", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "by run --profile failed:\n{stderr}" + ); + // python writes its output in text mode, which ends a line with `\r\n` on windows + assert_eq!( + String::from_utf8_lossy(&output.stdout).replace("\r\n", "\n"), + "6\n", + "{stderr}" + ); + let functions = profiled_functions(&python, &dir.path().join("run.pstat")); + + let runtime = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../by_transforms/src/runtime/_by_runtime.py"), + ) + .expect("the runtime's source"); + let runtime: Vec<&str> = runtime.lines().collect(); + let helper = |name: &str| { + let found: Vec<&Profiled> = functions + .iter() + .filter(|function| function.file == "" && function.name == name) + .collect(); + found + }; + + // written into `main.py`, and in `pkg/_by_runtime.py`: one function + let unwrap = helper("_force_unwrap"); + assert_eq!(unwrap.len(), 1, "{functions:#?}"); + let unwrap = unwrap[0]; + assert!( + runtime[unwrap.line as usize - 1].starts_with("def _force_unwrap("), + "line {} of the runtime is {:?}", + unwrap.line, + runtime[unwrap.line as usize - 1] + ); + let callers: Vec<&str> = unwrap + .callers + .iter() + .map(|(_, _, name)| name.as_str()) + .collect(); + assert!( + callers.contains(&"first") && callers.contains(&"twice"), + "{callers:?}" + ); + + // two methods of a helper written into `main.py` are two functions, each at its own line + for (name, text) in [ + ("__init__", " def __init__(self, namespace, names):"), + ("__getattr__", " def __getattr__(self, name):"), + ] { + assert!( + helper(name) + .iter() + .any(|function| runtime[function.line as usize - 1] == text), + "no {name} at {text:?}: {functions:#?}" + ); + } + + let at_line_0: Vec<&Profiled> = functions + .iter() + .filter(|function| { + function.file == main && function.line == 0 && function.name != "" + }) + .collect(); + assert!(at_line_0.is_empty(), "{at_line_0:#?}"); +} + /// a test runner compiles a test module itself, so the module's own top level runs /// as the staged `.py`, at the generated python's lines, and is still named by the /// `.by` diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 390af0bb80..c16ac07189 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -137,8 +137,18 @@ that ran. a viewer's way from a function to its source then leads to the file it was written in. that holds for a module a test runner compiled itself too, and two functions that turn out to be one — the same name at the same `.by` line — are added up as one, as `pstats` adds two profiles. a function the transpiler -wrote on its own, such as a helper pasted into a module, has no `.by` line, and -is named by its module's `.by` at line 0 +wrote on its own has no `.by` line, and is named by its module's `.by` at line 0 + +the runtime's helpers — what the emitted python calls for `!!`, a lazy import or +a soundness check — are named ``, at the line of `_by_runtime.py` +they start on, as `by build` writes that file, and by their own names: +`:103(_force_unwrap)`. one helper is then one function whether the +transpiler wrote it into a module or it ran from the copy of the runtime beside +a package's modules, and two helpers that share a name, such as two `__init__` +methods, stay two. like python's own ``, the name is no file: a +viewer's way to the source opens nothing, where line 0 of the module the helper +was written into would open that module at its first line, as PyCharm's viewer +opens a file it finds at the line it is given ```console $ by run --profile run.pstat main From 1f562d9aee5810c16c3ec0ab08e074d255fc7b2e Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:17:13 +1000 Subject: [PATCH 094/110] start a by repl session in a module of the project with --module, as by run names one: it runs first as __main__ with by run's __file__, package and spec, and its source is the session's first, held as the text of the module's own file so each input is checked and transpiled as the module's continuation, its names defined, its let read-only and its relative imports resolved, for an editor's run with console --- crates/by_stage/src/project.rs | 4 +- crates/by_stage/src/session.rs | 52 ++-- crates/ty/docs/cli.md | 6 +- crates/ty/docs/environment.md | 8 +- crates/ty/src/args.rs | 31 ++- crates/ty/src/by_commands.rs | 126 ++++++--- crates/ty/src/by_repl.rs | 445 +++++++++++++++++++++--------- crates/ty/src/lib.rs | 2 + crates/ty/tests/by_e2e.rs | 165 ++++++++++- crates/ty_static/src/env_vars.rs | 8 +- docs/basedpython/cli-reference.md | 5 +- docs/basedpython/features/repl.md | 47 +++- 12 files changed, 669 insertions(+), 230 deletions(-) diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index e15f1b39fe..fb0b8f519c 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -133,9 +133,9 @@ impl Rebuilder { } } - /// The project's db built again, and not frozen: what a caller holds that goes on + /// the project's db built again, and not frozen: what a caller holds that goes on /// changing an input after the build, as a `by repl` session changes its module with - /// every input. + /// every input pub fn project_database(&self) -> ProjectDatabase { let mut db = ProjectDatabase::use_defaults(self.metadata.clone(), OsSystem::new(&self.root)); diff --git a/crates/by_stage/src/session.rs b/crates/by_stage/src/session.rs index 5cf483fac5..cca8584935 100644 --- a/crates/by_stage/src/session.rs +++ b/crates/by_stage/src/session.rs @@ -2,9 +2,9 @@ //! //! a session is a module whose source is the inputs that ran, in order, and each new input is //! checked and transpiled as that module's continuation (see `docs/basedpython/features/repl.md`). -//! What is here is the part of that which is a question about source text: whether what has been +//! what is here is the part of that which is a question about source text: whether what has been //! typed is a whole input yet, which the command and the language server both answer, and which -//! of the python the module comes to is the new input's to run. +//! of the python the module comes to is the new input's to run use std::collections::HashSet; @@ -123,12 +123,12 @@ fn ends_with_blank_line(text: &str) -> bool { last_line.trim().is_empty() && before_last_newline.contains('\n') } -/// The inputs of a session that ran, as the source of the one module they make. +/// the inputs of a session that ran, as the source of the one module they make /// -/// Each new input is checked and transpiled as this module's continuation, by +/// each new input is checked and transpiled as this module's continuation, by /// [`Session::with_input`]; [`Session::select`] then picks out of the python the module came /// to the statements that are the input's to run, and [`Session::commit`] makes the input part -/// of the session once it is handed over to run. +/// of the session once it is handed over to run #[derive(Debug, Default)] pub struct Session { source: String, @@ -148,13 +148,13 @@ struct RanInput { line_count: u32, } -/// Where an input sits in the module made of a session and that input. +/// where an input sits in the module made of a session and that input #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct InputLines { /// the module line, 0-based, the input's first line is - pub first: u32, + first: u32, /// how many lines the input is - pub count: u32, + count: u32, } impl InputLines { @@ -166,7 +166,7 @@ impl InputLines { } } -/// What of the python a session and its new input came to runs for the input. +/// what of the python a session and its new input came to runs for the input #[derive(Debug, PartialEq, Eq)] pub struct Selection { /// the top-level statements of the python that run, by their index among them, in order @@ -178,7 +178,7 @@ pub struct Selection { texts: Vec, } -/// Why the python written for an input cannot run. +/// why the python written for an input cannot run #[derive(Debug, PartialEq, Eq)] pub enum Refusal { /// the python written for an earlier input, which has already run, came out differently @@ -191,14 +191,14 @@ pub enum Refusal { } impl Session { - /// The module made of the session's inputs: the session's source. + /// the module made of the session's inputs: the session's source pub fn source(&self) -> &str { &self.source } - /// The source of the module made of the session and `input` after it, and where in it - /// the input is. An input is a whole number of lines, so one that does not end in a line - /// ending is given one. + /// the source of the module made of the session and `input` after it, and where in it + /// the input is. an input is a whole number of lines, so one that does not end in a line + /// ending is given one pub fn with_input(&self, input: &str) -> (String, InputLines) { let mut source = self.source.clone(); let first = line_count(&source); @@ -210,25 +210,25 @@ impl Session { (source, InputLines { first, count }) } - /// The input of the session that holds `line` of its source, 0-based, as the number it is + /// the input of the session that holds `line` of its source, 0-based, as the number it is /// named by and the line of it, 0-based. `None` for a line past the session's, which is - /// the new input's. - pub fn input_at(&self, line: u32) -> Option<(u32, u32)> { + /// the new input's + fn input_at(&self, line: u32) -> Option<(u32, u32)> { self.inputs .iter() .find(|input| line >= input.first_line && line - input.first_line < input.line_count) .map(|input| (input.number, line - input.first_line)) } - /// Which of `python`, what the module made of the session and the input at `input` was + /// which of `python`, what the module made of the session and the input at `input` was /// transpiled to, runs for the input. `line_map` is the transpile's: for each line of - /// `python`, the module line it was written for. + /// `python`, the module line it was written for /// - /// A top-level statement written for the input's lines runs. One written for an earlier + /// a top-level statement written for the input's lines runs. one written for an earlier /// input's has run already, and is skipped — unless it came out differently this time, /// which means the input changed what an earlier input meant after it ran, and refuses - /// the input. One written for no line at all is the transpiler's own, a runtime helper - /// or an import it needs, and runs the first time it appears. + /// the input. one written for no line at all is the transpiler's own, a runtime helper + /// or an import it needs, and runs the first time it appears pub fn select( &self, python: &str, @@ -300,8 +300,8 @@ impl Session { }) } - /// Make `input`, named ``, part of the session, with the python - /// `selection` picked for it as what has run. + /// make `input`, named ``, part of the session, with the python + /// `selection` picked for it as what has run pub fn commit(&mut self, number: u32, input: &str, selection: Selection) { let (source, lines) = self.with_input(input); self.source = source; @@ -314,14 +314,14 @@ impl Session { } } -/// How many lines `text` is, counting a last line with no line ending. +/// how many lines `text` is, counting a last line with no line ending fn line_count(text: &str) -> u32 { let endings = text.bytes().filter(|byte| *byte == b'\n').count(); let unended = usize::from(!text.is_empty() && !text.ends_with('\n')); u32::try_from(endings + unended).unwrap_or(u32::MAX) } -/// The offset each line of `text` starts at. +/// the offset each line of `text` starts at fn line_starts(text: &str) -> Vec { std::iter::once(0) .chain(text.match_indices('\n').map(|(at, _)| at + 1)) diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index c93dc420a6..b3de89efee 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -288,10 +288,12 @@ by repl [OPTIONS]

Options

-
--events port

Tell an editor what the session reads standard input for, on a connection of its own.

-

by repl connects to PORT on the loopback interface and writes a line of JSON there each time it waits for an input, the program reads standard input, or an input finishes: a console that talks to it through pipes cannot tell otherwise whether a line it sends is an input or a line for the program. The first line carries the token BY_REPL_EVENTS_TOKEN holds, for the editor to know the connection is the session's.

+
--events port

tell an editor what the session reads standard input for, on a connection of its own

+

by repl connects to PORT on the loopback interface and writes a line of JSON there each time it waits for an input, the program reads standard input, or an input finishes: a console that talks to it through pipes cannot tell otherwise whether a line it sends is an input or a line for the program. the first line carries the token BY_REPL_EVENTS_TOKEN holds, for the editor to know the connection is the session's

--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the session's python must run on [default: the version of the interpreter that will run it]

+
--module module

start in MODULE, a module of the project named as by run takes one: it runs first, as __main__, and the session goes on from it with its names

+

the module's source is the session's first, and each input is checked and transpiled as its continuation, as the module itself, so a relative import resolves in an input as it does in the module. its def main is a function like any other in a session: nothing calls it

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

--python, --venv path

the interpreter to run on, or the environment holding it, as by run takes it

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 9088c47bb0..1400300c2c 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -15,11 +15,11 @@ Equivalent to the `--no-server` command-line argument. ### `BY_REPL_EVENTS_TOKEN` -The token `by repl --events` sends first on the connection it makes to an editor. +the token `by repl --events` sends first on the connection it makes to an editor -An editor listening for a session's events sets this to a value nothing else can guess, -and takes the connection whose first line carries it as the session's. The program the -session runs does not see it. +an editor listening for a session's events sets this to a value nothing else can guess, +and takes the connection whose first line carries it as the session's. the program the +session runs does not see it ### `BY_TEST_REPORTER` diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 56ba3a1afe..eda8ca4043 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -146,37 +146,46 @@ pub(crate) enum Command { profile: Option, }, - /// Start an interactive session in basedpython. + /// start an interactive session in basedpython /// - /// Each input is checked and transpiled as the continuation of the module made of the + /// each input is checked and transpiled as the continuation of the module made of the /// inputs that ran before it, and runs in one python process that lives as long as the - /// session. The project is staged as `by run` stages it, so its modules import as they do - /// in a program. + /// session. the project is staged as `by run` stages it, so its modules import as they do + /// in a program Repl { /// minimum Python version the session's python must run on /// [default: the version of the interpreter that will run it] #[arg(long, value_name = "VERSION")] min_version: Option, - /// The interpreter to run on, or the environment holding it, as `by run` takes it. + /// the interpreter to run on, or the environment holding it, as `by run` takes it #[arg(long, value_name = "PATH", alias = "venv")] python: Option, #[command(flatten)] lowering: LoweringArgs, - /// Keep PATH holding the session's source: the inputs that ran, in order. + /// keep PATH holding the session's source: the inputs that ran, in order /// - /// For an editor's console, whose completion is asked of the module this file holds - /// followed by the input being typed. + /// for an editor's console, whose completion is asked of the module this file holds + /// followed by the input being typed #[arg(long, value_name = "PATH")] session_file: Option, - /// Tell an editor what the session reads standard input for, on a connection of its own. + /// tell an editor what the session reads standard input for, on a connection of its own /// /// `by repl` connects to PORT on the loopback interface and writes a line of JSON there /// each time it waits for an input, the program reads standard input, or an input /// finishes: a console that talks to it through pipes cannot tell otherwise whether a - /// line it sends is an input or a line for the program. The first line carries the token - /// `BY_REPL_EVENTS_TOKEN` holds, for the editor to know the connection is the session's. + /// line it sends is an input or a line for the program. the first line carries the token + /// `BY_REPL_EVENTS_TOKEN` holds, for the editor to know the connection is the session's #[arg(long, value_name = "PORT")] events: Option, + /// start in MODULE, a module of the project named as `by run` takes one: it runs + /// first, as `__main__`, and the session goes on from it with its names + /// + /// the module's source is the session's first, and each input is checked and + /// transpiled as its continuation, as the module itself, so a relative import resolves + /// in an input as it does in the module. its `def main` is a function like any other in + /// a session: nothing calls it + #[arg(long, value_name = "MODULE")] + module: Option, }, /// Start a new project. diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 39fd6a4396..49f008181e 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -148,8 +148,8 @@ pub(crate) struct RunFlags<'a> { pub(crate) profile: Option<&'a Path>, } -/// What `by run` and `by repl` both start from: the project checked, transpiled and staged -/// into a temporary tree, and the interpreter that runs out of it. +/// what `by run` and `by repl` both start from: the project checked, transpiled and staged +/// into a temporary tree, and the interpreter that runs out of it pub(crate) struct StagedRun { /// where the command was invoked, which is where the program runs pub(crate) cwd: PathBuf, @@ -168,7 +168,7 @@ pub(crate) struct StagedRun { module: Option, } -/// What a staged tree is for. +/// what a staged tree is for #[derive(Clone, Copy)] pub(crate) enum RunTarget<'a> { /// `by run`'s program: the module named, or the project's entry point @@ -177,9 +177,9 @@ pub(crate) enum RunTarget<'a> { Session, } -/// Check, transpile and stage the project around the working directory for `target`, on the +/// check, transpile and stage the project around the working directory for `target`, on the /// interpreter `python_flag` names or discovery finds. `Err` is the exit status of a project -/// that cannot run: one that does not check, having said why. +/// that cannot run: one that does not check, having said why #[allow(clippy::print_stderr)] pub(crate) fn stage_run( min_version: Option<&str>, @@ -557,6 +557,16 @@ pub(crate) fn cmd_run( std::process::exit(code); } +/// the status `by` exits with for a program that ended with `status`: the program's own, or +/// for one a signal ended, 128 and the signal's number, as a shell reports it +pub(crate) fn exit_code(status: std::process::ExitStatus) -> i32 { + #[cfg(unix)] + if let Some(signal) = std::os::unix::process::ExitStatusExt::signal(&status) { + return 128 + signal; + } + status.code().unwrap_or(1) +} + /// The source roots a distribution's packages come from. /// /// The project root is always a module root — it is what lets `tests/` and a @@ -2068,7 +2078,9 @@ class _Session: # a thread of the program's can read standard input while the main thread talks to # `by repl`, and each message goes whole self._lock = threading.Lock() - self.tell({"token": token}) + # alone on its line, which is all `by repl` reads of a connection before it is the token + with _holding_ctrl_c(): + self._socket.sendall((token + "\n").encode("ascii")) def _send(self, message): data = (json.dumps(message) + "\n").encode("utf-8") @@ -2231,6 +2243,44 @@ def _read_input(session): return text +def _codes(reply, text, filename): + """The code objects of the python `by repl` wrote for `text`, in `reply`, which run for + it, named `filename` and at the lines of `text`.""" + generated = reply["lines"] + widths = [len(line.rstrip("\r\n")) for line in text.splitlines()] + + def by_line(line): + mapped = generated[line - 1] if line is not None and 0 < line <= len(generated) else None + return 0 if mapped is None else mapped + 1 + + def width(line): + return widths[line - 1] if 0 < line <= len(widths) else 0 + + module = ast.parse(reply["python"]) + body = [module.body[index] for index in reply["statements"]] + if not body: + return [] + if reply["display"]: + # compiled as python's repl compiles an input, each expression statement prints its value + code = compile(ast.Interactive(body=body), filename, "single", dont_inherit=True) + else: + code = compile(ast.Module(body=body, type_ignores=[]), filename, "exec", dont_inherit=True) + return [_by_runner._relined(code, filename, by_line, width)] + + +def _execute(session, namespace, number, codes): + """Run `codes`, input `number`'s, which is part of the session from the moment it starts, + whatever it raises.""" + try: + session.tell({"ran": number}) + for code in codes: + exec(code, namespace) + except SystemExit: + raise + except BaseException: + sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) + + def _run(session, namespace, text): """Run `text`, an input: the number it was given, and whether it ran, which makes it part of the session whatever it raised.""" @@ -2245,36 +2295,10 @@ def _run(session, namespace, text): # refused: `by repl` has said why return number, False filename = f"" - lines = text.splitlines(True) # what a traceback quotes for a line of the input, for as long as the session lasts - linecache.cache[filename] = (len(text), None, lines, filename) - generated = reply["lines"] - widths = [len(line.rstrip("\r\n")) for line in lines] - - def by_line(line): - mapped = generated[line - 1] if line is not None and 0 < line <= len(generated) else None - return 0 if mapped is None else mapped + 1 - - def width(line): - return widths[line - 1] if 0 < line <= len(widths) else 0 - - module = ast.parse(reply["python"]) - body = [module.body[index] for index in reply["statements"]] - shown = None - if reply["display"] and body and isinstance(body[-1], ast.Expr): - shown = body.pop() + linecache.cache[filename] = (len(text), None, text.splitlines(True), filename) try: - codes = [] - if body: - codes.append( - compile(ast.Module(body=body, type_ignores=[]), filename, "exec", dont_inherit=True) - ) - if shown is not None: - # as python's repl compiles an input, which prints an expression's value - codes.append( - compile(ast.Interactive(body=[shown]), filename, "single", dont_inherit=True) - ) - codes = [_by_runner._relined(code, filename, by_line, width) for code in codes] + codes = _codes(reply, text, filename) except Exception: sys.stderr.write("error: the python this input was transpiled to does not compile:\n") sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) @@ -2282,15 +2306,27 @@ def _run(session, namespace, text): except KeyboardInterrupt: sys.stderr.write("\nKeyboardInterrupt\n") return number, False + _execute(session, namespace, number, codes) + return number, True + + +def _start_in(session, main_module, start): + """Run the module the session starts in, which `start` is what `by repl` wrote for, as + `__main__`: in the session's namespace, with the module's file, package and spec, as `by + run` runs it, so its relative imports resolve and a spawned process finds it.""" + by_path = start["file"] try: - session.tell({"ran": number}) - for code in codes: - exec(code, namespace) - except SystemExit: - raise - except BaseException: + _, spec = _by_runner._entry_spec(start["module"]) + codes = _codes(start, start["text"], by_path) + except Exception: sys.stderr.write(_by_runner.format_exception(*sys.exc_info())) - return number, True + return + sys.argv = [by_path] + main_module.__file__ = by_path + main_module.__loader__ = spec.loader + main_module.__package__ = spec.parent + main_module.__spec__ = spec + _execute(session, main_module.__dict__, start["input"], codes) def main(): @@ -2303,6 +2339,14 @@ def main(): namespace = main_module.__dict__ if session.events: sys.stdin = _ProgramStdin(_STDIN, session) + try: + start = session.ask({"start": True}) + if "module" in start: + _start_in(session, main_module, start) + except KeyboardInterrupt: + sys.stderr.write("\nKeyboardInterrupt\n") + except EOFError: + return if _INTERACTIVE: try: import readline # noqa: F401 — line editing for `input` diff --git a/crates/ty/src/by_repl.rs b/crates/ty/src/by_repl.rs index a7cf6839bb..e7e7a11173 100644 --- a/crates/ty/src/by_repl.rs +++ b/crates/ty/src/by_repl.rs @@ -1,26 +1,28 @@ -//! `by repl`: basedpython typed an input at a time, run in one python process. +//! `by repl`: basedpython typed an input at a time, run in one python process //! -//! The project is staged as `by run` stages it, and the interpreter runs `_by_repl.py` in that -//! tree, which reads the inputs, runs them and prints what python's repl prints. Each input is -//! a question for this process first: whether it is whole yet, and what python it runs as. The +//! the project is staged as `by run` stages it, and the interpreter runs `_by_repl.py` in that +//! tree, which reads the inputs, runs them and prints what python's repl prints. each input is +//! a question for this process first: whether it is whole yet, and what python it runs as. the //! session is one module whose source is the inputs that ran (see -//! `docs/basedpython/features/repl.md`), held here as a file the project's db checks and -//! transpiles like any other, and [`by_stage::session`] picks the input's statements out of what -//! the module comes to. +//! `docs/basedpython/features/repl.md`), held here as the text of a file the project's db checks +//! and transpiles like any other, and [`by_stage::session`] picks the input's statements out of +//! what the module comes to. the file is one of its own outside the project, or, for a session +//! started in a module of the project with `--module`, that module's: its source is the +//! session's first, and the inputs follow it //! -//! The two halves talk over a socket on the loopback interface rather than over the +//! the two halves talk over a socket on the loopback interface rather than over the //! interpreter's standard streams, because those are the program's: `input()` reads standard -//! input, and the program's output is its own. The interpreter is told the port and a token in +//! input, and the program's output is its own. the interpreter is told the port and a token in //! its environment, and the first thing it sends is the token, so nothing else on the machine -//! that finds the port is answered. +//! that finds the port is answered //! -//! For the same reason an editor that asks with `--events` is told what standard input is +//! for the same reason an editor that asks with `--events` is told what standard input is //! being read for — an input, or the program — on a connection of its own: the interpreter says -//! so to this process, which passes it on. +//! so to this process, which passes it on use std::fs; use std::hash::{BuildHasher, RandomState}; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::net::{TcpListener, TcpStream}; use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; @@ -37,12 +39,15 @@ use ruff_db::diagnostic::{ }; use ruff_db::files::{File, system_path_to_file}; use ruff_db::panic::PanicActivity; +use ruff_db::source::source_text; use ruff_db::system::SystemPath; +use ruff_diagnostics::SourceMap; use ruff_notebook::NotebookIndex; -use ruff_python_ast::{PySourceType, Stmt}; +use ruff_python_ast::PySourceType; use ruff_python_parser::parse_unchecked_source; use ruff_source_file::{SourceFile, SourceFileBuilder}; use ruff_text_size::{TextRange, TextSize}; +use salsa::Setter as _; use serde_json::{Value, json}; use ty_project::{Db as _, ProjectDatabase}; @@ -52,11 +57,11 @@ use crate::ExitStatus; use crate::args::LoweringArgs; use crate::by_commands::{BY_REPL_FILENAME, RunTarget, StagedRun, stage_run}; -/// How long the interpreter has to connect once it has started, and to send its token once it -/// has connected: it does both before it runs anything of the session's. +/// how long the interpreter has to connect and send its token once it has started: it does both +/// before it runs anything of the session's const CONNECT_TIMEOUT: Duration = Duration::from_secs(60); -/// What `by repl` is given beyond what it stages the project with. +/// what `by repl` is given beyond what it stages the project with pub(crate) struct ReplFlags<'a> { /// `--python`: the interpreter, or the environment holding it pub(crate) python: Option<&'a Path>, @@ -65,9 +70,11 @@ pub(crate) struct ReplFlags<'a> { /// `--events`: the port on the loopback interface an editor listens for the session's /// events on pub(crate) events: Option, + /// `--module`: the module of the project the session starts in + pub(crate) module: Option<&'a str>, } -#[allow(clippy::exit, clippy::print_stderr)] +#[expect(clippy::exit, clippy::print_stderr)] pub(crate) fn cmd_repl( min_version: Option<&str>, lowering: &LoweringArgs, @@ -96,7 +103,11 @@ pub(crate) fn cmd_repl( fs::write(path, "").with_context(|| format!("could not write {}", path.display()))?; } let events = flags.events.map(connect_events).transpose()?; - let mut repl = Repl::new(&rebuilder, config, session_file, events)?; + let mut repl = Repl::new(rebuilder, config, flags.module, session_file, events)?; + // a session that cannot start in the module it was asked to has said why, and does not start + if !repl.prepare_start()? { + return Ok(ExitStatus::Failure); + } let listener = TcpListener::bind(("127.0.0.1", 0)).context("could not listen for the session")?; @@ -107,21 +118,24 @@ pub(crate) fn cmd_repl( .arg(tree.path().join(BY_REPL_FILENAME)) .env("BY_REPL_PORT", port.to_string()) .env("BY_REPL_TOKEN", &token) + // set either way, so a value this process was started with is never the session's + .env( + "BY_REPL_EVENTS", + if flags.events.is_some() { "1" } else { "0" }, + ) // the editor's, not the program's .env_remove(EnvVars::BY_REPL_EVENTS_TOKEN) .current_dir(&cwd); - if flags.events.is_some() { - command.env("BY_REPL_EVENTS", "1"); - } + // ctrl-c is the interpreter's to answer, with a `KeyboardInterrupt` in the input it is + // running; this process lives as long as the session does, from before the interpreter + // can be sent one + ctrlc::set_handler(|| {}).context("could not hand ctrl-c to the session")?; let mut child = command.spawn().with_context(|| { format!( "could not run `{}`, the interpreter from {}", interpreter.path, interpreter.origin ) })?; - // ctrl-c is the interpreter's to answer, with a `KeyboardInterrupt` in the input it is - // running; this process lives as long as the session does - ctrlc::set_handler(|| {}).context("could not hand ctrl-c to the session")?; let served = accept(&listener, &token, &mut child).and_then(|reader| match reader { Some(reader) => repl.serve(reader), @@ -138,11 +152,11 @@ pub(crate) fn cmd_repl( drop(repl); // `process::exit` skips destructors, and the tree is a temporary directory drop(tree); - std::process::exit(status.code().unwrap_or(1)); + std::process::exit(crate::by_commands::exit_code(status)); } -/// The editor's connection for the session's events, to `port` on the loopback interface, with -/// the token it set in `BY_REPL_EVENTS_TOKEN` sent on it. +/// the editor's connection for the session's events, to `port` on the loopback interface, with +/// the token it set in `BY_REPL_EVENTS_TOKEN` sent on it fn connect_events(port: u16) -> anyhow::Result { let mut stream = TcpStream::connect(("127.0.0.1", port)) .with_context(|| format!("could not connect to the editor's events port, {port}"))?; @@ -152,7 +166,7 @@ fn connect_events(port: u16) -> anyhow::Result { Ok(stream) } -/// A token nothing else can guess, from the process's own random hasher keys. +/// a token nothing else can guess, from the process's own random hasher keys fn token() -> String { let state = RandomState::new(); format!( @@ -162,61 +176,100 @@ fn token() -> String { ) } -/// The interpreter's connection, read through the one buffer every message after the token +/// the interpreter's connection, read through the one buffer every message after the token /// is read through, once it has sent `token`. `None` when the interpreter ended without -/// making one. +/// making one +/// +/// anything on the machine can connect to the port, so no connection holds up another: each +/// is read as its bytes arrive, only as far as the token goes, and one that sends anything but +/// the token is dropped fn accept( listener: &TcpListener, token: &str, child: &mut std::process::Child, ) -> anyhow::Result>> { listener.set_nonblocking(true)?; + let expected = format!("{token}\n").into_bytes(); + // each connection that has yet to send the whole token, with what it has sent of it + let mut connections: Vec<(TcpStream, Vec)> = Vec::new(); let started = Instant::now(); loop { match listener.accept() { Ok((stream, _)) => { - stream.set_nonblocking(false)?; - stream.set_read_timeout(Some(CONNECT_TIMEOUT))?; - let mut reader = BufReader::new(stream); - let mut first = String::new(); - // a connection that is not the interpreter's is dropped, and the wait goes on - if reader.read_line(&mut first).is_ok() - && serde_json::from_str::(&first) - .is_ok_and(|message| message["token"] == token) + stream.set_nonblocking(true)?; + connections.push((stream, Vec::new())); + continue; + } + Err(error) if is_transient(&error) => {} + Err(error) => return Err(error).context("could not accept the session's interpreter"), + } + let mut index = 0; + while index < connections.len() { + let (stream, sent) = &mut connections[index]; + // never past the token, so what the interpreter sends after it stays for the reader + let mut buffer = vec![0; expected.len() - sent.len()]; + match stream.read(&mut buffer) { + Ok(read) + if read > 0 && expected.starts_with(&[&sent[..], &buffer[..read]].concat()) => { - reader.get_ref().set_read_timeout(None)?; - return Ok(Some(reader)); + sent.extend_from_slice(&buffer[..read]); + if sent.len() == expected.len() { + let (stream, _) = connections.swap_remove(index); + stream.set_nonblocking(false)?; + return Ok(Some(BufReader::new(stream))); + } } - } - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - if child.try_wait()?.is_some() { - return Ok(None); + Err(error) if is_transient(&error) => {} + // closed, or sent something that is not the token + _ => { + connections.swap_remove(index); + continue; } - anyhow::ensure!( - started.elapsed() < CONNECT_TIMEOUT, - "the interpreter did not connect to the session" - ); - std::thread::sleep(Duration::from_millis(10)); } - Err(error) => return Err(error).context("could not accept the session's interpreter"), + index += 1; + } + if child.try_wait()?.is_some() { + return Ok(None); } + anyhow::ensure!( + started.elapsed() < CONNECT_TIMEOUT, + "the interpreter did not connect to the session" + ); + std::thread::sleep(Duration::from_millis(10)); } } -/// The session as this process holds it: the module, the db that checks it, and what each -/// input that ran was. +/// whether `error`, from a socket that does not block, only says to try again +fn is_transient(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) +} + +/// the session as this process holds it: the module, the db that checks it, and what each +/// input that ran was struct Repl { db: ProjectDatabase, - /// the project, with the module's directory one of its first-party roots + /// the project, with the module's directory one of its first-party roots when it is a file + /// of its own rebuilder: Rebuilder, /// what the module is transpiled for: the project's run configuration, as a session config: Config, session: Session, - /// the file the module is checked as, outside the project + /// the file the module is checked as, whose text the db holds as the session's source: one + /// of its own outside the project, or the module of the project the session started in module: File, - module_path: PathBuf, - /// keeps the module's directory as long as the session - _module_dir: tempfile::TempDir, + /// keeps the directory of a module of its own as long as the session + _module_dir: Option, + /// `--module`: the module of the project the session started in + started_in: Option, + /// how much of the session's source is the module it started in, which a diagnostic points + /// into as the module's file rather than as an input + prelude: TextSize, + /// what the interpreter runs before the first input: the module the session starts in, as + /// `run` answers for an input + start: Option, /// each input that ran, for a diagnostic of a later one to point into inputs: Vec, /// the input handed over to run, which is part of the session once the interpreter says it @@ -231,6 +284,16 @@ struct Repl { events: Option, } +/// the module of the project a session starts in +struct StartModule { + /// the name it was given, as `by run` takes one + name: String, + /// its file + path: PathBuf, + /// its source, as the session starts with it + text: String, +} + struct RanInput { /// where in the session's source the input starts start: TextSize, @@ -245,25 +308,52 @@ struct Pending { impl Repl { fn new( - rebuilder: &Rebuilder, + rebuilder: Rebuilder, config: Config, + start_in: Option<&str>, session_file: Option, events: Option, ) -> anyhow::Result { - let module_dir = tempfile::TempDir::new().context("failed to create temp directory")?; - // a module name nothing a project imports is called, since the directory is searched - // for first-party modules after the project's own roots - let module_path = module_dir.path().join("__by_session__.by"); - fs::write(&module_path, "")?; - let directory = SystemPath::from_std_path(module_dir.path()) - .with_context(|| format!("non-utf8 path: {}", module_dir.path().display()))?; - let rebuilder = rebuilder.with_first_party_root(directory); - let mut db = rebuilder.project_database(); - let system_path = SystemPath::from_std_path(&module_path) - .with_context(|| format!("non-utf8 path: {}", module_path.display()))?; - let module = system_path_to_file(&db, system_path) - .with_context(|| format!("could not read {}", module_path.display()))?; - // outside the project, so checked only as a file the project has open + let (rebuilder, mut db, module, module_dir, started_in) = match start_in { + None => { + let module_dir = + tempfile::TempDir::new().context("failed to create temp directory")?; + // a module name nothing a project imports is called, since the directory is + // searched for first-party modules after the project's own roots + let module_path = module_dir.path().join("__by_session__.by"); + fs::write(&module_path, "")?; + let directory = SystemPath::from_std_path(module_dir.path()) + .with_context(|| format!("non-utf8 path: {}", module_dir.path().display()))?; + let rebuilder = rebuilder.with_first_party_root(directory); + let db = rebuilder.project_database(); + let system_path = SystemPath::from_std_path(&module_path) + .with_context(|| format!("non-utf8 path: {}", module_path.display()))?; + let module = system_path_to_file(&db, system_path) + .with_context(|| format!("could not read {}", module_path.display()))?; + (rebuilder, db, module, Some(module_dir), None) + } + Some(name) => { + let db = rebuilder.project_database(); + // the module `by run` would run: the session is its continuation, checked as + // the module itself, so a relative import resolves in it and in an input alike + let module = by_stage::run_module::module_file(&db, name) + .with_context(|| format!("no module named `{name}` in the project"))?; + let path = module + .path(&db) + .as_system_path() + .with_context(|| format!("module `{name}` is no file"))? + .as_std_path() + .to_path_buf(); + let text = source_text(&db, module).as_str().to_owned(); + let started_in = StartModule { + name: name.to_owned(), + path, + text, + }; + (rebuilder, db, module, None, Some(started_in)) + } + }; + // one outside the project is checked only as a file the project has open db.project().open_file(&mut db, module); Ok(Self { db, @@ -279,8 +369,10 @@ impl Repl { }, session: Session::default(), module, - module_path, _module_dir: module_dir, + started_in, + prelude: TextSize::default(), + start: None, inputs: Vec::new(), pending: None, next: 1, @@ -289,7 +381,7 @@ impl Repl { }) } - /// Answer the interpreter until it hangs up. + /// answer the interpreter until it hangs up fn serve(&mut self, mut reader: BufReader) -> anyhow::Result<()> { let mut writer = reader.get_ref().try_clone()?; let mut line = String::new(); @@ -301,6 +393,8 @@ impl Repl { let message: Value = serde_json::from_str(&line).context("an unreadable message")?; let reply = if let Some(text) = message["status"].as_str() { Some(json!({ "status": input_status(text) })) + } else if message.get("start").is_some() { + Some(self.start.take().unwrap_or_else(|| json!({}))) } else if let Some(text) = message["run"].as_str() { Some(self.run(text)?) } else if let Some(number) = message["ran"].as_u64() { @@ -319,13 +413,48 @@ impl Repl { } } - /// What `text` runs as, or what is in the way of it running, said on standard error. + /// make ready what the interpreter runs before the first input: the module the session + /// starts in, if it starts in one. `false`, having said why on standard error, when the + /// module cannot run as the session's start + fn prepare_start(&mut self) -> anyhow::Result { + let Some(StartModule { name, path, text }) = &self.started_in else { + return Ok(true); + }; + let (name, path, text) = (name.clone(), path.clone(), text.clone()); + self.prelude = TextSize::try_from(text.len())?; + let file = SourceFileBuilder::new(path.display().to_string(), &*text).finish(); + let Some(mut reply) = self.prepare(0, &text, &file)? else { + return Ok(false); + }; + // a module prints no values, as a program does not + reply["display"] = json!(false); + reply["module"] = json!(name); + reply["file"] = json!(path); + reply["text"] = json!(text); + self.start = Some(reply); + Ok(true) + } + + /// what `text` runs as, or what is in the way of it running, said on standard error fn run(&mut self, text: &str) -> anyhow::Result { let number = self.next; self.next += 1; - self.pending = None; - let refused = json!({ "input": number }); let input_file = SourceFileBuilder::new(format!(""), text).finish(); + Ok(self + .prepare(number, text, &input_file)? + .unwrap_or_else(|| json!({ "input": number }))) + } + + /// what `text`, input `number`, runs as, and the run handed over for the interpreter to + /// start; `None` when something is in the way of it running, having said what on standard + /// error. `input_file` is the input as a diagnostic quotes it + fn prepare( + &mut self, + number: u32, + text: &str, + input_file: &SourceFile, + ) -> anyhow::Result> { + self.pending = None; // the input's own grammar first: a line that continues an earlier input, `else:` after // an `if`, would otherwise parse as part of it @@ -339,27 +468,25 @@ impl Repl { }) .collect(); self.render(&diagnostics); - return Ok(refused); + return Ok(None); } - let display = matches!(parsed.suite().last(), Some(Stmt::Expr(_))); let (source, lines) = self.session.with_input(text); - fs::write(&self.module_path, &source) - .with_context(|| format!("could not write {}", self.module_path.display()))?; - File::sync_path( - &mut self.db, - SystemPath::from_std_path(&self.module_path).context("non-utf8 path")?, - ); + // the module's text is the session's, whatever its file holds + let source = source_text(&self.db, self.module).with_text(source, &SourceMap::default()); + self.module + .set_source_text_override(&mut self.db) + .to(Some(source)); let start = TextSize::try_from(self.session.source().len())?; let errors: Vec = ty_project::check_file_ignoring_check_mode(&self.db, self.module) .into_iter() .filter(|diagnostic| diagnostic.severity() >= Severity::Error) - .map(|diagnostic| self.in_inputs(diagnostic, start, &input_file)) + .map(|diagnostic| self.in_inputs(diagnostic, start, input_file)) .collect(); if !errors.is_empty() { self.render(&errors); - return Ok(refused); + return Ok(None); } let rebuild = || Some(self.rebuilder.rebuild()); @@ -380,20 +507,20 @@ impl Repl { Ok(Ok((python, line_map, _))) => (python, line_map), Ok(Err(error)) => { let diagnostic = by_stage::emit::transpile_failure_diagnostic(self.module, &error); - let diagnostic = self.in_inputs(diagnostic, start, &input_file); + let diagnostic = self.in_inputs(diagnostic, start, input_file); self.render(&[diagnostic]); - return Ok(refused); + return Ok(None); } Err(panic) => { self.render(&[panic]); - return Ok(refused); + return Ok(None); } }; let selection = match self.session.select(&python, &line_map, lines) { Ok(selection) => selection, Err(refusal) => { - report_refusal(&refusal); - return Ok(refused); + self.report_refusal(&refusal); + return Ok(None); } }; let reply = json!({ @@ -401,18 +528,19 @@ impl Repl { "python": python, "statements": selection.statements, "lines": selection.lines, - "display": display, + // as python's repl runs an input, printing the value of each expression statement + "display": true, }); self.pending = Some(Pending { number, text: text.to_owned(), selection, }); - Ok(reply) + Ok(Some(reply)) } - /// The interpreter has started running input `number`, which is part of the session from - /// now on, whatever it raises. + /// the interpreter has started running input `number`, which is part of the session from + /// now on, whatever it raises fn ran(&mut self, number: u64) -> anyhow::Result<()> { let Some(pending) = self.pending.take() else { return Ok(()); @@ -420,10 +548,14 @@ impl Repl { if u64::from(pending.number) != number { return Ok(()); } - let start = TextSize::try_from(self.session.source().len())?; - let file = SourceFileBuilder::new(format!("", pending.number), &*pending.text) - .finish(); - self.inputs.push(RanInput { start, file }); + // the module the session started in is its own file to a diagnostic, not an input + if pending.number != 0 { + let start = TextSize::try_from(self.session.source().len())?; + let file = + SourceFileBuilder::new(format!("", pending.number), &*pending.text) + .finish(); + self.inputs.push(RanInput { start, file }); + } self.session .commit(pending.number, &pending.text, pending.selection); if let Some(path) = &self.session_file { @@ -433,7 +565,7 @@ impl Repl { Ok(()) } - /// Pass `event`, which the interpreter reported, on to the editor listening for it, if one is. + /// pass `event`, which the interpreter reported, on to the editor listening for it, if one is fn event(&mut self, event: &Value) -> anyhow::Result<()> { if let Some(events) = &mut self.events { writeln!(events, "{event}") @@ -444,7 +576,8 @@ impl Repl { } /// `diagnostic` with every place it points to in the session's module moved to the input - /// that place is in, `input` being the one at `start` that has not run yet. + /// that place is in, `input` being the one at `start` that has not run yet. a place in the + /// module the session started in stays where it is, in that module's file fn in_inputs( &self, mut diagnostic: Diagnostic, @@ -478,6 +611,9 @@ impl Repl { match span.file() { UnifiedFile::Ty(file) if *file == module => { let range = span.range().unwrap_or_default(); + if range.start() < self.prelude { + return None; + } let (file, range) = place(range); Some(Span::from(file).with_range(range)) } @@ -499,8 +635,37 @@ impl Repl { diagnostic } - /// Print `diagnostics` on standard error, as `by check` prints them. - #[allow(clippy::print_stderr)] + /// say why an input's python cannot run + #[expect(clippy::print_stderr)] + fn report_refusal(&self, refusal: &Refusal) { + match refusal { + Refusal::ChangesAnEarlierInput { input, line } => eprintln!( + "error: this input changes the python that {}, line {line} was transpiled to, \ + which has run already and cannot run again differently", + self.input_name(*input) + ), + Refusal::JoinsAnEarlierInput { input } => eprintln!( + "error: this input is transpiled into one statement with {}, which has run \ + already", + self.input_name(*input) + ), + Refusal::NotPython(error) => { + eprintln!("error: the python this input was transpiled to does not parse: {error}"); + } + } + } + + /// what input `number` is called: ``, or the file of the module the session + /// started in, which is input 0 + fn input_name(&self, number: u32) -> String { + match &self.started_in { + Some(started_in) if number == 0 => started_in.path.display().to_string(), + _ => format!(""), + } + } + + /// print `diagnostics` on standard error, as `by check` prints them + #[expect(clippy::print_stderr)] fn render(&self, diagnostics: &[Diagnostic]) { let config = DisplayDiagnosticConfig::new("ty") .color(colored::control::SHOULD_COLORIZE.should_colorize()) @@ -513,26 +678,8 @@ impl Repl { } } -/// Say why an input's python cannot run. -#[allow(clippy::print_stderr)] -fn report_refusal(refusal: &Refusal) { - match refusal { - Refusal::ChangesAnEarlierInput { input, line } => eprintln!( - "error: this input changes the python that , line {line} was \ - transpiled to, which has run already and cannot run again differently" - ), - Refusal::JoinsAnEarlierInput { input } => eprintln!( - "error: this input is transpiled into one statement with , \ - which has run already" - ), - Refusal::NotPython(error) => { - eprintln!("error: the python this input was transpiled to does not parse: {error}"); - } - } -} - -/// The project's db as the diagnostics of a session read their files: the project's files -/// through it, and the inputs, which are no file of its, as the text each diagnostic carries. +/// the project's db as the diagnostics of a session read their files: the project's files +/// through it, and the inputs, which are no file of its, as the text each diagnostic carries struct Inputs<'db>(&'db ProjectDatabase); impl FileResolver for Inputs<'_> { @@ -562,3 +709,51 @@ impl FileResolver for Inputs<'_> { FileResolver::current_directory(self.0) } } + +#[cfg(all(test, unix))] +mod tests { + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command}; + use std::time::{Duration, Instant}; + + use super::accept; + + /// a process that outlives the test, standing in for the interpreter + fn interpreter() -> Child { + Command::new("sleep").arg("30").spawn().expect("sleep runs") + } + + /// a connection that sends nothing, or something other than the token, does not hold up + /// the interpreter's, which is taken as soon as its token arrives + #[test] + fn accept_takes_the_interpreter_s_connection_whatever_else_connects() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listens"); + let port = listener.local_addr().expect("a port").port(); + let mut child = interpreter(); + + let _silent = TcpStream::connect(("127.0.0.1", port)).expect("connects"); + let mut endless = TcpStream::connect(("127.0.0.1", port)).expect("connects"); + endless.write_all(&[b'0'; 4096]).expect("sends"); + let mut interpreter_s = TcpStream::connect(("127.0.0.1", port)).expect("connects"); + interpreter_s + .write_all(b"secret\n{\"start\": true}\n") + .expect("sends"); + + let started = Instant::now(); + let reader = accept(&listener, "secret", &mut child) + .expect("accepts") + .expect("a connection"); + assert!(started.elapsed() < Duration::from_secs(5)); + // what follows the token is left for the session to read + let mut rest = String::new(); + reader + .into_inner() + .take(16) + .read_to_string(&mut rest) + .expect("reads"); + assert_eq!(rest, "{\"start\": true}\n"); + child.kill().expect("ends"); + child.wait().expect("ends"); + } +} diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index f1a4c796b6..17c9dc06c0 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -149,6 +149,7 @@ fn run_command(command: Command) -> anyhow::Result { lowering, session_file, events, + module, } => by_repl::cmd_repl( min_version.as_deref(), &lowering, @@ -156,6 +157,7 @@ fn run_command(command: Command) -> anyhow::Result { python: python.as_deref(), session_file: session_file.as_deref(), events, + module: module.as_deref(), }, ), Command::Init { diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index f5747322b4..fa53e02925 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1997,8 +1997,8 @@ fn run_profile_maps_a_module_a_test_runner_compiled() { ); } -/// What a `by repl` session in `dir` printed, and how it ended, given `input` on its standard -/// input, which is not a terminal: no prompts are written. +/// what a `by repl` session in `dir` printed, and how it ended, given `input` on its standard +/// input, which is not a terminal: no prompts are written fn repl_session(dir: &Path, input: &str, extra: &[&str]) -> (Option, String, String) { let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); let mut child = Command::new(env!("CARGO_BIN_EXE_by")) @@ -2027,7 +2027,7 @@ fn repl_session(dir: &Path, input: &str, extra: &[&str]) -> (Option, String ) } -/// A project with a `.by` module for a session to import. +/// a project with a `.by` module for a session to import fn project_with_a_module(dir: &Path) { let package = dir.join("pkg"); fs::create_dir_all(&package).unwrap(); @@ -2138,6 +2138,61 @@ fn repl_takes_a_framed_input_whole_and_hands_input_lines_to_the_program() { assert_eq!(stdout, "2\n'basedpython'\n", "{stderr}"); } +/// as in python's repl, each expression statement of an input prints its value, not only the +/// last: those on one line, and those in a block, but not those in a function's body +#[test] +fn repl_prints_the_value_of_every_expression_statement() { + let dir = tempfile::tempdir().expect("tempdir"); + let (status, stdout, stderr) = repl_session( + dir.path(), + "1; 2\n\ + x = 3; x\n\ + for i in range(2):\n i\n\n\ + def quiet() -> None:\n 4\n\n\ + quiet()\n\ + None\n", + &[], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!(stdout, "1\n2\n3\n0\n1\n", "{stderr}"); +} + +/// a line that opens a block, starts a clause after one, decorates a definition, leaves a +/// bracket or a triple-quoted string open or ends in `\` waits for the next line, and a block +/// runs at the blank line after it, whatever statement opened it +#[test] +fn repl_waits_for_the_rest_of_every_kind_of_input() { + let dir = tempfile::tempdir().expect("tempdir"); + let (status, stdout, stderr) = repl_session( + dir.path(), + "try:\n int(\"x\")\nexcept ValueError:\n print(\"caught\")\n\n\ + match 3:\n case 3:\n print(\"three\")\n\n\ + class Box:\n size = 2\n\n\ + def deco(f):\n return f\n\n\ + @deco\ndef twice(x: int) -> int:\n return x * 2\n\n\ + twice(Box.size)\n\ + flag = len(\"\") > 0\n\ + if flag:\n print(\"if\")\nelif not flag:\n print(\"elif\")\nelse:\n print(\"else\")\n\n\ + for i in range(1):\n pass\nelse:\n print(\"for else\")\n\n\ + with open(\"data.txt\", \"w\") as f:\n f.write(\"x\")\n\n\ + async def fetch() -> int:\n return 5\n\n\ + import asyncio\n\ + asyncio.run(fetch())\n\ + total = (1 +\n 2)\n\ + total\n\ + text = \"\"\"a\nb\"\"\"\n\ + text\n\ + value = 1 + \\\n 2\n\ + value\n", + &[], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!( + stdout, "caught\nthree\n4\nelif\nfor else\n1\n5\n3\n'a\\nb'\n3\n", + "{stderr}" + ); +} + /// the session's module is the project's own code, so a `sealed` class declared in it can be /// subclassed in a later input, as in any module of the project #[test] @@ -2172,8 +2227,94 @@ fn repl_session_file_holds_the_inputs_that_ran() { ); } -/// A `by repl` session in a project, driven as an editor's console drives one: through pipes, -/// listening for the session's events on a port of its own with `--events`. +/// a package whose module `app.main` imports a sibling relatively, for a session to start in +fn project_with_a_program(dir: &Path) { + let package = dir.join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write( + package.join("helpers.by"), + "def double(x: int) -> int:\n return x * 2\n", + ) + .unwrap(); + fs::write( + package.join("main.by"), + "from .helpers import double\n\nlet base = 21\nanswer = double(base)\nprint(f\"answer is {answer}\")\n\n\ndef main():\n print(\"main ran\")\n", + ) + .unwrap(); +} + +/// with `--module`, the session starts in a module of the project: it runs first, as +/// `__main__` and as `by run` runs it but for its `def main`, and each input is its +/// continuation — its names are defined, its `let` stays read-only, and its relative import +/// resolves, in the check and when the session runs +#[test] +fn repl_starts_in_a_module() { + let dir = tempfile::tempdir().expect("tempdir"); + project_with_a_program(dir.path()); + let (status, stdout, stderr) = repl_session( + dir.path(), + "answer + 1\n\ + base = 1\n\ + from .helpers import double as twice\n\ + twice(base)\n\ + main()\n\ + (__name__, __package__, __file__.endswith('main.by'))\n", + &["--module", "app.main", "--session-file", "session.by"], + ); + assert_eq!(status, Some(0), "{stderr}"); + assert_eq!( + stdout, "answer is 42\n43\n42\nmain ran\n('__main__', 'app', True)\n", + "{stderr}" + ); + // the refused input is reported against itself, and the declaration it breaks against the + // module's own file + let main = Path::new("app").join("main.by").display().to_string(); + assert!( + stderr.contains("error[invalid-assignment]") + && stderr.contains("--> :1:1") + && stderr.contains(&format!("::: {main}:3:1")), + "{stderr}" + ); + // the session's source starts with the module's + let session = fs::read_to_string(dir.path().join("session.by")).expect("the session file"); + assert!( + session.starts_with("from .helpers import double\n") + && session.ends_with("main()\n(__name__, __package__, __file__.endswith('main.by'))\n"), + "{session}" + ); +} + +/// a module the project does not have is no session to start in +#[test] +fn repl_does_not_start_in_a_module_the_project_does_not_have() { + let dir = tempfile::tempdir().expect("tempdir"); + project_with_a_program(dir.path()); + let (status, stdout, stderr) = repl_session(dir.path(), "1\n", &["--module", "app.nowhere"]); + assert_ne!(status, Some(0), "{stderr}"); + assert_eq!(stdout, ""); + assert!( + stderr.contains("no module named `app.nowhere` in the project"), + "{stderr}" + ); +} + +/// a session whose interpreter a signal ends exits as a shell reports that, with 128 and the +/// signal's number +#[cfg(unix)] +#[test] +fn repl_exits_with_the_signal_that_ended_its_interpreter() { + let dir = tempfile::tempdir().expect("tempdir"); + let (status, _, stderr) = repl_session( + dir.path(), + "import os\nimport signal\nos.kill(os.getpid(), signal.SIGKILL)\n", + &[], + ); + assert_eq!(status, Some(128 + 9), "{stderr}"); +} + +/// a `by repl` session in a project, driven as an editor's console drives one: through pipes, +/// listening for the session's events on a port of its own with `--events` struct EditorSession { child: Child, stdin: ChildStdin, @@ -2181,8 +2322,8 @@ struct EditorSession { } impl EditorSession { - /// The session started in `dir` with `extra` arguments, its connection taken once it has - /// sent the token the editor gave it. + /// the session started in `dir` with `extra` arguments, its connection taken once it has + /// sent the token the editor gave it fn start(dir: &Path, extra: &[&str]) -> Self { let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); let listener = TcpListener::bind(("127.0.0.1", 0)).expect("a port to listen on"); @@ -2240,7 +2381,7 @@ impl EditorSession { session } - /// The next event the session sent, `null` once it has closed the connection. + /// the next event the session sent, `null` once it has closed the connection fn next_event(&mut self) -> serde_json::Value { let mut line = String::new(); self.events.read_line(&mut line).expect("an event"); @@ -2250,13 +2391,13 @@ impl EditorSession { serde_json::from_str(&line).expect("an event is a line of json") } - /// Wait for `expected`, the next event. + /// wait for `expected`, the next event #[track_caller] fn expect(&mut self, expected: &serde_json::Value) { assert_eq!(&self.next_event(), expected); } - /// Send `text` to the session's standard input as it is. + /// send `text` to the session's standard input as it is fn send(&mut self, text: &str) { self.stdin .write_all(text.as_bytes()) @@ -2264,12 +2405,12 @@ impl EditorSession { self.stdin.flush().expect("the session reads"); } - /// Send `text` as an editor sends an input, framed as a paste. + /// send `text` as an editor sends an input, framed as a paste fn send_input(&mut self, text: &str) { self.send(&format!("\x1b[200~{text}\x1b[201~\n")); } - /// End the session's input, and what it printed and how it ended. + /// end the session's input, and what it printed and how it ended fn finish(self) -> (Option, String, String, Vec) { let Self { child, diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 220704940b..54cc4bef51 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -79,11 +79,11 @@ impl EnvVars { /// side was. Any other value is refused. pub const BY_TEST_REPORTER: &'static str = "BY_TEST_REPORTER"; - /// The token `by repl --events` sends first on the connection it makes to an editor. + /// the token `by repl --events` sends first on the connection it makes to an editor /// - /// An editor listening for a session's events sets this to a value nothing else can guess, - /// and takes the connection whose first line carries it as the session's. The program the - /// session runs does not see it. + /// an editor listening for a session's events sets this to a value nothing else can guess, + /// and takes the connection whose first line carries it as the session's. the program the + /// session runs does not see it pub const BY_REPL_EVENTS_TOKEN: &'static str = "BY_REPL_EVENTS_TOKEN"; /// Enable uv integration. diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index c16ac07189..a40a6bd49b 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -188,6 +188,7 @@ and `build` alike ```sh by repl # an interactive session, on the project's interpreter by repl --python python3.14 # on another interpreter +by repl --module app.main # starting in a module of the project, after running it ``` an interactive session in which you type basedpython, each input checked and transpiled as the @@ -195,7 +196,9 @@ continuation of the inputs before it and run in one python process. it starts wh would, with the project staged as `by run` stages it, so the project's modules import as they do in a program. `--session-file PATH` keeps `PATH` holding the inputs that ran, for an editor's console to complete against, and `--events PORT` tells the console, on a connection to `PORT`, -whether standard input is read for an input or by the program. see [the repl](features/repl.md) +whether standard input is read for an input or by the program. `--module MODULE` starts the +session in a module of the project: it runs first, and each input is its continuation. see +[the repl](features/repl.md) ## `by build` diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index c282ae3046..4be0453b8b 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -60,6 +60,48 @@ two things differ from `by run`, because a session is never run as a whole. a `f binds its names when it is entered rather than at their first use, so a name that cannot be imported says so then, and a `def main` is a function like any other: nothing calls it +## starting in a module + +`by repl --module app.main` starts the session in a module of the project, named as `by run` +takes one — what an editor's *run with console* does. the module runs first, as `__main__`, with +the `__file__`, package and spec `by run` gives it, and the session goes on from there with its +names. its source is the session's first: each input is checked and transpiled as the module's +continuation, as the module itself, so what it defines is defined, a `let` of its stays +read-only, and a relative import resolves, in the module and in an input + +```by +# app/main.by +from .helpers import double + +let base = 21 +answer = double(base) +print(f"answer is {answer}") +``` + +```console +$ by repl --module app.main +answer is 42 +>>> answer + 1 +43 +>>> base = 1 +error[invalid-assignment]: read-only symbol `base` cannot be reassigned + --> :1:1 + | +1 | base = 1 + | ^^^^^^^^ reassigned here + | + ::: app/main.by:3:1 + | +3 | let base = 21 + | ---- declared read-only here +``` + +it runs as the start of a session, not as a program: its `def main` is a function like any +other, which nothing calls — `main()` is one input away — and a value its last line computes is +not printed. a diagnostic or a traceback that points into it names its file and line. a module +the project does not have, or one whose python cannot run as a session's start, keeps the +session from starting + ## what is printed each expression statement of an input prints its value, through `sys.displayhook`, as python's @@ -155,8 +197,9 @@ an editor's console runs `by repl` as a process and talks to it through its stan `\`; `invalid` — no more lines can make it valid, and running it reports why. blank input, and input of comments only, is `complete` -- **completion.** `--session-file PATH` keeps `PATH` holding the session's source — the inputs - that ran, in order, each ending in a newline — from the moment the session starts. completion +- **completion.** `--session-file PATH` keeps `PATH` holding the session's source — the module + it started in, if it started in one, then the inputs that ran, in order, each ending in a + newline — from the moment the session starts. completion in the console is the language server's ordinary completion, asked of a document whose text is that file's followed by the input being typed, at the caret's offset past the file's length: the same module the session checks the input as, so what is offered is what the From 26e43351f5d8788a18e175cfdb15071884368602 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:06:39 +1000 Subject: [PATCH 095/110] let an exception a by run program does not catch leave the interpreter as it leaves python main.py, rather than the runner catching it, printing it and raising SystemExit(1) in its own frame, so that a debugger watching for uncaught exceptions stops for the program's own exception with the program's frames on top, where debugpy stopped for a SystemExit at _by_runner.py and bpd reported SystemExit: 1; and say what a debugger finds under by run's launcher --- crates/ty/src/by_commands.rs | 13 +++-- crates/ty/tests/by_e2e.rs | 80 +++++++++++++++++++++++++++++++ docs/basedpython/cli-reference.md | 11 +++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 49f008181e..311fc40cc6 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -3784,13 +3784,12 @@ def main(): profiler = None if "--profile" in written: profiler = _start_profile(written["--profile"]) - try: - _run_as_main(module, profiler) - except SystemExit: - raise - except BaseException: - sys.excepthook(*sys.exc_info()) - sys.exit(1) + # nothing here catches what the program raises. an exception it does not catch leaves + # the interpreter as it leaves `python main.py`: python reports it through the + # excepthook above and exits 1. a debugger watching for an exception the program did not + # catch then sees the program's own, with the program's frames on top, where catching + # it here had it see a `SystemExit` the runner raised in a frame of its own + _run_as_main(module, profiler) if __name__ == "__main__": diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index fa53e02925..a115ca837e 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -4266,6 +4266,86 @@ main() ); } +/// a debugger stops for an exception the program did not catch as it leaves the outermost +/// frame, and shows the frames it came through. that has to be the program's own exception, +/// raised where the program raised it, as it is under `python main.py` — not a `SystemExit` +/// the runner raised after catching it, whose frames are all the runner's +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test prints why it was skipped" +)] +fn run_lets_an_exception_the_program_did_not_catch_leave_the_interpreter() { + let Some(python) = python_at_least_312() else { + eprintln!("skipping: no python 3.12+ interpreter available"); + return; + }; + let dir = tempfile::tempdir().expect("tempdir"); + // what a debugger listens for: `PY_UNWIND`, in the frame nothing called + fs::write( + dir.path().join("watch.py"), + "\ +import atexit +import os +import sys + +left = [] + + +def unwound(code, offset, exception): + if sys._getframe(1).f_back is None: + tb = exception.__traceback__ + while tb.tb_next is not None: + tb = tb.tb_next + raised_in = os.path.basename(tb.tb_frame.f_code.co_filename) + left.append(f'{type(exception).__name__} from {raised_in}:{tb.tb_lineno}') + + +monitoring = sys.monitoring +monitoring.use_tool_id(monitoring.DEBUGGER_ID, 'watch') +monitoring.register_callback(monitoring.DEBUGGER_ID, monitoring.events.PY_UNWIND, unwound) +monitoring.set_events(monitoring.DEBUGGER_ID, monitoring.events.PY_UNWIND) +atexit.register(lambda: print(left)) +", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "\ +import watch + + +def boom() -> None: + raise ValueError('bad') + + +print(watch.left) +boom() +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", &python, "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + // python writes its output in text mode, which ends a line with `\r\n` on windows + let stderr = String::from_utf8_lossy(&output.stderr).replace("\r\n", "\n"); + assert_eq!(output.status.code(), Some(1), "{stderr}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).replace("\r\n", "\n"), + "[]\n['ValueError from main.by:5']\n", + "{stderr}" + ); + assert!( + stderr.contains("line 5, in boom") && stderr.contains("ValueError: bad"), + "{stderr}" + ); +} + #[test] fn run_traceback_of_a_failed_import_shows_no_import_machinery() { // python removes its own import machinery from a traceback raised while a diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index a40a6bd49b..b2a371edb6 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -74,6 +74,17 @@ interpreter for it. discovery runs exactly as above, and the program starts as interpreter `by run` chose and runs once, for the program: the version probe is still made against the interpreter itself +what a debugger finds in that process is the program under ``, the +`_by_runner.py` `by run` writes into the tree it staged. the runner's frames are +the outermost of the main thread, beneath the program's module, and they are +by's alone: every frame whose file is `` is the runner's. the runner +catches nothing the program raises, so an exception the program does not catch +leaves the interpreter as it leaves `python main.py` — the program's own +exception, with the program's frames on top of its traceback — and python +reports it and exits 1. a debugger that stops where such an exception leaves the +outermost frame stops in the program by treating the runner's frames as it +treats `runpy`'s under `python -m`: as the launcher's, not the program's + `--in-build` runs the program in the tree `by run` staged rather than in the directory it was typed in. the default is right for an application: a relative path on the command line, and anything the program reads or writes beside the From 38f6d619547dd334f7319afc973f5efc6889339b Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:14:50 +1000 Subject: [PATCH 096/110] keep by's own python out of a by run --profile snapshot and a by repl traceback: the profile ends with python calling the profiler's disable rather than the runner's _write_profile, by's functions are folded into the functions that called them, the runner's finder and loader into python's import machinery as python's own are under python main.py, and the excepthook's traceback leaves with what it called; and a traceback drops every frame of by's files below the program's first, so a KeyboardInterrupt while the program reads standard input in by repl ends at the line that read rather than in _by_repl.py's readline --- crates/ty/docs/cli.md | 4 +- crates/ty/src/args.rs | 8 +- crates/ty/src/by_commands.rs | 169 ++++++++++++++++++++++++++---- crates/ty/tests/by_e2e.rs | 131 ++++++++++++++++++++++- crates/ty_static/src/env_vars.rs | 6 +- docs/basedpython/cli-reference.md | 10 +- docs/basedpython/features/repl.md | 4 +- 7 files changed, 300 insertions(+), 32 deletions(-) diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index b3de89efee..b8320d5a92 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -265,8 +265,8 @@ by run [OPTIONS] [MODULE] [ARGS]...

The interpreter is chosen exactly as it is without this, and the program is started as LAUNCHER <interpreter> <runner> <module> <args>: the launcher is handed the interpreter by run chose rather than having to name one, and is run once, for the program — never for the version probe, which is made against the interpreter itself.

--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

-
--profile path

Profile the program with cProfile and write the snapshot to PATH, as the pstats file cProfile writes (.pstat).

-

Every function of a transpiled module is named by its .by file and the .by line it starts at, and one of a .py the project carries over by the file in the project rather than the staged copy that ran, so a profile viewer's way to a function's source leads to the file it was written in. The runtime's helpers are named <by runtime>, at the line of _by_runtime.py they start on. The profile starts at the program's first line and ends after its last atexit handler.

+
--profile path

profile the program with cProfile and write the snapshot to PATH, as the pstats file cProfile writes (.pstat)

+

every function of a transpiled module is named by its .by file and the .by line it starts at, and one of a .py the project carries over by the file in the project rather than the staged copy that ran, so a profile viewer's way to a function's source leads to the file it was written in. the runtime's helpers are named <by runtime>, at the line of _by_runtime.py they start on. the profile starts at the program's first line and ends after its last atexit handler, and by's own code is not in it: the loader it puts on each module is folded into python's import machinery, which called it

--python, --venv path

The interpreter to run on, or the environment holding it. A bare name that is neither, like python3.14, is found on PATH.

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index eda8ca4043..898ada3756 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -139,9 +139,11 @@ pub(crate) enum Command { /// the `.by` line it starts at, and one of a `.py` the project carries over /// by the file in the project rather than the staged copy that ran, so a /// profile viewer's way to a function's source leads to the file it was - /// written in. The runtime's helpers are named ``, at the line - /// of `_by_runtime.py` they start on. The profile starts at the program's - /// first line and ends after its last `atexit` handler. + /// written in. the runtime's helpers are named ``, at the line + /// of `_by_runtime.py` they start on. the profile starts at the program's + /// first line and ends after its last `atexit` handler, and by's own code is + /// not in it: the loader it puts on each module is folded into python's + /// import machinery, which called it #[arg(long, value_name = "PATH", conflicts_with = "coverage")] profile: Option, }, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 311fc40cc6..79aec558e8 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -2786,20 +2786,46 @@ fn write_project_module( /// the line [`BY_RUNNER_SRC`] holds for the runtime's definitions, which [`runner_source`] fills in const RUNTIME_LINES_PLACEHOLDER: &str = "_RUNTIME_LINES = {}\n"; +/// the line [`BY_RUNNER_SRC`] holds for the files of by's own python in the tree, which +/// [`runner_source`] fills in +const MACHINERY_FILES_PLACEHOLDER: &str = "_MACHINERY_FILES = []\n"; + +/// the python `by` writes into the tree to run the program, none of which is the program's: +/// what a traceback and a profile leave out +const MACHINERY_FILES: [&str; 4] = [ + BY_RUNNER_FILENAME, + BY_REPL_FILENAME, + BY_PYTEST_PLUGIN_FILENAME, + BY_COVERAGE_PLUGIN_FILENAME, +]; + /// the runner shim, with the line of `_by_runtime.py` each of the runtime's definitions starts -/// on, by the names it binds: what a profile names the runtime's code by +/// on, by the names it binds — what a profile names the runtime's code by — and the files of +/// by's own python beside it fn runner_source() -> anyhow::Result { anyhow::ensure!( BY_RUNNER_SRC.contains(RUNTIME_LINES_PLACEHOLDER), "the runner has no place for the runtime's lines" ); - // a json object of strings and integers is a python dict literal + anyhow::ensure!( + BY_RUNNER_SRC.contains(MACHINERY_FILES_PLACEHOLDER), + "the runner has no place for the files of by's own python" + ); + // a json object of strings and integers is a python dict literal, and a json array of + // strings a python list literal let lines = serde_json::to_string(&by_transforms::runtime::definition_lines())?; - Ok(BY_RUNNER_SRC.replacen( - RUNTIME_LINES_PLACEHOLDER, - &format!("_RUNTIME_LINES = {lines}\n"), - 1, - )) + let machinery = serde_json::to_string(&MACHINERY_FILES)?; + Ok(BY_RUNNER_SRC + .replacen( + RUNTIME_LINES_PLACEHOLDER, + &format!("_RUNTIME_LINES = {lines}\n"), + 1, + ) + .replacen( + MACHINERY_FILES_PLACEHOLDER, + &format!("_MACHINERY_FILES = {machinery}\n"), + 1, + )) } const BY_RUNNER_SRC: &str = r#"# generated by `by run` — runs the target module with .by-aware tracebacks @@ -2844,6 +2870,12 @@ _RUNNER = os.path.realpath(__file__) _SESSION_INPUT = " 0 or row[4]: + return + del rows[key] + if key in pending: + pending.remove(key) + for callee in callees.pop(key): + if callee in rows: + drop(callee, rows[callee][4].pop(key)) + + while pending: + key = next( + ( + key + for key in pending + if not any(caller in pending and caller != key for caller in rows[key][4]) + ), + pending[0], + ) + pending.remove(key) + _, _, inline, _, callers = rows.pop(key) + callers.pop(key, None) + called = callees.pop(key) + called.discard(key) + outgoing = {callee: rows[callee][4].pop(key) for callee in called if callee in rows} + for caller in callers: + callees[caller].discard(key) + if not callers: + for callee, timing in outgoing.items(): + drop(callee, timing) + continue + calls = sum(timing[0] for timing in callers.values()) + shares = { + caller: timing[0] / calls if calls else 1 / len(callers) + for caller, timing in callers.items() + } + for caller, share in shares.items(): + rows[caller][2] += inline * share + for callee, (count, primitive, inline_time, total_time) in outgoing.items(): + counts = _apportioned(count, shares) + primitives = _apportioned(primitive, shares) + for caller, share in shares.items(): + part = (counts[caller], primitives[caller], inline_time * share, total_time * share) + had = rows[callee][4].get(caller) + rows[callee][4][caller] = part if had is None else _summed(had, part) + callees[caller].add(callee) + return {key: (*row[:4], row[4]) for key, row in rows.items()} + + def _write_profile(profiler, snapshot): """What `profiler` measured, as the `pstats` file `cProfile` writes, with every function of a staged module named by the project file and line it was written at. @@ -3636,12 +3765,11 @@ def _write_profile(profiler, snapshot): The runtime's own code is the exception: a helper the transpiler wrote into a module, and one in a copy of the runtime written beside a package's modules, are - `` at the line of `_by_runtime.py` they are, under their own names. A - helper is then one function wherever it ran from, two helpers of one name are two, - and a viewer's way to a function's source finds no file to open, where line 0 of - the module it was written into was the module's first line. + `` at the line of `_by_runtime.py` they are, under their own names, so a + helper is one function wherever it ran from, and two helpers of one name are two. + + By's own functions are not in it: see `_without_machinery`. """ - profiler.disable() located = {} parsed = {} @@ -3695,6 +3823,9 @@ def _write_profile(profiler, snapshot): ) callers = stats[callee][4] callers[caller] = _summed(callers[caller], timing) if caller in callers else timing + stats = _without_machinery( + stats, {key for key in stats if os.path.realpath(key[0]) in _MACHINERY} + ) with open(snapshot, "wb") as handle: marshal.dump(stats, handle) @@ -3706,11 +3837,13 @@ def _runtime_line(path, line, name, parsed): transpiler wrote into a module. `None` for any other code. `parsed` keeps each module's top-level statements once read.""" import ast + import pathlib real = os.path.realpath(path) - if ( - os.path.basename(real) == "_by_runtime.py" - and os.path.commonpath([real, _TREE]) == _TREE + # `is_relative_to` rather than `os.path.commonpath`, which raises for a path on another + # drive than the tree's + if os.path.basename(real) == "_by_runtime.py" and pathlib.PurePath(real).is_relative_to( + _TREE ): # no project file can take this name in the tree, which the staging refuses return line diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index a115ca837e..d5d86c672c 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1951,6 +1951,94 @@ fn run_profile_names_the_runtime_s_helpers_by_the_runtime() { assert!(at_line_0.is_empty(), "{at_line_0:#?}"); } +/// the functions of `functions` in a file of by's own python, or called from one +fn by_s_own(functions: &[Profiled]) -> Vec<&Profiled> { + let by_s = |file: &str| { + [ + "_by_runner.py", + "_by_repl.py", + "_by_pytest.py", + "_by_coverage_plugin.py", + ] + .iter() + .any(|name| file.ends_with(name)) + }; + functions + .iter() + .filter(|function| { + by_s(&function.file) || function.callers.iter().any(|(file, ..)| by_s(file)) + }) + .collect() +} + +/// a profile is the program's: the loader the runner puts on every staged module is +/// folded into python's import machinery, which called it, as python's own finder and +/// loader are there under `python main.py`; the runner ending the profile and writing +/// it is not in it; and neither is the traceback the runner prints for an exception the +/// program did not catch, which python prints once the program has ended +#[test] +fn run_profile_leaves_out_by_s_own_machinery() { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("pkg"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write( + package.join("calc.by"), + "def double(n: int) -> int:\n return n * 2\n", + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "from pkg.calc import double\n\n\nprint(double(2))\nraise ValueError('done')\n", + ) + .unwrap(); + let project = dir.path().canonicalize().unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", &python, "--profile", "run.pstat", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{stderr}"); + assert!(stderr.contains("ValueError: done"), "{stderr}"); + let functions = profiled_functions(&python, &dir.path().join("run.pstat")); + + let calc = project.join("pkg/calc.by").display().to_string(); + assert!( + functions + .iter() + .any(|function| function.file == calc && function.name == "double"), + "{functions:#?}" + ); + assert_eq!(by_s_own(&functions).len(), 0, "{:#?}", by_s_own(&functions)); + let machinery: Vec<&Profiled> = functions + .iter() + .filter(|function| { + function.name.contains("'disable' of '_lsprof.Profiler'") + || function.file.ends_with("traceback.py") + }) + .collect(); + assert!(machinery.is_empty(), "{machinery:#?}"); + // what the runner's finder did is python's import machinery's: each function is + // called as often as it was, by a function that is in the profile + for function in &functions { + if function.callers.is_empty() { + continue; + } + for (file, line, name) in &function.callers { + assert!( + functions + .iter() + .any(|caller| (&caller.file, &caller.line, &caller.name) == (file, line, name)), + "{function:#?} is called by {name} in {file}, which is not in the profile" + ); + } + } +} + /// a test runner compiles a test module itself, so the module's own top level runs /// as the staged `.py`, at the generated python's lines, and is still named by the /// `.by` @@ -1995,6 +2083,8 @@ fn run_profile_maps_a_module_a_test_runner_compiled() { .any(|function| function.file.ends_with("test_calc.py")), "the staged test module is named: {functions:#?}" ); + // pytest's plugin is pytest's to call, and is folded into what called it + assert_eq!(by_s_own(&functions).len(), 0, "{:#?}", by_s_own(&functions)); } /// what a `by repl` session in `dir` printed, and how it ended, given `input` on its standard @@ -2490,10 +2580,47 @@ fn repl_tells_an_editor_what_standard_input_is_read_for() { assert_eq!(rest, Vec::::new()); } -/// A project of `.by` tests for pytest to run, with a module they test, checked +/// a `KeyboardInterrupt` while the program reads standard input is the program's: its +/// traceback ends at the line that read, as python's does, and not in the standard input +/// `by repl` reads the program's lines through +/// +/// unix only: the program interrupts itself with `os.kill(pid, SIGINT)`, which windows has no +/// equivalent of, its `os.kill` ending the process for any signal but a console event +#[cfg(unix)] +#[test] +fn repl_interrupts_a_read_of_the_program_s_at_the_program_s_line() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut session = EditorSession::start(dir.path(), &[]); + session.expect(&serde_json::json!({ "reading": "input" })); + + // ctrl-c, as the console sends it, once the program is waiting for its line + session.send_input( + "import os, signal, threading\n\ + threading.Timer(0.2, os.kill, (os.getpid(), signal.SIGINT)).start()\n\ + name = input('who? ')", + ); + session.expect(&serde_json::json!({ "reading": "program" })); + session.expect(&serde_json::json!({ "finished": 1, "ran": true })); + session.expect(&serde_json::json!({ "reading": "input" })); + + let (status, _, stderr, _) = session.finish(); + assert_eq!(status, Some(0), "{stderr}"); + let frames: Vec<&str> = stderr + .lines() + .filter(|line| line.trim_start().starts_with("File ")) + .collect(); + assert_eq!( + frames, + [r#" File "", line 3, in "#], + "{stderr}" + ); + assert!(stderr.ends_with("KeyboardInterrupt\n"), "{stderr}"); +} + +/// a project of `.by` tests for pytest to run, with a module they test, checked /// against the environment the tests run in: the `conftest` imports pytest, which /// `--python` does not configure. `None`, having said why, when `$PYTHON` (or -/// `python3`) cannot import pytest. +/// `python3`) cannot import pytest #[expect( clippy::print_stderr, reason = "a skipped test must say why it skipped, or it reads as a pass" diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 54cc4bef51..4515a02160 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -71,12 +71,12 @@ impl EnvVars { /// Equivalent to the `--no-server` command-line argument. pub const BY_NO_PROJECT_SERVER: &'static str = "BY_NO_PROJECT_SERVER"; - /// Report the tests pytest runs under `by run` as service messages, `TeamCity`'s protocol. + /// report the tests pytest runs under `by run` as service messages, `TeamCity`'s protocol /// - /// Set this to `teamcity` for an IDE's test runner to read the run: a suite for each test + /// set this to `teamcity` for an IDE's test runner to read the run: a suite for each test /// file, named by the `.by` it was transpiled from, and one for each class in it, each test /// with its pytest node id and where it is in the `.by`, and a failing `==` with what each - /// side was. Any other value is refused. + /// side was. any other value is refused pub const BY_TEST_REPORTER: &'static str = "BY_TEST_REPORTER"; /// the token `by repl --events` sends first on the connection it makes to an editor diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index b2a371edb6..7a58f63bdf 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -167,10 +167,14 @@ $ python -m pstats run.pstat ``` the profile starts at the program's first line and ends after its last `atexit` -handler, so it holds the program and not `by run`'s search for it. what `by run` +handler, so it holds the program and not `by run`'s search for it, nor the +writing of the snapshot or the traceback of an exception the program did not +catch. `by`'s own python in the staged tree is not in it either: what `by run` does to load a module — finding it, and compiling it under its `.by` lines — is -time the program spends, and is named by the runner's file in the staged tree. -cProfile measures every thread on python 3.12 and later, and only the thread the +time the program spends, so it is folded into python's import machinery, which +called it, as python's own finder and loader are under `python main.py`. so is +pytest's plugin into pytest, under a test run. a function of `by`'s called from +two places is shared between them by how often each called it. cProfile measures every thread on python 3.12 and later, and only the thread the program starts on before that, as `python -m cProfile` does. a program that leaves without returning to `by run` — on `os._exit`, an `os.exec*` or a signal — leaves no snapshot, and `--coverage` cannot be combined with it, since a profile taken under coverage.py's tracer diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index 4be0453b8b..e3ae8eda93 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -108,7 +108,9 @@ each expression statement of an input prints its value, through `sys.displayhook repl prints it: both of `1; 2`, and one in a loop's body, but not one in a function's. an input is numbered whether it runs or not, as python numbers its own. an exception prints a traceback whose frames name the input they ran in, at the input's line, with the line quoted, and the project's modules at -their `.by` lines +their `.by` lines. the python `by` runs the session with is not in it: a `KeyboardInterrupt` +while the program reads standard input ends at the line that read, as it does in python's +repl ```console >>> def ratio(a: int, b: int) -> float: From f4f93b39d47bc523e396b8f0d0945d5080660e19 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:18:25 +1000 Subject: [PATCH 097/110] =?UTF-8?q?answer=20initialize=20with=20the=20flag?= =?UTF-8?q?s=20of=20by=20repl=20and=20by=20run=20an=20editor=20starts=20by?= =?UTF-8?q?=20with,=20under=20experimental=20=E2=80=94=20repl:=20events,?= =?UTF-8?q?=20module,=20sessionFile;=20run:=20coverage,=20inBuild,=20launc?= =?UTF-8?q?her,=20profile=20=E2=80=94=20so=20an=20editor=20learns=20what?= =?UTF-8?q?=20the=20by=20it=20talks=20to=20takes=20from=20the=20language?= =?UTF-8?q?=20server=20rather=20than=20by=20running=20by=20repl=20--help,?= =?UTF-8?q?=20with=20a=20test=20that=20each=20flag=20named=20is=20one=20th?= =?UTF-8?q?e=20command=20takes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/ty/src/args.rs | 38 +++++++++++++++++++ crates/ty_server/src/capabilities.rs | 32 ++++++++++++++++ crates/ty_server/src/lib.rs | 1 + crates/ty_server/tests/e2e/initialize.rs | 21 ++++++++++ .../e2e__initialize__initialization.snap | 13 +++++++ ...ialize__initialization_with_workspace.snap | 13 +++++++ docs/basedpython/features/editor.md | 20 ++++++++++ docs/basedpython/features/repl.md | 5 ++- 8 files changed, 142 insertions(+), 1 deletion(-) diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 898ada3756..44a86b5ba5 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -991,3 +991,41 @@ fn resolve_bool_arg(yes: bool, no: bool) -> Option { (..) => unreachable!("Clap should make this impossible"), } } + +#[cfg(test)] +mod tests { + use clap::CommandFactory; + + use super::Cli; + + /// what the language server tells an editor `by` takes on its command line is what it + /// takes: each flag it names is a flag of the command it names it under + #[test] + fn the_language_server_names_only_flags_the_command_line_takes() { + let cli = Cli::command(); + for (name, flags) in ty_server::COMMAND_LINE_FLAGS { + let command = cli + .find_subcommand(name) + .unwrap_or_else(|| panic!("`by` has no command `{name}`")); + for flag in flags { + // `sessionFile` is `--session-file` + let long: String = flag + .chars() + .flat_map(|c| { + [ + c.is_ascii_uppercase().then_some('-'), + Some(c.to_ascii_lowercase()), + ] + }) + .flatten() + .collect(); + assert!( + command + .get_arguments() + .any(|argument| argument.get_long() == Some(long.as_str())), + "`by {name}` has no `--{long}`, which the language server says it takes as `{flag}`" + ); + } + } + } +} diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index d852c5d29d..6d902e4090 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -605,6 +605,37 @@ impl ResolvedClientCapabilities { } } +/// the flags of `by`'s commands that an editor starts `by` with, for the `by` this server is +/// part of: each command by name, and each flag by its long name in camelCase — +/// `sessionFile` is `--session-file` +/// +/// `initialize` answers with them under `experimental`, each command an object of its flags +/// set to `true`, so that an editor knows what the `by` it already talks to takes without +/// running it to read its `--help`: +/// +/// ```json +/// { "repl": { "events": true, "module": true, "sessionFile": true }, "run": { ... } } +/// ``` +pub const COMMAND_LINE_FLAGS: [(&str, &[&str]); 2] = [ + ("repl", &["events", "module", "sessionFile"]), + ("run", &["coverage", "inBuild", "launcher", "profile"]), +]; + +/// [`COMMAND_LINE_FLAGS`], as `initialize` answers with them +fn command_line_capabilities() -> serde_json::Value { + COMMAND_LINE_FLAGS + .iter() + .map(|(command, flags)| { + let flags = flags + .iter() + .map(|flag| ((*flag).to_owned(), serde_json::Value::Bool(true))) + .collect(); + ((*command).to_owned(), serde_json::Value::Object(flags)) + }) + .collect::>() + .into() +} + /// Creates the server capabilities based on the resolved client capabilities and resolved global /// settings from the initialization options. pub(crate) fn server_capabilities( @@ -767,6 +798,7 @@ pub(crate) fn server_capabilities( }), type_hierarchy_provider: Some(true.into()), call_hierarchy_provider: Some(true.into()), + experimental: Some(command_line_capabilities()), ..Default::default() } } diff --git a/crates/ty_server/src/lib.rs b/crates/ty_server/src/lib.rs index 49a78042d1..c785aec637 100644 --- a/crates/ty_server/src/lib.rs +++ b/crates/ty_server/src/lib.rs @@ -4,6 +4,7 @@ use anyhow::Context; use lsp_server::Connection; use ruff_db::system::{OsSystem, SystemPathBuf}; +pub use crate::capabilities::COMMAND_LINE_FLAGS; use crate::db::Db; pub use crate::logging::{LogLevel, init_logging}; pub use crate::server::Server; diff --git a/crates/ty_server/tests/e2e/initialize.rs b/crates/ty_server/tests/e2e/initialize.rs index bb42299f9f..e46c938964 100644 --- a/crates/ty_server/tests/e2e/initialize.rs +++ b/crates/ty_server/tests/e2e/initialize.rs @@ -121,6 +121,27 @@ fn deferred_request_burst_does_not_block_responses() -> Result<()> { Ok(()) } +/// an editor that starts `by repl` or `by run` learns which of their flags the `by` it talks to +/// takes from `initialize`'s answer, rather than by running it to read its `--help` +#[test] +fn initialization_names_the_command_line_flags_an_editor_starts_by_with() -> Result<()> { + let server = TestServerBuilder::new()? + .build() + .wait_until_workspaces_are_initialized(); + + let initialization_result = server.initialization_result().unwrap(); + + assert_eq!( + initialization_result.capabilities.experimental, + Some(json!({ + "repl": { "events": true, "module": true, "sessionFile": true }, + "run": { "coverage": true, "inBuild": true, "launcher": true, "profile": true }, + })) + ); + + Ok(()) +} + /// Tests that the server sends a registration request for diagnostics if workspace diagnostics /// are enabled via initialization options and dynamic registration is enabled, even if the /// workspace configuration is not supported by the client. diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 2b639cc041..ad0f8ca81a 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -182,6 +182,19 @@ expression: initialization_result ] } } + }, + "experimental": { + "repl": { + "events": true, + "module": true, + "sessionFile": true + }, + "run": { + "coverage": true, + "inBuild": true, + "launcher": true, + "profile": true + } } }, "serverInfo": { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 2b639cc041..ad0f8ca81a 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -182,6 +182,19 @@ expression: initialization_result ] } } + }, + "experimental": { + "repl": { + "events": true, + "module": true, + "sessionFile": true + }, + "run": { + "coverage": true, + "inBuild": true, + "launcher": true, + "profile": true + } } }, "serverInfo": { diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 2fb0668911..913764829b 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1170,6 +1170,26 @@ continued with `\` — or `invalid`, input no further line can make valid, which runs to report why. it is the answer `by repl` gives itself for the lines it reads, and needs no document open +## what `by` takes on its command line + +an editor starts `by` itself for some things — a console's `by repl`, a run +configuration's `by run` — and has to know which of their flags the `by` it has +takes. the language server is that `by`, so `initialize` answers it under +`experimental`, rather than the editor running `by` to read its `--help`: each +command by name, and each flag the editor can start it with by its long name in +camelCase, `true` + +```json +{ + "repl": { "events": true, "module": true, "sessionFile": true }, + "run": { "coverage": true, "inBuild": true, "launcher": true, "profile": true } +} +``` + +`sessionFile` is `--session-file`. a flag that is not named — or an answer with +no `experimental` at all, from a `by` before this — is one this `by` does not +take + ## suppressing a diagnostic every diagnostic has two quick fixes that silence it rather than fix it. diff --git a/docs/basedpython/features/repl.md b/docs/basedpython/features/repl.md index e3ae8eda93..e285efed2a 100644 --- a/docs/basedpython/features/repl.md +++ b/docs/basedpython/features/repl.md @@ -148,7 +148,10 @@ python's, and an editor is told when that is with [`--events`](#in-an-editor) ## in an editor -an editor's console runs `by repl` as a process and talks to it through its standard streams: +an editor's console runs `by repl` as a process and talks to it through its standard streams. +which of its flags the `by` it has takes — `--events` and `--session-file` below, `--module` +above — is in the language server's answer to `initialize`, under `experimental.repl`: see +[what `by` takes on its command line](editor.md#what-by-takes-on-its-command-line) - **sending.** each input is written to `by repl`'s standard input framed, as above, followed by a newline. a line for the program — what `input()` reads — is written as it is, unframed From ba2deec4956861babf95b7a1e21b7926f40213e1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:57:48 +1000 Subject: [PATCH 098/110] compile a test module and a conftest that pytest's assertion rewriter would load as the rewriter compiles them and then under the .by's path and lines, as by run compiles every other module, rather than leaving them to the rewriter, which compiled them as the staged .py at the generated python's lines, so that their top level, class bodies, comprehensions and lambdas ran at the staged .py and a debugger never stopped at a breakpoint on one of those lines of a .by test module --- crates/ty/src/by_commands.rs | 159 ++++++++++++++++----- crates/ty/src/by_coverage.rs | 6 +- crates/ty/tests/by_e2e.rs | 46 ++++++ docs/basedpython/cli-reference.md | 5 +- docs/basedpython/development/sourcemaps.md | 54 ++++--- docs/basedpython/frameworks/pytest.md | 1 + 6 files changed, 206 insertions(+), 65 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 79aec558e8..500dc6b808 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -2377,6 +2377,7 @@ if __name__ == "__main__": const BY_PYTEST_PLUGIN_FILENAME: &str = "_by_pytest.py"; const BY_PYTEST_PLUGIN_SRC: &str = r###"# generated by `by run` — reports a test at the `.by` line it failed at, for pytest +import ast import os import pprint import sys @@ -2384,6 +2385,7 @@ import traceback import types import pytest +from _pytest.assertion.rewrite import AssertionRewritingHook, rewrite_asserts import _by_runner @@ -2391,6 +2393,21 @@ import _by_runner _REPORTER = "BY_TEST_REPORTER" +def _rewritten(hook, source, filename): + # a test module, or a `conftest`, as pytest's assertion rewriter compiles it, each + # `assert` rewritten to explain how it failed + tree = ast.parse(source, filename=filename) + rewrite_asserts(tree, source, filename, hook.config) + return compile(tree, filename, "exec", dont_inherit=True) + + +# pytest compiles a test module itself to rewrite its assertions, and installs the +# rewriter before it imports this plugin, so before any test module or `conftest`. the +# runner compiles each such module as the rewriter does and moves all of it to its +# `.by`, its top level included, as it does every other module of the project +_by_runner.compile_like(AssertionRewritingHook, _rewritten) + + def pytest_pycollect_makeitem(collector): # called for each name of a test module or class pytest collects, once the module is # imported and before any of its tests runs @@ -2404,10 +2421,9 @@ def pytest_plugin_registered(plugin): def pytest_exception_interact(node, call, report): - # an error raised while a test module is imported comes from its top level, which - # ran as pytest compiled it, at lines of the generated python, before - # `reline_module` saw the module. it is reported as `by run` reports an error in - # a program, as python prints it with each frame at its `.by` line + # an error raised while a test module is imported comes from its top level. it is + # reported as `by run` reports an error in a program, as python prints it with + # each frame at its `.by` line if not isinstance(report, pytest.CollectReport) or call.excinfo is None: return error = call.excinfo.value @@ -2417,7 +2433,7 @@ def pytest_exception_interact(node, call, report): error = error.__cause__ if not isinstance(error, ImportError): return - if _by_runner.runs_generated_lines(error.__traceback__): + if _by_runner.runs_transpiled_code(error.__traceback__): report.longrepr = _Traceback(error) @@ -3035,12 +3051,13 @@ def format_exception(etype, evalue, tb): return "".join(out) -def runs_generated_lines(tb): - """Whether a frame of `tb` runs a transpiled module's code at the lines of the - generated python rather than of its `.by`: code a test runner compiled, run - before `reline_module` moved it — a test module's own top level.""" +def runs_transpiled_code(tb): + """Whether a frame of `tb` runs a transpiled module's code: at the lines of its + `.by`, as `_by_code` compiled it, or at those of the generated python, as a loader + `reline_module` had yet to reach compiled it.""" while tb is not None: - if os.path.realpath(tb.tb_frame.f_code.co_filename) in _BY_MAP: + filename = os.path.realpath(tb.tb_frame.f_code.co_filename) + if filename in _BY_MAP or filename in _BY_SOURCES: return True tb = tb.tb_next return False @@ -3061,15 +3078,16 @@ def _project_file_of(origin): return _CARRIED.get(real) if entry is None else entry[1][0] -def _project_code(origin): - """The staged module at `origin` compiled as the project file it stands for, or - `None` when it stands for none, which leaves it to python's own loader. +def _project_code(origin, compile_source): + """The staged module at `origin` compiled by `compile_source` as the project file + it stands for, or `None` when it stands for none, which leaves it to be compiled + where it is. A hand-written `.py` is a copy, so it is compiled from its text under the project file's path: what python derives from the code object — a traceback, `inspect`, what coverage.py measures — names the file the author wrote, as it does for a `.by`.""" - code = _by_code(origin) + code = _by_code(origin, compile_source) if code is not None: return code project_file = _CARRIED.get(os.path.realpath(origin)) if origin else None @@ -3077,7 +3095,7 @@ def _project_code(origin): return None with open(origin, "rb") as handle: source = handle.read() - return compile(source, project_file, "exec", dont_inherit=True) + return compile_source(source, project_file) def _varint(out, value): @@ -3226,18 +3244,13 @@ def project_location(path, line): def reline_module(module): """Every function a staged module compiled by another loader defines, moved to the `.by` the module was transpiled from, as `_by_code` compiles the modules no other - loader stands behind. - - The other loader is a test runner's, which compiles a test module itself to rewrite - its assertions. A failure in a test is then reported from frames of the staged `.py`, - at lines of the generated python and quoting it — `tests/test_shapes.py:129`, in a - file of 24 lines. Once its functions are relined, the report names the `.by`, at the - line that failed, and quotes the `.by`. + loader stands behind. The other loader is an import hook that compiles a module itself + and that `compile_like` was not told how to compile like. Only the code a test runs is reached: the module's functions, the methods of the classes it defines, and what a decorator wraps, each with the functions nested in it. - The module's own top-level code ran as it was imported, before this, so an error - raised while importing it still names the staged `.py`. + The module's top level ran as it was imported, so an error raised while importing it + still names the staged `.py`. """ if id(module) in _RELINED_MODULES: return @@ -3289,8 +3302,15 @@ def reline_module(module): _RELINED_MODULES = set() -def _by_code(origin): - """The staged `.py` at `origin` compiled as the `.by` it was transpiled from. +def _compiled(source, filename): + # a module compiled as python's own loader compiles it + return compile(source, filename, "exec", dont_inherit=True) + + +def _by_code(origin, compile_source): + """The staged `.py` at `origin` compiled by `compile_source` — `_compiled`, or as + the loader `compile_like` names compiles it — and moved to the `.by` it was + transpiled from. A program reads its own locations back — `traceback.format_exc()`, the file and line `warnings` attributes a warning to, `co_filename`, `inspect` — and @@ -3303,14 +3323,14 @@ def _by_code(origin): marks no part of a line a location covers, and reads what it can from the `.by` text. - The `.py` is compiled from its text, as python's own loader would, and only + The `.py` is compiled from its text, as the loader that found it would, and only then moved to the `.by`: python 3.9 to 3.11 refuse, at the interpreter's recursion limit, a syntax tree nested deeper than the source they accept. A warning the compiler raises is moved to its `.by` line the same way, and so is the `SyntaxError` it raises for one the warning filters make an error. `None` when there is no `.by` to answer with, or it has changed since it was - transpiled, which leaves the module to python's own loader. + transpiled, which leaves the module to be compiled where it is. """ lines = _by_lines(origin) if lines is None: @@ -3321,7 +3341,7 @@ def _by_code(origin): with warnings.catch_warnings(record=True) as caught: try: - code = compile(source, origin, "exec", dont_inherit=True) + code = compile_source(source, origin) except SyntaxError as error: # a warning the filters turn into an error (`-W error`) is raised by the # compiler as a `SyntaxError` at the line it would have warned about, so @@ -3354,12 +3374,13 @@ class _ByLoader: hand-written `.py`, is the answer to every question those paths are asked, so it is what the module reports. - This is the loader of a module some other loader stands behind — a test runner - that rewrites assertions as it compiles. It is left to that loader, and only its - `__file__` moves; its frames are rewritten to `.by` lines when they reach the - excepthook, and pytest's plugin, `_by_pytest`, moves its functions to the `.by` - with `reline_module` before a test runs. A module python's own loader would - compile gets `_BySourceLoader`. + This is the loader of a module some other loader stands behind that + `compile_like` was not told how to compile like: an import hook that compiles + the module itself. It is left to that loader, and only its `__file__` moves; its + frames are rewritten to `.by` lines when they reach the excepthook, and pytest's + plugin, `_by_pytest`, moves its functions to the `.by` with `reline_module` + before a test runs. A module python's own loader would compile gets + `_BySourceLoader`, and one a loader `compile_like` names would, `_ByCompiledLike`. `__file__` is set as the module is created, and `exec_module` is not wrapped: the inner loader's own is the one python calls. A wrapper's frame between @@ -3409,12 +3430,40 @@ class _BySourceLoader(importlib.machinery.SourceFileLoader): module = sys.modules.get(fullname) if module is not None and getattr(module, "__loader__", None) is self: module.__file__ = self._by_path - code = _project_code(self.get_filename(fullname)) + code = _project_code(self.get_filename(fullname), _compiled) if code is not None: return code return super().get_code(fullname) +class _ByCompiledLike(importlib.machinery.SourceFileLoader): + """The loader of a staged module a loader `compile_like` names would load, which + compiles the module itself — pytest's assertion rewriter. + + It compiles the module with `_project_code` as that loader compiles it, so every + code object of a `.by` — the module's top level, a class body, a comprehension, a + lambda, as much as a function — is the `.by`'s, at its lines, from the moment it + is compiled: a debugger stops at a breakpoint on any line of it, and python + reports any frame of it at its `.by` line, as for a module `_BySourceLoader` + loads. Its `__file__` is left the staged path the other loader would have given + it: pytest checks that a test module it imported is the file it collected, and + the staged `.py` is that file. + """ + + def __init__(self, fullname, path, compile_source): + super().__init__(fullname, path) + self._by_compile_source = compile_source + + def get_code(self, fullname): + path = self.get_filename(fullname) + code = _project_code(path, self._by_compile_source) + if code is not None: + return code + # a `.by` saved over since it was transpiled: compiled where it is, still as + # the other loader would compile it + return self._by_compile_source(self.get_data(path), path) + + class _ByFinder: """Puts a loader of its own on every staged module, whoever ends up finding it.""" @@ -3429,14 +3478,45 @@ class _ByFinder: if spec is None: continue by_path = _project_file_of(getattr(spec, "origin", None)) + compile_source = _COMPILERS.get(type(spec.loader)) if by_path is not None and type(spec.loader) is importlib.machinery.SourceFileLoader: spec.loader = _BySourceLoader(spec.loader.name, spec.loader.path, by_path) + elif by_path is not None and compile_source is not None: + loader = spec.loader + spec.loader = _ByCompiledLike( + spec.name, + spec.origin, + lambda source, filename: compile_source(loader, source, filename), + ) elif by_path is not None and spec.loader is not None: spec.loader = _ByLoader(spec.loader, by_path) return spec return None +# the types of loader that compile a module themselves, each with how it compiles one, +# from `compile_like` +_COMPILERS = {} + + +def compile_like(loader_type, compile_source): + """Load each staged module a loader of `loader_type` would load with + `_ByCompiledLike`, compiled as `compile_source(loader, source, filename)` answers + — the code object a loader of that type would execute — and then moved to the + `.by`. + + Such a loader is an import hook that puts itself at the head of `sys.meta_path`, + as pytest's assertion rewriter does before it loads the plugin that calls this, + and a finder there answers ahead of `_ByFinder`. So `_ByFinder` is put back at + the head, where it asks every other finder in turn and sees what they answer. + """ + _COMPILERS[loader_type] = compile_source + ours = [finder for finder in sys.meta_path if isinstance(finder, _ByFinder)] + sys.meta_path[:] = ours + [ + finder for finder in sys.meta_path if not isinstance(finder, _ByFinder) + ] + + def _entry_spec(module): """The spec whose code `by run ` should execute. @@ -3576,7 +3656,8 @@ def _write_coverage(measurement, report): coverage.py records a module by the filename and lines of its code objects. one `_by_code` compiled is already the `.by` with its lines, and one another loader - compiled — a test runner rewriting assertions — is the staged `.py` with its own. + compiled — an import hook `compile_like` was not told of — is the staged `.py` + with its own. either way the statements are the generated python's, so each module's lines are put back on its `.py`, analysed there, and only then moved through the sourcemap to the `.by`. a `.by` line is covered when any statement written for it ran. @@ -3757,8 +3838,8 @@ def _write_profile(profiler, snapshot): function of a staged module named by the project file and line it was written at. A function is the file, first line and name of its code object. One `_by_code` - compiled is the `.by` already; one another loader compiled — a test module's own - top level, which pytest compiles — is the staged `.py`, at the generated python's + compiled is the `.by` already; one another loader compiled — an import hook + `compile_like` was not told of — is the staged `.py`, at the generated python's lines, and moves to its `.by` here, as a staged copy of a hand-written `.py` moves to the project file. The two can then name one function, and are added up as one. A line the transpiler wrote on its own has no `.by` line, and is line 0. diff --git a/crates/ty/src/by_coverage.rs b/crates/ty/src/by_coverage.rs index aa1b49383e..00beb5ab9b 100644 --- a/crates/ty/src/by_coverage.rs +++ b/crates/ty/src/by_coverage.rs @@ -5,9 +5,9 @@ //! the runner `by run` stages measures the program, moves each transpiled //! module's lines to its `.by` through the sourcemap it already holds, and writes //! what it found as JSON (see `_write_coverage` in the runner). A `.py` the -//! project carries over runs as its project file, but one a test runner compiled -//! itself runs as the copy it compiled, in a temporary directory that is deleted -//! when the run ends. The staging knows what each copy is a copy of, so the +//! project carries over runs as its project file, but one an import hook other +//! than pytest's assertion rewriter compiled itself runs as the copy it compiled, +//! in a temporary directory that is deleted when the run ends. the staging knows what each copy is a copy of, so the //! report is finished here: every staged copy is put back on the file it was //! copied from, and the runner's own files, which belong to no project, are left //! out. diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index d5d86c672c..a41b9dcfae 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2791,6 +2791,52 @@ fn run_pytest_reports_an_error_importing_a_test_module_at_the_by_line() { } } +/// pytest compiles a test module, and a `conftest`, itself, to rewrite its assertions. +/// what ran as the module was imported — its top level, a class body, a comprehension +/// or a lambda there — ran as pytest compiled it, named the staged `.py` and at the +/// lines of the generated python, so a debugger never stopped at a breakpoint on any +/// of those lines of the `.by`. the runner compiles such a module as pytest does, and +/// every line of it runs as the `.by` it was transpiled from, at that line +#[test] +fn run_pytest_runs_all_of_a_test_module_at_its_by_lines() { + let dir = tempfile::tempdir().expect("tempdir"); + let Some(python) = pytest_project(dir.path()) else { + return; + }; + fs::write( + dir.path().join("tests/test_where.by"), + "import os\nimport sys\n\n\ndef record(name: str):\n let frame = sys._getframe(1)\n with open(os.environ[\"WHERE_OUT\"], \"a\") as out:\n out.write(f\"{name} {frame.f_code.co_filename}:{frame.f_lineno}\\n\")\n\n\nrecord(\"module\")\n\n\nclass TestWhere:\n record(\"class\")\n\n def test_method(self):\n record(\"method\")\n\n\nlet listed = [record(\"comprehension\") for _ in range(1)]\nlet called = (lambda: record(\"lambda\"))()\n", + ) + .unwrap(); + fs::write( + dir.path().join("tests/conftest.by"), + "import os\nimport sys\n\nimport pytest\n\nwith open(os.environ[\"WHERE_OUT\"], \"a\") as out:\n out.write(f\"conftest {sys._getframe().f_code.co_filename}:{sys._getframe().f_lineno}\\n\")\n\n\n@pytest.fixture\ndef broken() -> int:\n let value = 0\n assert value == 1\n return value\n", + ) + .unwrap(); + let out = dir.path().join("where.txt"); + let project = dir.path().canonicalize().unwrap(); + + let stdout = run_pytest( + dir.path(), + &python, + &[("WHERE_OUT", out.to_str().expect("a utf-8 path"))], + ); + let recorded = fs::read_to_string(&out) + .unwrap_or_else(|error| panic!("nothing was recorded ({error}):\n{stdout}")); + let tests = project.join("tests"); + let expected = [ + ("conftest", tests.join("conftest.by"), 7), + ("module", tests.join("test_where.by"), 11), + ("class", tests.join("test_where.by"), 15), + ("comprehension", tests.join("test_where.by"), 21), + ("lambda", tests.join("test_where.by"), 22), + ("method", tests.join("test_where.by"), 18), + ] + .map(|(name, file, line)| format!("{name} {}:{line}", file.display())) + .join("\n"); + assert_eq!(recorded.trim_end(), expected, "{stdout}"); +} + /// with `BY_TEST_REPORTER=teamcity` the run is reported as the `TeamCity` service /// messages an IDE's test runner reads: suites named by the `.by` files and the /// classes in them, each test with its node id and its line in the `.by`, a failing diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 7a58f63bdf..50c97f63f0 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -129,8 +129,9 @@ coverage.py configuration (`[tool.coverage]`, `.coveragerc`) applies, except tha no `.coverage` data file is written: the report is the result. its `source`, `include` and `omit` name the project's files as they are in the project, with `--in-build` too, since every module runs as the file it was written as. the one -exception is a module a test runner compiled itself, which runs as the staged -copy it compiled: a `source` or an `omit` naming it does not reach it. a `.by` +exception is a module an import hook other than pytest's assertion rewriter +compiled itself, which runs as the staged copy it compiled: a `source` or an +`omit` naming it does not reach it. a `.by` module under `source` that never ran is reported as not run, as coverage.py reports a `.py` that never ran. a program that leaves without returning to `by run` — `os._exit` — leaves no report, and diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index 8c524a8877..8b3a564c06 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -114,27 +114,39 @@ that machinery and the module's code, and a wrapper's `exec_module` did: a modul that failed as it was imported showed `` three times and the shim once above the line that failed, in every traceback no excepthook rewrites. it now shows what python shows for a program it runs itself, the line -that imported and the line that failed. a module some other loader stands behind -— a test runner that rewrites assertions — keeps that loader, which python calls -directly; only its `__file__` is set, as the module is created - -pytest is that test runner, and it reports a failure from the frames it caught, -not through the excepthook, so its report used to name the staged `.py` at a line -of the generated python and quote the generated python. `by run` puts a pytest -plugin in the tree, `_by_pytest`, declared by a `pytest11` entry point in a -`_by_pytest-0.dist-info` beside it, so pytest loads it wherever the tree is on -`sys.path`. once pytest has imported a test module or a `conftest`, and before any -of its tests runs, the plugin moves the code of every function the module defines -— the methods of its classes, what a decorator wraps, a fixture's function, and -the functions nested in each — to the `.by` lines, as the shim's own loader does. -a failure in a test or a fixture is then reported at the `.by` line, quoting the -`.by`. the module's top level ran as it was imported, before the plugin saw it, at -the lines of the generated python: an error it raises, or an import in it fails -with, is reported in place of pytest's report of it as `by run` reports an error in -a program, python's traceback with each frame at its `.by` line. pytest's node ids -still name the staged `.py`, `tests/test_calc.py::test_sum`: they are what pytest -is given back to run a test again, and what its cache and its `--deselect` read, -and they have to name a file in the tree pytest runs in +that imported and the line that failed + +pytest compiles a test module and a `conftest` itself, to rewrite their +assertions, from an import hook it puts at the head of `sys.meta_path`. left to +itself, that hook compiles the staged `.py` at the lines of the generated python: a +failure would be reported naming it, and a debugger would never stop at a +breakpoint in the `.by`. `by run` puts a pytest plugin in the tree, `_by_pytest`, declared by a +`pytest11` entry point in a `_by_pytest-0.dist-info` beside it, so pytest loads it +wherever the tree is on `sys.path` — after it installs its hook and before it +imports any test module or `conftest`. the plugin tells the shim how pytest's hook +compiles a module, and the shim puts its own finder back ahead of the hook. a +staged module the hook would load is then compiled as the hook compiles it, its +assertions rewritten, and moved to the `.by` lines like any other: its top level, a +class body, a comprehension and a lambda as much as a function, from the moment it +is compiled. a failure in a test or a fixture is reported at the `.by` line, +quoting the `.by`, a breakpoint on any line of a test module stops there, and an +error the module's top level raises, or an import in it fails with, is reported in +place of pytest's report of it as `by run` reports an error in a program, python's +traceback with each frame at its `.by` line. a hand-written `.py` the build carries +over is compiled under the project file's path the same way. the module's +`__file__` stays the staged `.py`, as do pytest's node ids, +`tests/test_calc.py::test_sum`: pytest checks that a module it imported is the file +it collected, and a node id is what pytest is given back to run a test again, and +what its cache and its `--deselect` read, so both have to name a file in the tree +pytest runs in + +a module some other import hook compiles itself keeps that hook's loader, which +python calls directly; only its `__file__` is set, as the module is created. under +pytest, once pytest has imported such a module and before any of its tests runs, +the plugin moves the code of every function the module defines — the methods of +its classes, what a decorator wraps, a fixture's function, and the functions nested +in each — to the `.by` lines. its top level ran as it was imported, before the +plugin saw it, at the lines of the generated python both tables are keyed by the generated path exactly as the map spells it. the shim resolves symlinks before matching a frame's filename against it — a diff --git a/docs/basedpython/frameworks/pytest.md b/docs/basedpython/frameworks/pytest.md index e450bdbbd9..716811692e 100644 --- a/docs/basedpython/frameworks/pytest.md +++ b/docs/basedpython/frameworks/pytest.md @@ -25,6 +25,7 @@ basedpython understands pytest's fixture system. fixture parameters are typed fr - fixture functions written in `.by` work correctly: decorators survive, parameter names survive, type injection works at runtime - `yield` fixtures transpose correctly - `?:` and other basedpython features work inside test bodies and fixtures +- under `by run pytest`, every line of a `.by` test module or `conftest` runs as that line of the `.by` — its top level, class bodies, comprehensions and lambdas as much as its tests — with pytest's assertion rewriting intact, so a failure is reported, and a debugger stops at a breakpoint, at the line as written ## limitations and workarounds From 3e0c64e770f88d89a388f44ee8c28695f9228176 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:10:06 +1000 Subject: [PATCH 099/110] charge each line of the __main__ guard the transpiler appends to the line main is defined on, which is what asks for the call, so that while main runs a traceback names that line for the module's frame and quotes def main, and a debugger's stack shows the module there, where both named line 0 of the .by; and start a module's code at line 1 as python's own does, so that a profile names it main.by:1() rather than line 0 when the transpiler wrote the module's first line --- crates/by_transforms/src/lib.rs | 88 ++++++++++++++----- .../src/transforms/ast_driver.rs | 46 ++++++++-- .../src/transforms/main_function.rs | 26 ++++-- .../by_transforms/src/transforms/modifiers.rs | 8 +- crates/ty/src/by_commands.rs | 5 +- crates/ty/tests/by_e2e.rs | 52 +++++++++-- .../development/how-transpilation-works.md | 6 +- docs/basedpython/features/main-function.md | 31 +++++++ 8 files changed, 215 insertions(+), 47 deletions(-) diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 26581c5ed9..df93669338 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -3251,20 +3251,30 @@ mod cross_file { ); } - /// The map for one source, spelled with and without a final newline. - fn map_both_ways(body: &str) -> (Vec>, Vec>) { - let mut maps = Vec::new(); + /// the map for one source, spelled with and without a final newline, and the + /// python the one without is transpiled to + fn map_both_ways(body: &str) -> (String, Vec>, Vec>) { + let mut transpiled = Vec::new(); for src in [body.to_owned(), format!("{body}\n")] { let project = project_db(&[("/m.by", &src)]); let file = system_path_to_file(project.db(), "/m.by").expect("file not in db"); - let (_, map) = + transpiled.push( transpile_typed_with_map(project.db(), file, &Config::test_default(), None) - .expect("transpile failed"); - maps.push(map); + .expect("transpile failed"), + ); } - let with_newline = maps.pop().expect("two maps"); - let without = maps.pop().expect("two maps"); - (without, with_newline) + let (_, with_newline) = transpiled.pop().expect("two maps"); + let (out, without) = transpiled.pop().expect("two maps"); + (out, without, with_newline) + } + + /// the entry of `map` for the line of `out` that is `line` + fn mapped(out: &str, map: &[Option], line: &str) -> Option { + let index = out + .lines() + .position(|written| written == line) + .unwrap_or_else(|| panic!("`{line}` is not in the output:\n{out}")); + map[index] } /// A trailing newline terminates the last line; it does not add one. So the @@ -3279,19 +3289,18 @@ mod cross_file { #[test] fn a_missing_final_newline_does_not_cost_the_last_line_its_mapping() { let body = "def g():\n print(2)\n\ndef main():\n g()"; - let (without, with_newline) = map_both_ways(body); + let (out, without, with_newline) = map_both_ways(body); assert_eq!( without, with_newline, "the two spellings of one program must map alike" ); - // the last line the user wrote is the last line that maps to source, and - // it maps to itself: ` g()` is `.by` line 4 - let last_mapped = without - .iter() - .rposition(Option::is_some) - .expect("some line maps to source"); - assert_eq!(without[last_mapped], Some(4), "map:\n{without:?}"); + // the last line the user wrote maps to itself: ` g()` is `.by` line 4 + assert_eq!( + mapped(&out, &without, " g()"), + Some(4), + "map:\n{without:?}" + ); } /// the same, through the enum lowering — a second table producer, which @@ -3299,23 +3308,56 @@ mod cross_file { #[test] fn a_missing_final_newline_maps_the_last_line_through_the_enum_lowering() { let body = "enum class Colour:\n case Red\n case Green\n\ndef main():\n print(Colour.Red)"; - let (without, with_newline) = map_both_ways(body); + let (out, without, with_newline) = map_both_ways(body); assert_eq!( without, with_newline, "the two spellings of one program must map alike" ); - let last_mapped = without - .iter() - .rposition(Option::is_some) - .expect("some line maps to source"); assert_eq!( - without[last_mapped], + mapped(&out, &without, " print(Colour.Red)"), Some(5), "` print(Colour.Red)` is .by line 5, map:\n{without:?}" ); } + /// the `__main__` guard the transpiler appends is the call of `main` the author + /// asked for by writing `main`, so each line of it is charged to the line `main` is + /// defined on, which a traceback and a debugger then name for the module's frame + /// while `main` runs, where they named line 0 of the `.by` + #[test] + fn the_main_guard_is_charged_to_the_line_main_is_defined_on() { + for (src, defined_on) in [ + // a function defined after `main`, so the guard follows lines of another + ( + "import sys\n\n\ndef main():\n helper()\n\n\ndef helper():\n print(sys.argv)\n", + 3, + ), + // a command line, parsed by lines of the guard's own ahead of the call + ( + "def main(name: str, count: int = 1):\n print(name * count)\n", + 0, + ), + // `asyncio.run`, and the import of `asyncio` the guard needs + ("async def main():\n print(1)\n", 0), + ] { + let project = project_db(&[("/m.by", src)]); + let file = system_path_to_file(project.db(), "/m.by").expect("file not in db"); + let (out, map) = + transpile_typed_with_map(project.db(), file, &Config::test_default(), None) + .expect("transpile failed"); + let guard = out + .lines() + .position(|line| line == "if __name__ == \"__main__\":") + .unwrap_or_else(|| panic!("no guard in:\n{out}")); + assert_eq!( + map[guard..], + vec![Some(defined_on); out.lines().count() - guard], + "the guard of:\n{out}\nis not charged to `.by` line {defined_on}: {map:?}" + ); + } + } + /// the map is indexed by generated line, so it has to be exactly as long as /// the generated python has lines — an entry short and every lookup past the /// gap answers for its neighbour diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index d98374c1ec..b08435f4c9 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -117,6 +117,36 @@ fn quote_delimiter(text: &str) -> &'static str { .unwrap_or("\"") } +/// a line of [`PassContext::epilogue`], with the source line it is charged to in the +/// line table +pub(crate) struct EpilogueLine { + text: String, + /// the offset in the source of what the line runs on the author's behalf, whose + /// line every location in it then names: a traceback, a debugger's stack and a + /// profile read it from the code object `by run` compiles, and a debugger stops + /// there. `None` for a line no one place in the source asks for, such as + /// `__all__`, which every `export` adds a name to + charged_to: Option, +} + +impl EpilogueLine { + /// a line no line of the source stands for + pub(crate) fn unattributed(text: String) -> Self { + Self { + text, + charged_to: None, + } + } + + /// a line that runs what the source written at `at` asks for + pub(crate) fn charged_to(text: String, at: TextSize) -> Self { + Self { + text, + charged_to: Some(at), + } + } +} + /// Mutable state shared across every pass during a single transpile. #[derive(Default)] pub(crate) struct PassContext { @@ -187,7 +217,7 @@ pub(crate) struct PassContext { pub(crate) errors: Vec, /// Lines to append AFTER the spliced body (e.g. modifiers' auto- /// generated `__all__ = [...]`). Driver emits each as its own line - pub(crate) epilogue: Vec, + pub(crate) epilogue: Vec, /// The modules a *synthesized* annotation qualifies a class with that the source /// never imported under their own names (`decimal` for an inferred /// `decimal.Decimal` annotation). The annotation is written as a string @@ -2254,7 +2284,8 @@ pub(crate) fn run_against_source<'a>( ctx.errors.extend(lost); // line table for the spliced body, built from the ascending edit list // before the descending application sort consumes it. generated lines from - // the import prefix (top) and epilogue (bottom) have no source origin + // the import prefix (top) have no source origin, and those of the epilogue + // (bottom) the one each names let mut body_edits = edits.clone(); body_edits.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); let body_table = crate::source_map::line_table(source_ref, &body_edits); @@ -2273,7 +2304,7 @@ pub(crate) fn run_against_source<'a>( // under a name of the lowering's own, bound here, ahead of everything that reads it let everything_written = std::iter::once(out.as_str()) .chain(ctx.required_imports.iter().map(String::as_str)) - .chain(ctx.epilogue.iter().map(String::as_str)) + .chain(ctx.epilogue.iter().map(|line| line.text.as_str())) .collect::>() .join("\n"); let builtins = written.builtin_imports(&everything_written, config.min_version.minor); @@ -2323,10 +2354,15 @@ pub(crate) fn run_against_source<'a>( out.push('\n'); } for line in &ctx.epilogue { - out.push_str(line); + out.push_str(&line.text); out.push('\n'); } - table.extend(std::iter::repeat_n(None, ctx.epilogue.len())); + table.extend(ctx.epilogue.iter().map(|line| { + line.charged_to.map(|at| { + u32::try_from(crate::newline_count(&source_ref[..usize::from(at)])) + .unwrap_or(u32::MAX) + }) + })); } // normalise trailing newline. AST-mutation passes splice rendered // multi-line statements that may bring their own internal newlines; diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index 60d438dba4..819f20c8fa 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -20,7 +20,7 @@ use ruff_python_ast::{ self as ast, CmpOp, Expr, ModModule, ParameterWithDefault, Parameters, Stmt, StmtFunctionDef, }; -use super::ast_driver::{AstPass, PassContext}; +use super::ast_driver::{AstPass, EpilogueLine, PassContext}; use super::repeated_underscore::{WrittenNames, positional_only_count}; use super::source_util::{is_synthetic_decorator, python_string_literal}; @@ -62,7 +62,12 @@ impl AstPass for MainFunction<'_> { return; } - ctx.epilogue.push("if __name__ == \"__main__\":".to_owned()); + // the guard is the call of `main` the author asked for by writing `main`, so + // it runs at the line `main` is defined on: a traceback and a debugger's stack + // name that line for the module's frame, where a `.py` program names its own + // call of `main`, and a debugger stops at a breakpoint on it both as `main` + // is defined and as it is called + let mut guard = vec!["if __name__ == \"__main__\":".to_owned()]; let spec: Vec = entry .exposed() .map(|parameter| parameter.spec_entry(self.written)) @@ -72,28 +77,33 @@ impl AstPass for MainFunction<'_> { "main()".to_owned() } else { ctx.runtime.insert(crate::runtime::MAIN_ARGS); - ctx.epilogue - .push(" _by_args, _by_kwargs = _by_main_args(main, [".to_owned()); + guard.push(" _by_args, _by_kwargs = _by_main_args(main, [".to_owned()); for entry in spec { - ctx.epilogue.push(format!(" {entry},")); + guard.push(format!(" {entry},")); } let close = match extra { Some(converter) => format!(" ], {})", converter.spelled(self.written)), None => " ])".to_owned(), }; - ctx.epilogue.push(close); + guard.push(close); "main(*_by_args, **_by_kwargs)".to_owned() }; if entry.function.is_async { - ctx.epilogue.push(format!( + guard.push(format!( " {}.run({call})", self.written.imported_module("asyncio") )); ctx.required_imports .push(self.written.import_module("asyncio")); } else { - ctx.epilogue.push(format!(" {call}")); + guard.push(format!(" {call}")); } + let defined_at = entry.function.name.range.start(); + ctx.epilogue.extend( + guard + .into_iter() + .map(|line| EpilogueLine::charged_to(line, defined_at)), + ); } } diff --git a/crates/by_transforms/src/transforms/modifiers.rs b/crates/by_transforms/src/transforms/modifiers.rs index d5d7bd74b1..7cee082298 100644 --- a/crates/by_transforms/src/transforms/modifiers.rs +++ b/crates/by_transforms/src/transforms/modifiers.rs @@ -39,7 +39,7 @@ use ruff_python_stdlib::basedpython::visibility_rename; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use std::collections::HashMap; -use super::ast_driver::{AstPass, Fragment, PassContext}; +use super::ast_driver::{AstPass, EpilogueLine, Fragment, PassContext}; use super::repeated_underscore::WrittenNames; use crate::type_info::TypeInfo; use crate::{Config, PythonVersion}; @@ -939,8 +939,10 @@ impl AstPass for ModifiersPass<'_> { .map(|name| format!("\"{name}\"")) .collect::>(); if !entries.is_empty() { - ctx.epilogue - .push(format!("__all__ = [{}]", entries.join(", "))); + ctx.epilogue.push(EpilogueLine::unattributed(format!( + "__all__ = [{}]", + entries.join(", ") + ))); } // the members tuple is built as the module runs, and a stub never runs. it diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 500dc6b808..df234e30d5 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -3129,7 +3129,10 @@ def _relined_one(code, filename, by_line, width): _relined(const, filename, by_line, width) if isinstance(const, types.CodeType) else const for const in code.co_consts ) - first = by_line(code.co_firstlineno) + # a module's code starts at its first line whatever is written there, as python's + # own does, rather than at line 0 when the transpiler wrote that line — an import + # it needed — on its own. a profile names the module by it, `main.by:1()` + first = 1 if code.co_name == "" else by_line(code.co_firstlineno) table = bytearray() if sys.version_info >= (3, 11): # an entry for each run of up to 8 code units at one line, in the long form, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index a41b9dcfae..510327189b 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3141,6 +3141,46 @@ main() ); } +/// the `__main__` guard the transpiler appends calls `main` at the line `main` is +/// defined on, so a traceback names that line for the module's frame, and quotes it, +/// where a `.py` program's names its own call of `main`. it named line 0 and quoted +/// nothing. the module itself starts at line 1 whatever the transpiler wrote there — +/// the import of `Final` a `let` needs — as python's own module does, which is the +/// line a profile names it by, where it named line 0 +#[test] +fn run_reports_the_call_of_main_at_the_line_main_is_defined_on() { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "let numbers: dict[str, int] = {}\n\n\ndef main():\n print(numbers[\"missing\"])\n", + ) + .unwrap(); + let main = dir.path().canonicalize().unwrap().join("main.by"); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--python", &python, "--profile", "run.pstat", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + // python writes its traceback in text mode, which ends a line with `\r\n` on windows + let stderr = String::from_utf8_lossy(&output.stderr).replace("\r\n", "\n"); + assert!( + stderr.contains(&format!( + " File \"{0}\", line 4, in \n def main():\n File \"{0}\", line 5, in main\n", + main.display() + )), + "the call of `main` is not at the line `main` is defined on:\n{stderr}" + ); + let functions = profiled_functions(&python, &dir.path().join("run.pstat")); + let module = functions + .iter() + .find(|function| function.file == main.display().to_string() && function.name == "") + .unwrap_or_else(|| panic!("no of main.by: {functions:#?}")); + assert_eq!(module.line, 1, "{functions:#?}"); +} + /// inference recurses with the shape of the expression it is checking, and `run` /// checks on the thread it was dispatched to rather than through the rayon pool. /// on the stack a process starts with — 1 MiB on windows — a file like this one @@ -5425,8 +5465,7 @@ fn a_source_without_a_trailing_newline_still_maps_its_last_line() { "one entry per generated line:\n{generated}" ); - // ` print(helper())` is `.by` line 4, and it is the last line that has a - // `.by` line behind it at all + // ` print(helper())` is `.by` line 4 let printed = generated .lines() .position(|line| line.contains("print(helper())")) @@ -5437,15 +5476,16 @@ fn a_source_without_a_trailing_newline_still_maps_its_last_line() { "the user's last line maps to itself:\n{without:?}\n{generated}" ); - // and the entry-point guard `by` appended is prelude, which is what `None` - // is for — the fix must not buy the last line back by mapping those + // and the entry-point guard `by` appended is charged to the line `main` is + // defined on, `.by` line 3, which is what asks for it — the fix must not buy + // the last line back by mapping the guard onto it let epilogue = generated .lines() .position(|line| line.starts_with("if __name__ ==")) .expect("the entry-point guard is appended"); assert!( - without[epilogue..].iter().all(Option::is_none), - "the appended guard has no `.by` line behind it:\n{without:?}\n{generated}" + without[epilogue..].iter().all(|line| *line == Some(3)), + "the appended guard is not at `def main`'s line:\n{without:?}\n{generated}" ); assert!( epilogue > printed, diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index cd1fb9f9b3..ba0ab7e0e7 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -287,7 +287,11 @@ after the passes run, the driver assembles the output in one pass: `transpile_typed_with_map` returns a line table (`output line → .by line`, `None` for generated lines), composed from the phase-0 table plus the count of -generated preamble lines. it powers `by run`'s traceback rewriting and the +generated preamble lines. a line appended after the body is charged to the source +it runs for, when one place in the source asks for it: each line of the +[`__main__` guard](../features/main-function.md#the-guards-line) is charged to the +line `main` is defined on, and the `__all__` an `export` adds a name to is +generated. it powers `by run`'s traceback rewriting and the `.by` source span on transpiler-error diagnostics. the table is line-level only; the byte-accurate, bidirectional design is in [sourcemaps](sourcemaps.md) diff --git a/docs/basedpython/features/main-function.md b/docs/basedpython/features/main-function.md index a809ab5aa5..3b78171333 100644 --- a/docs/basedpython/features/main-function.md +++ b/docs/basedpython/features/main-function.md @@ -151,6 +151,37 @@ positional-only parameter is still passed positionally to `main` even when the command line named it with `--`, and a keyword-only one never takes a positional slot. +## the guard's line + +the guard is written by the transpiler, so no line of the `.by` is its own. it +runs at the line `main` is defined on, since writing `main` is what asks for it: +while `main` runs, a traceback names that line for the module's frame and quotes +`def main` there, and a debugger's stack shows the module at it, where those of a +`.py` program name its own `main()` call + +```by +let numbers: dict[str, int] = {} + + +def main(): + print(numbers["missing"]) +``` + +```console +$ by run main +Traceback (most recent call last): + File "/project/main.by", line 4, in + def main(): + File "/project/main.by", line 5, in main + print(numbers["missing"]) + ~~~~~^^^^^^^^^^^^^^^^^^^^ +KeyError: 'missing' +``` + +a breakpoint on that line stops as `main` is defined and again as the guard calls +it, as a breakpoint on a `for` line stops on each pass, and stepping over it as +the guard calls `main` runs the whole of `main` + ## the project entry point `main` makes a *module* runnable; `run.main` says which module that is for the From c8da0593fb633e16be2cf58d30d80910a94ef588 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:49:20 +1000 Subject: [PATCH 100/110] =?UTF-8?q?read=20what=20a=20function's=20body=20a?= =?UTF-8?q?sks=20of=20its=20own=20signature=20=E2=80=94=20an=20unannotated?= =?UTF-8?q?=20parameter's=20type,=20the=20default=20it=20inherits,=20the?= =?UTF-8?q?=20type=20variables=20and=20ParamSpec=20the=20signature=20binds?= =?UTF-8?q?=20=E2=80=94=20from=20parameter=5Fsignature,=20a=20tracked=20qu?= =?UTF-8?q?ery=20of=20the=20signature=20short=20of=20the=20return=20type?= =?UTF-8?q?=20recovered=20from=20the=20body,=20rather=20than=20from=20the?= =?UTF-8?q?=20raw=20signature,=20which=20fills=20that=20return=20type=20in?= =?UTF-8?q?,=20so=20that=20inferring=20a=20body=20no=20longer=20depends=20?= =?UTF-8?q?on=20the=20return=20type=20inferred=20from=20that=20same=20body?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/ty_ide/src/inlay_hints.rs | 15 +--- .../mdtest/basedpython_sound_types.md | 11 +++ .../resources/mdtest/cycle/basic.md | 10 +-- crates/ty_python_semantic/src/types.rs | 77 ------------------- .../ty_python_semantic/src/types/function.rs | 70 ++++++++++++++--- .../ty_python_semantic/src/types/generics.rs | 9 ++- .../src/types/infer/builder/function.rs | 11 +-- .../src/types/infer/tests.rs | 72 +++++++++++++++++ .../src/types/set_theoretic/builder.rs | 2 +- 9 files changed, 160 insertions(+), 117 deletions(-) diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 617331743c..0146ae3a04 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -10752,7 +10752,7 @@ Source with applied edits: ); assert_snapshot!(test.inlay_hints(), @" - def twice(value)[ -> list[Unknown]]: + def twice(value)[ -> list[value@twice]]: return [value, value] --------------------------------------------- @@ -10764,19 +10764,8 @@ Source with applied edits: info: Source --> main2.py:LL:22 | - LL | def twice(value)[ -> list[Unknown]]: + LL | def twice(value)[ -> list[value@twice]]: | ^^^^ - - info[inlay-hint-location]: Inlay Hint Target - --> stdlib/ty_extensions/_internal.pyi:LL:1 - | - LL | Unknown: _SpecialForm - | ^^^^^^^ - info: Source - --> main2.py:LL:27 - | - LL | def twice(value)[ -> list[Unknown]]: - | ^^^^^^^ "); // We still suggest edits/imports for a user-defined class named Unknown. diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md index 26fe5e607f..146f62c6ec 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md @@ -125,6 +125,17 @@ reveal_type(ident(1)) # revealed: Literal[1] reveal_type(ident("a")) # revealed: Literal["a"] ``` +the return type can mention the hole anywhere, and each call solves it from its own argument: + +```py +def twice(value): + return [value] + +reveal_type(twice) # revealed: def twice(value) -> list[value] +reveal_type(twice(1)) # revealed: list[int] +reveal_type(twice("a")) # revealed: list[str] +``` + the hole's [PEP 696](https://peps.python.org/pep-0696/) default is the parameter's own default value, so a call that omits the argument still names a type rather than leaving the hole unsolved: diff --git a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md index 5aafe59ecd..7cc58a66c1 100644 --- a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md +++ b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md @@ -758,12 +758,10 @@ reveal_type(holder()) # revealed: Box[Literal[1]] ## a string built in a loop inside a larger cycle widens rather than diverging -A loop that prepends to a string is a different literal on every trip. On its own the loop is a -cycle, and its literals widen to `str`. Here it is part of a larger one: the bound of `many`'s -unannotated parameter `x` is read off `many`'s body, and checking the call `one(x)` in that body -consults the bound. Each round of that cycle used to add one more literal — `" "`, then `" "` — -until salsa gave up on the iteration and the checker panicked. A kind of literal that keeps growing -from round to round is widened, as it would be in the loop alone. +A loop that prepends to a string is a different literal on every trip, and its literals widen to +`str`. Here the loop is also part of a larger cycle: the bound of `many`'s unannotated parameter `x` +is read off `many`'s body, and checking the call `one(x)` in that body consults the bound. The +literals widen all the same, and the iteration settles. ```py def one(x): diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 046ecf20ae..35c9033f87 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -2850,7 +2850,6 @@ impl<'db> Type<'db> { // would normalize it, and the widened and marked forms would alternate without // either being reached .without_growing_tuple_lengths(db, env, previous) - .without_growing_literals(db, env, previous) .without_growing_self_nesting(db, env, previous, cycle) .recursive_type_normalized_impl_with_cycle(db, env, cycle) } @@ -3066,82 +3065,6 @@ impl<'db> Type<'db> { UnionType::from_elements_cycle_recovery(db, env, rest) } - /// basedpython: `self`, with a family of literals the fixed-point iteration is growing - /// widened to the class they are instances of. - /// - /// A string built up in a loop is a different literal every time round: - /// - /// ```python - /// result = "" - /// while x: - /// result = " " + result - /// ``` - /// - /// The loop is ordinarily a cycle of its own, and the union builder widens it: its values - /// carry the cycle's `Divergent` marker, which marks the literals built from them as - /// recursively defined, and a union holding [`MAX_RECURSIVE_UNION_LITERALS`] of those is - /// widened to `str`. But a loop can also sit inside a larger cycle — a body whose inference - /// depends on itself, as one does when an unannotated parameter's bound is read off that very - /// body — and then the loop is no cycle of its own: the marker belongs to the outer cycle, is - /// gone after its first round, and each round adds one more literal, `" "`, then `" "`, until - /// salsa gives up on the iteration. - /// - /// So the growth is recognized where it shows, in the rounds themselves: a kind of literal - /// that already had as many members as the union builder lets a recursively defined union - /// hold, and gains more past the rounds [`crate::TAINTED_CYCLES`] leaves alone, is the - /// iteration enumerating values rather than the program naming them. That kind is widened, - /// which is what the same loop infers on its own. A kind that stopped growing is left as it - /// is, and so is one that arrives whole: `Literal["a", "b", "c", "d", "e", "f"]` written in an - /// annotation can arrive in a single round, after rounds that held fewer of its members. - /// - /// [`MAX_RECURSIVE_UNION_LITERALS`]: crate::types::set_theoretic::builder::MAX_RECURSIVE_UNION_LITERALS - fn without_growing_literals( - self, - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - previous: Self, - ) -> Self { - let literal_counts = |ty: Self| { - let mut counts: FxHashMap, usize> = FxHashMap::default(); - for element in ty.union_elements(db) { - if let Type::LiteralValue(literal) = element { - *counts - .entry(literal.fallback_instance(db, env)) - .or_default() += 1; - } - } - counts - }; - let after = literal_counts(self); - if after.is_empty() { - return self; - } - let before = literal_counts(previous); - let is_growing = |class: &Type<'db>| { - let before = before.get(class).copied().unwrap_or(0); - before >= set_theoretic::builder::MAX_RECURSIVE_UNION_LITERALS && after[class] > before - }; - if !after.keys().any(is_growing) { - return self; - } - // each growing kind takes the place of its first literal, so the union keeps its order - let mut widened: FxHashSet> = FxHashSet::default(); - let elements = self.union_elements(db).filter_map(|element| match element { - Type::LiteralValue(literal) => { - let class = literal.fallback_instance(db, env); - if !is_growing(&class) { - Some(element) - } else if widened.insert(class) { - Some(class) - } else { - None - } - } - _ => Some(element), - }); - UnionType::from_elements_cycle_recovery(db, env, elements.collect::>()) - } - /// collapse a union that is accumulating one tuple per *length* into a single /// variable-length tuple /// diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index c22c16cc51..11ae25e4e2 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1167,18 +1167,22 @@ impl<'db> OverloadLiteral<'db> { } } - /// Typed internally-visible "raw" signature for this function. - /// That is, the return types of async functions are not wrapped in `CoroutineType[...]`. - /// The `return_callable_typevar_scope` controls whether type variables that only appear in a - /// return-position `Callable` stay bound to the function or move to the returned callable. + /// the [raw signature](Self::raw_signature) of this function short of the return type it + /// recovers: every parameter with the type it is written with or takes from elsewhere, and the + /// return type only when one is written (or an untyped property's initialiser declares it) /// - /// ## Warning - /// - /// This uses the semantic index to find the definition of the function. This means that if the - /// calling query is not in the same file as this function is defined in, then this will create - /// a cross-module dependency directly on the full AST which will lead to cache - /// over-invalidation. - pub(super) fn raw_signature( + /// a parameter's type never depends on what the body returns, so whatever reads a parameter + /// of this function from inside its body — the parameter's own definition, the default it + /// inherits, a type variable or `ParamSpec` of its signature that the body names — reads it + /// from here rather than from the raw signature. through the raw signature, inferring the body + /// would depend on the return type recovered from that same body, and every such body would be + /// a cycle through itself + #[salsa::tracked( + returns(ref), + cycle_initial=|_, _, _, _| Signature::bottom(), + heap_size=ruff_memory_usage::heap_size, + )] + pub(super) fn parameter_signature( self, db: &'db dyn Db, return_callable_typevar_scope: ReturnCallableTypeVarScope, @@ -1413,7 +1417,10 @@ impl<'db> OverloadLiteral<'db> { if !overload_list.is_empty() { let overload_sigs: Vec> = overload_list .iter() - .map(|ol| ol.raw_signature(db, ReturnCallableTypeVarScope::Public)) + .map(|ol| { + ol.parameter_signature(db, ReturnCallableTypeVarScope::Public) + .clone() + }) .collect(); raw_signature.inherit_unannotated_from_overloads(db, env, &overload_sigs, false); } @@ -1472,6 +1479,33 @@ impl<'db> OverloadLiteral<'db> { ); } + raw_signature + } + + /// typed internally-visible "raw" signature for this function. + /// that is, the return types of async functions are not wrapped in `CoroutineType[...]`. + /// the `return_callable_typevar_scope` controls whether type variables that only appear in a + /// return-position `Callable` stay bound to the function or move to the returned callable + /// + /// this is [`Self::parameter_signature`] with the return type that is not written filled in, + /// which can mean inferring the function's body + /// + /// ## Warning + /// + /// this uses the semantic index to find the definition of the function. this means that if the + /// calling query is not in the same file as this function is defined in, then this will create + /// a cross-module dependency directly on the full AST which will lead to cache + /// over-invalidation + pub(super) fn raw_signature( + self, + db: &'db dyn Db, + return_callable_typevar_scope: ReturnCallableTypeVarScope, + ) -> Signature<'db> { + let env = &ProgramEnvironment::from_scope(self.body_scope(db)); + let mut raw_signature = self + .parameter_signature(db, return_callable_typevar_scope) + .clone(); + // basedpython: a return type nobody wrote is drawn from the first source that supplies // one; see `return_type_source`. the body is the last of them, and recovering from it // takes the union of what it returns, and `None` when it can also fall off the end @@ -3234,6 +3268,18 @@ impl<'db> FunctionType<'db> { .last_definition_raw_signature(db, return_callable_typevar_scope) } + /// [`OverloadLiteral::parameter_signature`] of the last overload or implementation of this + /// function: its raw signature short of a return type recovered from its body + pub(super) fn last_definition_parameter_signature( + self, + db: &'db dyn Db, + return_callable_typevar_scope: ReturnCallableTypeVarScope, + ) -> &'db Signature<'db> { + self.literal(db) + .last_definition + .parameter_signature(db, return_callable_typevar_scope) + } + /// Return the kind for this function when it is converted into a [`CallableType`]. pub(crate) fn callable_type_kind(self, db: &'db dyn Db) -> CallableTypeKind { if self.is_classmethod(db) { diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index 11373e5a98..8046a858b1 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -257,7 +257,7 @@ fn find_typevar_binding<'db>( infer_definition_types(db, definition).function_type(definition) { let signature = function_ty - .last_definition_raw_signature(db, ReturnCallableTypeVarScope::Lexical); + .last_definition_parameter_signature(db, ReturnCallableTypeVarScope::Lexical); if let Some(bound) = signature.paramspec_component_binding(db, typevar) && bound.binding_context(db).definition() != Some(definition) && is_visible_across_class_boundary(db, bound, crossed_class_scope) @@ -515,11 +515,14 @@ impl<'db> GenericContext<'db> { let definition = index.expect_single_definition(class); original_class_type(db, definition)?.generic_context(db) } + // a function's type variables are the ones its parameters and written return type + // bind, so they are read without the return type recovered from its body: a body + // naming one of them would otherwise depend on what that same body returns NodeWithScopeKind::Function(function) => { let definition = index.expect_single_definition(function); infer_definition_types(db, definition) .function_type(definition)? - .last_definition_signature(db) + .last_definition_parameter_signature(db, ReturnCallableTypeVarScope::Public) .generic_context } NodeWithScopeKind::TypeAlias(type_alias) => { @@ -546,7 +549,7 @@ impl<'db> GenericContext<'db> { let definition = index.expect_single_definition(function); infer_definition_types(db, definition) .function_type(definition)? - .last_definition_raw_signature(db, ReturnCallableTypeVarScope::Lexical) + .last_definition_parameter_signature(db, ReturnCallableTypeVarScope::Lexical) .generic_context } else { Self::of_node(db, node, index) diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index 28345b2f85..b32f003ba8 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -2207,9 +2207,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } /// basedpython: for an unannotated parameter, look up the type that - /// [`OverloadLiteral::raw_signature`] filled in for it — from the sibling overloads when this - /// is an overload implementation, or (under `sound-types`) from the overridden base method. - /// matching is by name + /// [`OverloadLiteral::parameter_signature`] filled in for it — from the sibling overloads when + /// this is an overload implementation, or (under `sound-types`) from the overridden base + /// method. matching is by name fn inherited_parameter_type(&self, parameter: &ast::Parameter) -> Option> { let ty = self.signature_parameter_type(parameter)?; // an anonymous hole is not inherited from anywhere; it is offered separately, after @@ -2232,7 +2232,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let db = self.db(); let enclosing = nearest_enclosing_function(db, self.index, self.scope())?; enclosing - .last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public) + .last_definition_parameter_signature(db, ReturnCallableTypeVarScope::Public) .parameters() .iter() .find(|candidate| candidate.name() == Some(¶meter.name.id))? @@ -2247,8 +2247,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { fn signature_parameter_type(&self, parameter: &ast::Parameter) -> Option> { let db = self.db(); let enclosing = nearest_enclosing_function(db, self.index, self.scope())?; + // not the raw signature: that one's return type can be read off this very body let signature = - enclosing.last_definition_raw_signature(db, ReturnCallableTypeVarScope::Public); + enclosing.last_definition_parameter_signature(db, ReturnCallableTypeVarScope::Public); let matched = signature .parameters() .iter() diff --git a/crates/ty_python_semantic/src/types/infer/tests.rs b/crates/ty_python_semantic/src/types/infer/tests.rs index 94cc1150a0..cad3b83b75 100644 --- a/crates/ty_python_semantic/src/types/infer/tests.rs +++ b/crates/ty_python_semantic/src/types/infer/tests.rs @@ -6,6 +6,8 @@ use crate::db::tests::{TestDb, TestDbBuilder, setup_db}; use crate::lint::{LintSource, RuleSelection}; use crate::place::symbol; use crate::place::{ConsideredDefinitions, Place, PlaceAndQualifiers}; +use crate::types::function::OverloadLiteral; +use crate::types::inferred_signature::inferred_return_type; use crate::types::{KnownClass, KnownInstanceType, check_types}; use ruff_db::diagnostic::{Diagnostic, DiagnosticId, Severity}; use ruff_db::files::{File, system_path_to_file}; @@ -1391,6 +1393,76 @@ fn function_inference_regions_are_disjoint() -> anyhow::Result<()> { Ok(()) } +/// basedpython: what a function's body reads of its own signature — an unannotated parameter, a +/// type variable or `ParamSpec` the signature binds — it reads without the return type recovered +/// from that same body, so none of it is a cycle through that return type +/// +/// each is inferred on its own: inferring the whole body would also infer `y`'s annotation, which +/// is in the module's scope, and checking the module's `def f` asks for `f`'s return type +#[test] +fn a_body_reads_its_own_signature_without_the_return_type_it_recovers() -> anyhow::Result<()> { + fn function(db: &TestDb, file: File) -> Option> { + let definition = first_public_binding(db, file, "f"); + infer_definition_types(db, definition) + .function_type(definition) + .map(|function| function.literal(db).last_definition) + } + + /// the definition of `name` in `f`'s body + fn body_definition<'db>( + db: &'db TestDb, + function: OverloadLiteral<'db>, + name: &str, + ) -> Option> { + let scope = function.body_scope(db); + let symbol = place_table(db, scope).symbol_id(name)?; + let use_def = use_def_map(db, scope); + use_def + .end_of_scope_symbol_declarations(symbol) + .find_map(|declaration| declaration.declaration.definition()) + .or_else(|| { + use_def + .end_of_scope_symbol_bindings(symbol) + .find_map(|binding| binding.binding.definition()) + }) + } + + let mut db = setup_db(); + db.write_dedented( + "/src/main.py", + r#" + from typing import Callable, ParamSpec, TypeVar + + T = TypeVar("T") + P = ParamSpec("P") + + def f(x, y: T, *args: P.args, **kwargs: P.kwargs): + w: T + forwarded: Callable[P, None] + return x + "#, + )?; + let file = system_path_to_file(&db, "/src/main.py")?; + + db.clear_salsa_events(); + let Some(literal) = function(&db, file) else { + anyhow::bail!("expected a function"); + }; + for name in ["x", "w", "forwarded"] { + let Some(definition) = body_definition(&db, literal, name) else { + anyhow::bail!("expected a definition of `{name}`"); + }; + infer_definition_types(&db, definition); + } + let events = db.take_salsa_events(); + + let Some(literal) = function(&db, file) else { + anyhow::bail!("expected a function"); + }; + assert_function_query_was_not_run(&db, inferred_return_type, literal, &events); + Ok(()) +} + #[test] fn lazy_parameter_defaults() -> anyhow::Result<()> { let mut db = setup_db(); diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index 8761d36d82..1116fe6475 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -618,7 +618,7 @@ enum ReduceResult<'db> { /// If the value ​​is defined recursively, widening is performed from fewer literal elements, /// resulting in faster convergence of the fixed-point iteration. -pub(crate) const MAX_RECURSIVE_UNION_LITERALS: usize = 5; +const MAX_RECURSIVE_UNION_LITERALS: usize = 5; /// If the value ​​is defined non-recursively, the fixed-point iteration will converge in one go, /// so in principle we can have as many literal elements as we want. /// We set a large limit for union and enum literals. From fdbbc6700e0f3d092144522b15393e61f55789d1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:01:13 +1000 Subject: [PATCH 101/110] leave an import of a declaration out of its usage count, as one that names the declaration without using it, while textDocument/references and by/references go on listing it, by/references as an import; a write to it through its module or an instance, from another method or through global stays counted --- crates/ty_ide/src/goto_implementation.rs | 4 +- crates/ty_ide/src/usage_counts.rs | 75 +++++++++++++--- crates/ty_ide/src/usage_kinds.rs | 87 ++++++++++++------- .../src/types/ide_support.rs | 6 +- .../api/requests/document_usage_counts.rs | 16 ++-- crates/ty_server/tests/e2e/usage_counts.rs | 62 ++++++++----- docs/basedpython/features/editor.md | 20 +++-- 7 files changed, 187 insertions(+), 83 deletions(-) diff --git a/crates/ty_ide/src/goto_implementation.rs b/crates/ty_ide/src/goto_implementation.rs index e78857af84..a94d143f9a 100644 --- a/crates/ty_ide/src/goto_implementation.rs +++ b/crates/ty_ide/src/goto_implementation.rs @@ -384,8 +384,8 @@ mod tests { "); } - /// An override with a decorator is its own `def`, not the function the decorator returns, - /// which is defined somewhere else entirely. + /// an override with a decorator is its own `def`, not the function the decorator returns, + /// which is defined somewhere else entirely #[test] fn implementation_decorated_override_is_its_def() { let test = cursor_test( diff --git a/crates/ty_ide/src/usage_counts.rs b/crates/ty_ide/src/usage_counts.rs index 45c0064809..d90f72a4d9 100644 --- a/crates/ty_ide/src/usage_counts.rs +++ b/crates/ty_ide/src/usage_counts.rs @@ -4,9 +4,12 @@ //! //! a count is what asking of the declaration's name answers, found by the same //! code: [`crate::find_references()`] without the declaration for the usages, -//! and [`crate::goto_implementation()`] less the declaration itself for the -//! subclasses and overrides. so the count over a declaration is the number of -//! places its click goes to +//! less the imports of it, and [`crate::goto_implementation()`] less the +//! declaration itself for the subclasses and overrides. an import names the +//! declaration without using it: *find usages* still lists it, in the `import` +//! group [`crate::usage_kinds()`] puts it in, but it is not counted. so the +//! count over a declaration is the number of places its click goes to that are +//! not imports //! //! each of those is a search of every file of the project, which asked of each //! declaration in turn reads every file once per declaration. the counts are @@ -41,6 +44,7 @@ use crate::declaration::ClassName; use crate::find_references::find_references_to_each; use crate::goto_implementation::implementations_at_each; use crate::symbols::symbols_for_file; +use crate::usage_kinds::is_import; use crate::{Db, NavigationTarget, SymbolKind}; /// which counts to work out: each is a search of the workspace @@ -72,8 +76,8 @@ pub struct DeclarationUsages { pub full_range: TextRange, /// its name where it is declared pub name_range: TextRange, - /// how many places use it, the declaration itself not among them. `None` - /// when not asked for, and for a dunder method + /// how many places use it, neither the declaration itself nor an import of + /// it among them. `None` when not asked for, and for a dunder method pub references: Option, /// how many classes subclass a class, directly or not, and how many members /// of those override a method. `None` when not asked for, and for a function @@ -120,8 +124,12 @@ pub fn document_usage_counts( .unzip(); let found = find_references_to_each(db, file, &offsets, false); for (index, references) in counted.into_iter().zip(found) { - declarations[index].references = - Some(references.map_or(0, |references| references.len())); + declarations[index].references = Some(references.map_or(0, |references| { + references + .iter() + .filter(|reference| !is_import(db, file, reference)) + .count() + })); } } if kinds.implementations { @@ -386,14 +394,61 @@ print(Shape().area(), Square().area()) ) .build(); assert_snapshot!(test.usage_counts(), @" - class Shape 2 refs=3 impls=1 + class Shape 2 refs=2 impls=1 method area 3 refs=1 impls=1 function unused 6 refs=0 "); } - /// A decorated method is counted by its own `def`: what the decorator hands back is defined - /// elsewhere, and is neither an override of the method nor overridden by it. + /// an import names a declaration without using it, and is not counted, though *find usages* + /// lists it. what writes the name outside the scope that declares it does use it, and is + /// counted: an assignment through the module or through an instance, one from another method, + /// and a function's `global` statement and its assignment + #[test] + fn an_import_is_not_counted_and_a_write_from_elsewhere_is() { + let test = CursorTest::builder() + .source( + "lib.py", + " +class Shape: + def area(self) -> float: + return 0.0 + + def freeze(self) -> None: + self.area = lambda: 1.0 + +def make() -> Shape: + return Shape() + +def reset() -> None: + global make + make = lambda: Shape() +", + ) + .source( + "main.py", + " +import lib +from lib import Shape, make +from lib import make as build + +lib.make = build +shape = Shape() +shape.area = lambda: 2.0 +", + ) + .build(); + assert_snapshot!(test.usage_counts(), @" + class Shape 2 refs=4 impls=0 + method area 3 refs=2 impls=0 + method freeze 6 refs=0 impls=0 + function make 9 refs=3 + function reset 12 refs=0 + "); + } + + /// a decorated method is counted by its own `def`: what the decorator hands back is defined + /// elsewhere, and is neither an override of the method nor overridden by it #[test] fn a_decorated_method_is_counted_as_its_def() { let test = cursor_test( diff --git a/crates/ty_ide/src/usage_kinds.rs b/crates/ty_ide/src/usage_kinds.rs index cb6ee13819..fcc50cc492 100644 --- a/crates/ty_ide/src/usage_kinds.rs +++ b/crates/ty_ide/src/usage_kinds.rs @@ -1,5 +1,5 @@ -//! What each reference a search finds does with the name: what an editor's *find usages* -//! groups them by. +//! what each reference a search finds does with the name: what an editor's *find usages* +//! groups them by //! //! ```py //! from shapes.base import Shape # import @@ -14,10 +14,10 @@ //! print(Shape) # read //! ``` //! -//! These are the groups PyCharm's Python sorts usages into, in the order it tries them: a +//! these are the groups PyCharm's Python sorts usages into, in the order it tries them: a //! reference inside an import statement is an import whatever else it is, then the class list of -//! an `isinstance` call, then a type expression, then a class's bases. What is none of those is a -//! read or a write of the name, or the declaration itself. +//! an `isinstance` call, then a type expression, then a class's bases. what is none of those is a +//! read or a write of the name, or the declaration itself use ruff_db::parsed::parsed_module; use ruff_python_ast::find_node::covering_node; @@ -32,30 +32,30 @@ use ty_python_semantic::types::ide_support::{ use crate::django_template::is_django_template_path; use crate::{Db, ReferenceKind, ReferenceTarget}; -/// What a reference does with the name it refers to. +/// what a reference does with the name it refers to #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum UsageKind { - /// The name where it is declared: a `def`'s or a `class`'s name, or a first assignment. + /// the name where it is declared: a `def`'s or a `class`'s name, or a first assignment Declaration, - /// In an `import` or a `from ... import` statement. + /// in an `import` or a `from ... import` statement Import, - /// Among the classes an `isinstance` call tests against: its second argument, or an element - /// of a tuple there. + /// among the classes an `isinstance` call tests against: its second argument, or an element + /// of a tuple there Isinstance, - /// In a type expression: an annotation, a return or `raises` annotation, a type alias's + /// in a type expression: an annotation, a return or `raises` annotation, a type alias's /// value, a type parameter's bound or default, or an argument a call takes as a type, such - /// as `cast`'s first. + /// as `cast`'s first TypeHint, - /// In the parentheses of a class's header, among its bases or its keywords. + /// in the parentheses of a class's header, among its bases or its keywords Superclass, - /// Assigning, deleting or otherwise binding the name. + /// assigning, deleting or otherwise binding the name Write, - /// Reading the name's value, which calling it is. + /// reading the name's value, which calling it is Read, } -/// What each of `targets` does with the name it refers to, in the same order. `program` is the -/// program the search that found them ran in. +/// what each of `targets` does with the name it refers to, in the same order. `program` is the +/// program the search that found them ran in pub fn usage_kinds( db: &dyn Db, program: ProgramFile<'_>, @@ -68,14 +68,20 @@ pub fn usage_kinds( .collect() } +/// whether `target`, found by a search that ran in `program`, is in an `import` or a +/// `from ... import` statement: what [`usage_kinds`] calls a [`UsageKind::Import`], and what a +/// usage count leaves out, since an import names a declaration without using it +pub(crate) fn is_import(db: &dyn Db, program: ProgramFile<'_>, target: &ReferenceTarget) -> bool { + if is_template(db, target) { + return false; + } + let file = ProgramFile::new(db, target.file(), program.program(db)); + let module = parsed_module(db, file.python_file(db)).load(db); + in_import(covering_node(module.syntax().into(), target.range()).ancestors()) +} + fn usage_kind(db: &dyn Db, file: ProgramFile<'_>, target: &ReferenceTarget) -> UsageKind { - // a django template's names are only read, and its text is no python to classify - let template = target - .file() - .path(db) - .as_system_path() - .is_some_and(is_django_template_path); - (!template) + (!is_template(db, target)) .then(|| syntactic_kind(db, file, target.range())) .flatten() .unwrap_or(match target.kind() { @@ -85,19 +91,34 @@ fn usage_kind(db: &dyn Db, file: ProgramFile<'_>, target: &ReferenceTarget) -> U }) } -/// The group the syntax around `range` puts a reference in, `None` when it is only a read or a -/// write. -fn syntactic_kind(db: &dyn Db, file: ProgramFile<'_>, range: TextRange) -> Option { - let module = parsed_module(db, file.python_file(db)).load(db); - let covering = covering_node(module.syntax().into(), range); - let ancestors: Vec> = covering.ancestors().collect(); +/// whether `target` is in a django template, whose names are only read, and whose text is no +/// python to classify +fn is_template(db: &dyn Db, target: &ReferenceTarget) -> bool { + target + .file() + .path(db) + .as_system_path() + .is_some_and(is_django_template_path) +} - if ancestors.iter().any(|node| { +/// whether one of a reference's `ancestors` is an `import` or a `from ... import` statement +fn in_import<'a>(mut ancestors: impl Iterator>) -> bool { + ancestors.any(|node| { matches!( node, AnyNodeRef::StmtImport(_) | AnyNodeRef::StmtImportFrom(_) ) - }) { + }) +} + +/// the group the syntax around `range` puts a reference in, `None` when it is only a read or a +/// write +fn syntactic_kind(db: &dyn Db, file: ProgramFile<'_>, range: TextRange) -> Option { + let module = parsed_module(db, file.python_file(db)).load(db); + let covering = covering_node(module.syntax().into(), range); + let ancestors: Vec> = covering.ancestors().collect(); + + if in_import(ancestors.iter().copied()) { return Some(UsageKind::Import); } @@ -149,7 +170,7 @@ fn syntactic_kind(db: &dyn Db, file: ProgramFile<'_>, range: TextRange) -> Optio kind } -/// The group a reference at `range`, inside one of `call`'s arguments or its callee, is in. +/// the group a reference at `range`, inside one of `call`'s arguments or its callee, is in fn call_argument_kind( model: &SemanticModel<'_>, call: &ast::ExprCall, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 56bce18924..7cdee98aa6 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -1166,7 +1166,7 @@ fn member_implementation_definition<'db>( // implementation. If inference cannot produce a function literal, keep the original `def` as a // conservative fallback. // - // This is the function the `def` defines, before any decorator. The name's binding type is + // this is the function the `def` defines, before any decorator. the name's binding type is // what the decorators return, which for // // ```py @@ -1179,7 +1179,7 @@ fn member_implementation_definition<'db>( // def area(self) -> float: ... // ``` // - // is `wrapper`, a function defined somewhere else entirely. + // is `wrapper`, a function defined somewhere else entirely let Some(function) = infer_definition_types(db, definition).function_type(definition) else { return Some(ResolvedDefinition::Definition(definition)); }; @@ -1886,7 +1886,7 @@ pub fn unpacking_target_type<'db>( }) } -/// Whether `call` calls python's `isinstance`, by whatever name it is reached. +/// whether `call` calls python's `isinstance`, by whatever name it is reached pub fn calls_isinstance(model: &SemanticModel<'_>, call: &ast::ExprCall) -> bool { matches!( call.func.inferred_type(model), diff --git a/crates/ty_server/src/server/api/requests/document_usage_counts.rs b/crates/ty_server/src/server/api/requests/document_usage_counts.rs index 4174c0e9a3..7e60896d8d 100644 --- a/crates/ty_server/src/server/api/requests/document_usage_counts.rs +++ b/crates/ty_server/src/server/api/requests/document_usage_counts.rs @@ -3,10 +3,12 @@ //! //! the counts an editor draws above each declaration, "3 usages" or "2 //! overrides", for the whole document at once. each count is what -//! `textDocument/references` (without the declaration) or -//! `textDocument/implementation` (less the declaration itself) answers at the -//! declaration's name, from the same code, so a client goes to as many places as -//! the count says by sending that request when one is clicked. asking either of +//! `by/references` (without the declaration, less the references of kind +//! `import`) or `textDocument/implementation` (less the declaration itself) +//! answers at the declaration's name, from the same code, so a client goes to as +//! many places as the count says, and to the imports besides, by sending that +//! request when one is clicked. an import names a declaration without using it, +//! so it is listed but not counted. asking either of //! those of each declaration instead would be two requests per declaration on //! every pass over the document //! @@ -88,9 +90,9 @@ pub(crate) struct DeclarationUsages { /// its name, where `textDocument/references` and `textDocument/implementation` /// answer for it selection_range: Range, - /// how many places use it, as `textDocument/references` answers without the - /// declaration. there when asked for, except on a dunder method, which the - /// language calls rather than its name + /// how many places use it, as `by/references` answers without the declaration, + /// less its imports. there when asked for, except on a dunder method, which + /// the language calls rather than its name #[serde(skip_serializing_if = "Option::is_none")] references: Option, /// how many classes subclass a class, directly or not, and how many of their diff --git a/crates/ty_server/tests/e2e/usage_counts.rs b/crates/ty_server/tests/e2e/usage_counts.rs index 374e672717..134f82287d 100644 --- a/crates/ty_server/tests/e2e/usage_counts.rs +++ b/crates/ty_server/tests/e2e/usage_counts.rs @@ -2,9 +2,9 @@ //! //! what counts as a usage or an override is `ty_ide`'s and its unit tests cover it. what these //! cover is the contract a client reads: each declaration with its ranges, only the counts asked -//! for, and each count the number of places `textDocument/references` or -//! `textDocument/implementation` goes to from the declaration's name, which is what a client asks -//! when the count is clicked +//! for, and each count the number of places `by/references` finds from the declaration's name +//! less the imports of it, or the number `textDocument/implementation` goes to, which is what a +//! client asks when the count is clicked use std::time::Duration; @@ -12,9 +12,8 @@ use anyhow::Result; use lsp_server::ErrorCode; use lsp_types::{ Definition, ImplementationParams, ImplementationRequest, ImplementationResponse, - LspRequestMethod, MessageDirection, PartialResultParams, Position, ReferenceContext, - ReferenceParams, ReferencesRequest, Request, TextDocumentIdentifier, - TextDocumentPositionParams, WorkDoneProgressParams, + LspRequestMethod, MessageDirection, PartialResultParams, Position, Request, + TextDocumentIdentifier, TextDocumentPositionParams, WorkDoneProgressParams, }; use ruff_db::system::SystemPath; use ty_server::ClientOptions; @@ -31,6 +30,16 @@ impl Request for DocumentUsageCounts { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } +/// `by/references`, in json throughout +enum ClassifiedReferences {} + +impl Request for ClassifiedReferences { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/references"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + const SHAPES: &str = "\ from abc import ABC, abstractmethod @@ -168,10 +177,10 @@ fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result assert_eq!( counts, [ - ("Shape".to_string(), Some(3), Some(1), Some(false)), + ("Shape".to_string(), Some(2), Some(1), Some(false)), ("area".to_string(), Some(1), Some(1), Some(true)), ("describe".to_string(), Some(1), Some(0), Some(false)), - ("total".to_string(), Some(2), None, Some(false)), + ("total".to_string(), Some(1), None, Some(false)), ] ); @@ -181,18 +190,31 @@ fn a_count_is_the_number_of_places_the_request_it_stands_for_goes_to() -> Result position: position(declaration), }; let references = server - .send_request_await::(ReferenceParams { - text_document_position_params: at.clone(), - work_done_progress_params: WorkDoneProgressParams::default(), - partial_result_params: PartialResultParams::default(), - context: ReferenceContext { - include_declaration: false, - }, - }) - .unwrap_or_default(); + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": at.position, + "context": { "includeDeclaration": false }, + })) + .expect("references of a declaration"); + let (imports, uses): (Vec<_>, Vec<_>) = references + .as_array() + .expect("a list of references") + .iter() + .partition(|reference| reference["kind"] == "import"); + // an import names the declaration without using it: find usages lists it, the count + // leaves it out assert_eq!( declaration["references"].as_u64(), - Some(references.len() as u64), + Some(uses.len() as u64), + "{declaration:#}" + ); + // `main.by` imports `Shape` and `total` + assert_eq!( + imports.len(), + usize::from(matches!( + declaration["name"].as_str(), + Some("Shape" | "total") + )), "{declaration:#}" ); @@ -348,8 +370,8 @@ fn a_count_the_client_cancels_while_it_runs_is_answered_once_and_the_next_is_ans .expect("an answer for a document"); let shape = &declarations.as_array().expect("a list of declarations")[0]; assert_eq!(shape["name"], "Shape"); - // `total` names it, and `main.by` and every module import it and subclass it - assert_eq!(shape["references"].as_u64(), Some(2 * MODULES as u64 + 3)); + // `total` names it, and `main.by` and every module subclass it (an import is not a usage) + assert_eq!(shape["references"].as_u64(), Some(MODULES as u64 + 2)); assert_eq!(shape["implementations"].as_u64(), Some(MODULES as u64 + 1)); // and the cancelled count does not bring a second answer diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 913764829b..dfc6e35075 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -1303,9 +1303,11 @@ class in `cast`'s first argument in a base list is a `typeHint`. the answer is without `includeDeclaration`, both leave out only where the name is declared, in the scope that declares it: its `def` or `class`, an annotation of it, or an assignment no other assignment of it comes before. what binds it anywhere else -uses it and stays, as PyCharm's Python counts it: an import of it, an assignment -to it through its module or through an instance, from another method than the -one that declares the attribute, or from a function naming it `global` +stays, as PyCharm's Python lists it: an import of it, an assignment to it +through its module or through an instance, from another method than the one +that declares the attribute, or from a function naming it `global`. an import +names the name without using it, so the [usage counts](#usage-counts) leave it +out; the writes are uses, and are counted ## type info @@ -1443,13 +1445,15 @@ each declaration comes with its `name`, its `kind` (`class`, `function` or `method`), its whole `range` from its first decorator, its `selectionRange` on its name, and the counts named in the request's `counts`: -| count | what it is | -| ----------------- | ------------------------------------------------------------------------------------------- | -| `references` | the places `textDocument/references` finds from the name, the declaration not among them | -| `implementations` | of a class or a method, what `textDocument/implementation` finds from the name, less itself | +| count | what it is | +| ----------------- | ----------------------------------------------------------------------------------------------------------- | +| `references` | the places `by/references` finds from the name, the declaration not among them, less those of kind `import` | +| `implementations` | of a class or a method, what `textDocument/implementation` finds from the name, less itself | so the count is the number of places the click on it goes to, when a client -sends that request from the name. every declaration has `abstract`, `true` for a +sends that request from the name, but for the imports: an import names the +declaration without using it, so *find usages* lists it and the count leaves it +out. every declaration has `abstract`, `true` for a method with implementations that is an `@abstractmethod` or a protocol member with no body, so that what overrides it implements it, and `false` otherwise. a count not named is not worked out From 2b519efa8a7b3fee7bcd50e15634a4a793272da3 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:05:07 +1000 Subject: [PATCH 102/110] update crates/basedpython's own lock file for the dependencies by_stage, ty and ty_ide gained and the one ty_ide dropped on this branch, so a build of that crate with --locked accepts it --- crates/basedpython/Cargo.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index b658bfdafd..d50577244d 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -317,6 +317,9 @@ dependencies = [ "dunce", "rayon", "ruff_db", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2", @@ -3677,9 +3680,12 @@ dependencies = [ "ruff_command_line", "ruff_db", "ruff_diagnostics", + "ruff_notebook", "ruff_python_ast", "ruff_python_parser", + "ruff_python_trivia", "ruff_ranged_value", + "ruff_source_file", "ruff_text_size", "salsa", "serde", @@ -3726,6 +3732,7 @@ dependencies = [ "ruff_db", "ruff_diagnostics", "ruff_index", + "ruff_linter", "ruff_memory_usage", "ruff_python_ast", "ruff_python_codegen", @@ -3735,6 +3742,7 @@ dependencies = [ "ruff_python_trivia", "ruff_source_file", "ruff_text_size", + "ruff_workspace", "rustc-hash", "salsa", "smallvec", From d4d28723acddc928b7e302638b126915f81e6bc1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:20:59 +1000 Subject: [PATCH 103/110] write an unannotated parameter's inferred type parameter by the parameter's name in an inferred return type hint, def twice(value) -> list[value] rather than list[value@twice], and offer such a return type as no annotation, since only a parameter written with some names its hole --- crates/ty_ide/src/inlay_hints.rs | 78 ++++++++++++++++--- .../src/types/ide_support.rs | 45 +++++++++-- 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 0146ae3a04..b28c22121d 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -28,11 +28,12 @@ use ty_python_semantic::types::ide_support::{ BaseTypeArgument, InferredInvalidations, InferredStateReads, InlayHintCallArgumentDetails, StateRead, WriteSite, hintable_parameter_type, implicit_enum_member_value, implicit_type_name_module, inferred_base_type_arguments, inferred_derived_dependencies, - inferred_invalidations, inferred_override, inferred_parameter_bound, inferred_property_type, - inferred_raises, inferred_return_type, inferred_state_reads, inferred_type_param_variance, - inherited_parameter_annotation, inherited_parameter_default, inlay_hint_call_argument_details, - is_composable_function, is_reveal_type_function, is_union_special_form, numeric_promotion, - parameter_stability, trailing_lambda_implicit_parameters, type_parameter_names, + inferred_invalidations, inferred_override, inferred_parameter_bound, inferred_parameter_holes, + inferred_property_type, inferred_raises, inferred_return_type, inferred_state_reads, + inferred_type_param_variance, inherited_parameter_annotation, inherited_parameter_default, + inlay_hint_call_argument_details, is_composable_function, is_inferred_parameter_hole, + is_reveal_type_function, is_union_special_form, numeric_promotion, parameter_stability, + trailing_lambda_implicit_parameters, type_parameter_names, }; use ty_python_semantic::types::{ BoundTypeVarInstance, DisplaySettings, Type, TypeDetail, TypeDisplayDetails, @@ -1417,6 +1418,11 @@ fn spell_type<'db>( if ty.is_unknown() { editable = false; } + // basedpython: an unannotated parameter's inferred type parameter is written by + // the parameter's name, which only names it where the parameter writes `some` + if is_inferred_parameter_hole(db, *ty) { + editable = false; + } let start = target.start().to_usize(); let end = target.end().to_usize(); // If we skipped over some bytes, push them with no target @@ -2463,7 +2469,8 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { file: self.model.program_file(), importer: &self.importer, }; - let type_params_in_scope = self.model.type_params_in_signature(function); + let mut type_params_in_scope = self.model.type_params_in_signature(function); + type_params_in_scope.extend(inferred_parameter_holes(&self.model, function)); self.push(InlayHint::inferred_return( context, function, @@ -10741,9 +10748,10 @@ Source with applied edits: "); } + /// basedpython: an unannotated parameter's inferred type parameter is written by the + /// parameter's name, and a return type mentioning one is offered as no annotation #[test] - fn inferred_unknown_type() { - // We do not suggest edits/imports for ty-internal types (Unknown), even if they are nested. + fn inferred_return_names_an_inferred_parameter_by_its_parameter() { let mut test = return_type_hint_test( r#" def twice(value): @@ -10752,7 +10760,7 @@ Source with applied edits: ); assert_snapshot!(test.inlay_hints(), @" - def twice(value)[ -> list[value@twice]]: + def twice(value)[ -> list[value]]: return [value, value] --------------------------------------------- @@ -10764,8 +10772,58 @@ Source with applied edits: info: Source --> main2.py:LL:22 | - LL | def twice(value)[ -> list[value@twice]]: + LL | def twice(value)[ -> list[value]]: | ^^^^ + + info[inlay-hint-location]: Inlay Hint Target + --> main.py:2:11 + | + 2 | def twice(value): + | ^^^^^ + info: Source + --> main2.py:2:27 + | + 2 | def twice(value)[ -> list[value]]: + | ^^^^^ + "); + } + + #[test] + fn inferred_unknown_type() { + // we do not suggest edits/imports for ty-internal types (Unknown), even if they are nested + let mut test = return_type_hint_test( + r#" + def firsts(values: list): + return [values[0]] + "#, + ); + assert_snapshot!(test.inlay_hints(), @" + + def firsts(values: list)[ -> list[Unknown]]: + return [values[0]] + + --------------------------------------------- + info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | + LL | class list[in out Element](MutableSequence[Element]): + | ^^^^ + info: Source + --> main2.py:LL:30 + | + LL | def firsts(values: list)[ -> list[Unknown]]: + | ^^^^ + + info[inlay-hint-location]: Inlay Hint Target + --> stdlib/ty_extensions/_internal.pyi:LL:1 + | + LL | Unknown: _SpecialForm + | ^^^^^^^ + info: Source + --> main2.py:LL:35 + | + LL | def firsts(values: list)[ -> list[Unknown]]: + | ^^^^^^^ "); // We still suggest edits/imports for a user-defined class named Unknown. diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 7cdee98aa6..ea63cbb56d 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -4636,6 +4636,42 @@ pub fn inferred_parameter_bound<'db>( model: &SemanticModel<'db>, parameter: &ast::Parameter, ) -> Option> { + inferred_parameter_hole(model, parameter)? + .typevar(model.db()) + .upper_bound(model.db(), &model.program_environment()) + .filter(|bound| !bound.is_dynamic()) +} + +/// basedpython: the `some` holes an inferred signature opens for the unannotated parameters of +/// `function`, each with the name of its parameter, which is how a type mentioning one is +/// written: `def twice(value) -> list[value]` +pub fn inferred_parameter_holes<'db>( + model: &SemanticModel<'db>, + function: &ast::StmtFunctionDef, +) -> Vec<(BoundTypeVarInstance<'db>, Name)> { + function + .parameters + .iter() + .filter_map(|parameter| { + let parameter = parameter.as_parameter(); + let hole = inferred_parameter_hole(model, parameter)?; + Some((hole, parameter.name.id.clone())) + }) + .collect() +} + +/// basedpython: whether `ty` is a `some` hole an inferred signature opened, which no annotation +/// can name: the parameter it is named after writes no `some` +pub fn is_inferred_parameter_hole<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { + ty.is_inferred_parameter_hole(db) +} + +/// the `some` hole an inferred signature opens for the unannotated `parameter`, or `None` when +/// its type comes from anywhere else +fn inferred_parameter_hole<'db>( + model: &SemanticModel<'db>, + parameter: &ast::Parameter, +) -> Option> { let db = model.db(); let index = semantic_index(db, db.program_file(model.file())); let definition = index.try_definition(parameter)?; @@ -4647,12 +4683,9 @@ pub fn inferred_parameter_bound<'db>( else { return None; }; - if !Type::TypeVar(hole).is_inferred_parameter_hole(db) { - return None; - } - hole.typevar(db) - .upper_bound(db, &model.program_environment()) - .filter(|bound| !bound.is_dynamic()) + Type::TypeVar(hole) + .is_inferred_parameter_hole(db) + .then_some(hole) } /// basedpython: the default a parameter takes from the method its `def` overrides, written the From 49f9c7f9ef5a763893c668f7d6bcccacf8a35de0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:28:57 +1000 Subject: [PATCH 104/110] exit by run with 128 plus the signal's number when a signal ends the program, as by repl does --- crates/ty/src/by_commands.rs | 2 +- crates/ty/tests/by_e2e.rs | 25 +++++++++++++++++++++++++ docs/basedpython/cli-reference.md | 3 +++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index df234e30d5..0ebe71cbae 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -520,7 +520,7 @@ pub(crate) fn cmd_run( ), })?; - let mut code = status.code().unwrap_or(1); + let mut code = exit_code(status); if let (Some(report), Some((_, file))) = (coverage, &measured) { if file.is_file() { let report = cwd.join(report); diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 510327189b..2152c9097e 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1072,6 +1072,31 @@ fn run_checks_against_the_environment_a_launcher_runs_in() { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); } +/// a program a signal ends makes `by run` exit as a shell reports that, with 128 and the +/// signal's number +#[cfg(unix)] +#[test] +fn run_exits_with_the_signal_that_ended_the_program() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "import os\nimport signal\n\nos.kill(os.getpid(), signal.SIGKILL)\n", + ) + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + assert_eq!( + output.status.code(), + Some(128 + 9), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + /// the program runs out of a directory of transpiled copies that is deleted when /// the run ends, so every path python derives from a module's origin named a /// temporary file — leaving a tool started anywhere but the project root with diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 50c97f63f0..e73947e604 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -85,6 +85,9 @@ reports it and exits 1. a debugger that stops where such an exception leaves the outermost frame stops in the program by treating the runner's frames as it treats `runpy`'s under `python -m`: as the launcher's, not the program's +a signal that ends the program ends `by run` with 128 and the signal's number, +as a shell reports it + `--in-build` runs the program in the tree `by run` staged rather than in the directory it was typed in. the default is right for an application: a relative path on the command line, and anything the program reads or writes beside the From a1bf9771c3c93107f2cb7b7c463b15f5b1503e21 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:30:07 +1000 Subject: [PATCH 105/110] find by run --coverage's project directory without os.path.relpath, which raises across windows drives --- crates/ty/src/by_commands.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 0ebe71cbae..6ee02920cd 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -160,7 +160,7 @@ pub(crate) struct StagedRun { /// the staged tree, deleted when this is dropped pub(crate) tree: tempfile::TempDir, /// the project file behind each copy in the tree - pub(crate) carried: BTreeMap, + carried: BTreeMap, traceback_entries: Vec, /// the means to build the project's db again, unfrozen pub(crate) rebuilder: Rebuilder, @@ -537,7 +537,8 @@ pub(crate) fn cmd_run( // said already eprintln!( "note: no coverage report was written: the program ended without returning \ - to `by run`, as `os._exit` does" + to `by run`, as one that calls `os._exit`, replaces itself with `os.exec*` or \ + is killed by a signal does" ); } } @@ -3586,6 +3587,8 @@ def _start_coverage(report): """Measure the program with coverage.py, and write what ran to `report` once the program has ended: after its other threads and its own `atexit` handlers, which python runs before the ones registered ahead of them.""" + import pathlib + try: import coverage from coverage.exceptions import CoverageWarning @@ -3602,8 +3605,14 @@ def _start_coverage(report): # the project is. a program run in the tree (`--in-build`) is in the project's # directory that the tree's directory is a copy of here = os.getcwd() - within = os.path.relpath(os.path.realpath(here), _TREE) - project_dir = here if within.split(os.sep)[0] == os.pardir else os.path.join(ROOT, within) + try: + within = pathlib.PurePath(os.path.realpath(here)).relative_to(_TREE) + except ValueError: + # outside the tree, on another drive than it among them, which `os.path.relpath` + # raises for + project_dir = here + else: + project_dir = os.path.join(ROOT, within) os.chdir(project_dir) try: # no data file: what was measured is read back below, and a `.coverage` left From 328ed292222bdd00fc4fd27a4a5309b3da32cc15 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:30:18 +1000 Subject: [PATCH 106/110] install pinned pytest and coverage.py into ci's test interpreter, so the pytest and --coverage e2e tests run there instead of skipping --- .github/workflows/ci.yaml | 11 +++ AGENTS.md | 4 + crates/ty/tests/by_e2e.rs | 37 +++++--- pyproject.toml | 7 ++ uv.lock | 180 +++++++++++++++++++++++++++++++++++++- 5 files changed, 223 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08d97020d2..7ab68b7557 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -537,6 +537,17 @@ jobs: # interpreters as externally managed, though this one belongs to the job alone - name: "Install `typing_extensions` into the interpreter" run: uv pip install --python "$PYTHON" --break-system-packages "typing_extensions==4.16.0" + # the end-to-end tests of `by run --coverage` and of a pytest run need coverage.py and + # pytest, and skip on an interpreter without them, as a contributor's may be. this one is + # given both, as the `test-python` dependency group locks them, and says so: while + # `BY_TEST_PYTHON_PROVIDES` names a module, a test that cannot import it fails rather + # than skips + - name: "Install pytest and coverage.py into the interpreter" + run: | + uv export --only-group test-python --no-emit-project --output-file "$RUNNER_TEMP/test-python.txt" + uv pip install --python "$PYTHON" --break-system-packages --require-hashes \ + --requirements "$RUNNER_TEMP/test-python.txt" + echo "BY_TEST_PYTHON_PROVIDES=pytest coverage" >> "$GITHUB_ENV" # a separate invocation on purpose: the backend's differential tests fork processes # and compile C, and sharing a run with `by_transforms`' ~1800 other tests starves # them into failures that pass on their own diff --git a/AGENTS.md b/AGENTS.md index 886b07c2a5..5b9d15f208 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,10 @@ PYTHON=$(which python3.13) CARGO_PROFILE_DEV_OPT_LEVEL=1 CARGO_PROFILE_DEV_LTO=o CI pins it in a dedicated step, so it never sees this. +The end-to-end tests of a `pytest` or `--coverage` run skip when `$PYTHON` cannot import `pytest` or +`coverage`. Setting `BY_TEST_PYTHON_PROVIDES="pytest coverage"`, the modules the interpreter was +given, makes them fail instead when it cannot import one. CI sets it. + ### Fallback without nextest If `cargo nextest` is not available, use `cargo test` with the same environment variables. For the workspace, exclude `ty` and then run its other test targets explicitly. If new non-watcher test targets are added to `ty`, add them to the `ty` command below; `cargo metadata --no-deps --format-version 1` lists each crate's targets: diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 2152c9097e..91925850d5 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1244,15 +1244,33 @@ fn run_in_build_runs_in_the_tree_it_staged() { ); } +/// the modules, separated by whitespace, that whoever installed them says the interpreter +/// `$PYTHON` names can import. a test that needs one fails when it cannot import it, rather +/// than skipping as it does on an interpreter that was not given it +const BY_TEST_PYTHON_PROVIDES: &str = "BY_TEST_PYTHON_PROVIDES"; + +/// whether `python` can import `module`. an interpreter `BY_TEST_PYTHON_PROVIDES` says can +/// import it and cannot fails the test +fn imports(python: &str, module: &str) -> bool { + let importable = Command::new(python) + .args(["-c", &format!("import {module}")]) + .status() + .is_ok_and(|status| status.success()); + let provided = std::env::var(BY_TEST_PYTHON_PROVIDES) + .is_ok_and(|provided| provided.split_whitespace().any(|name| name == module)); + assert!( + importable || !provided, + "`{python}` cannot import {module}, which {BY_TEST_PYTHON_PROVIDES} says it can" + ); + importable +} + /// The interpreter the coverage tests run on, and whether it can import /// coverage.py: `$PYTHON` when it is set, as for every python-executing test, /// and `python3` otherwise. fn coverage_interpreter() -> (String, bool) { let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); - let importable = Command::new(&python) - .args(["-c", "import coverage"]) - .status() - .is_ok_and(|status| status.success()); + let importable = imports(&python, "coverage"); (python, importable) } @@ -1682,10 +1700,7 @@ fn run_coverage_measures_threads_and_exit_handlers() { )] fn run_coverage_maps_a_module_a_test_runner_compiled() { let (python, importable) = coverage_interpreter(); - let pytest = Command::new(&python) - .args(["-c", "import pytest"]) - .status() - .is_ok_and(|status| status.success()); + let pytest = imports(&python, "pytest"); if !importable || !pytest { eprintln!("skipping: {python} cannot import both coverage and pytest"); return; @@ -2652,11 +2667,7 @@ fn repl_interrupts_a_read_of_the_program_s_at_the_program_s_line() { )] fn pytest_project(dir: &Path) -> Option { let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); - let pytest = Command::new(&python) - .args(["-c", "import pytest"]) - .status() - .is_ok_and(|status| status.success()); - if !pytest { + if !imports(&python, "pytest") { eprintln!("skipping: {python} cannot import pytest"); return None; } diff --git a/pyproject.toml b/pyproject.toml index a4af6ca0eb..39cf4f498e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,12 @@ docs = [ ty-ecosystem = [ "ecosystem-analyzer", ] +# what the end-to-end tests of `by run --coverage` and of a pytest run import, which ci installs +# into each interpreter it tests +test-python = [ + "coverage", + "pytest", +] typeshed-docstrings = [ "docstring-adder", ] @@ -258,6 +264,7 @@ basedpython-docs = { requires-python = ">=3.12" } docs = { requires-python = ">=3.12" } release = { requires-python = ">=3.12" } ty-ecosystem = { requires-python = ">=3.13" } +test-python = { requires-python = ">=3.11" } typeshed-docstrings = { requires-python = ">=3.10" } typeshed-formatting = { requires-python = ">=3.10" } diff --git a/uv.lock b/uv.lock index 01a7226f83..433261bc62 100644 --- a/uv.lock +++ b/uv.lock @@ -4,7 +4,8 @@ requires-python = ">=3.7" resolution-markers = [ "python_full_version >= '3.13'", "python_full_version == '3.12.*'", - "python_full_version >= '3.10' and python_full_version < '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", "python_full_version >= '3.8' and python_full_version < '3.10'", "python_full_version < '3.8'", ] @@ -223,6 +224,10 @@ docs = [ release = [ { name = "rooster", marker = "python_full_version >= '3.12'" }, ] +test-python = [ + { name = "coverage", marker = "python_full_version >= '3.11'" }, + { name = "pytest", marker = "python_full_version >= '3.11'" }, +] ty-ecosystem = [ { name = "ecosystem-analyzer", marker = "python_full_version >= '3.13'" }, ] @@ -238,6 +243,7 @@ basedpython-docs = ">=3.12" dev = ">=3.12" docs = ">=3.12" release = ">=3.12" +test-python = ">=3.11" ty-ecosystem = ">=3.13" typeshed-docstrings = ">=3.10" typeshed-formatting = ">=3.10" @@ -268,6 +274,10 @@ docs = [ { name = "pyyaml", marker = "python_full_version >= '3.12'", specifier = ">=6.0.3" }, ] release = [{ name = "rooster", marker = "python_full_version >= '3.12'", specifier = "==0.1.1" }] +test-python = [ + { name = "coverage", marker = "python_full_version >= '3.11'" }, + { name = "pytest", marker = "python_full_version >= '3.11'" }, +] ty-ecosystem = [{ name = "ecosystem-analyzer", marker = "python_full_version >= '3.13'", git = "https://github.com/astral-sh/ecosystem-analyzer" }] typeshed-docstrings = [{ name = "docstring-adder", marker = "python_full_version >= '3.10'", git = "https://github.com/astral-sh/docstring-adder.git" }] typeshed-formatting = [{ name = "black", marker = "python_full_version >= '3.10'", specifier = ">=26.10.0" }] @@ -550,7 +560,8 @@ version = "8.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.12.*'", - "python_full_version >= '3.10' and python_full_version < '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", ] dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -581,6 +592,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/72/db17bf5f87568ab524413693385be2eeb03e652ab56c54ea05fa85515675/coverage-7.16.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:23219888477edd736b6fcaec1272d47d93b926e999641ffea7e53a1738e70b2b", size = 223670, upload-time = "2026-09-27T12:25:34.675Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e1/285727a8a74d48de256e8605413ee4cf82301228253ea58e5ab4270138b0/coverage-7.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:40c0f00899fe6181ae7f434ceb200e51f5ee4b8ed10e3b5f0b605f0cae15da87", size = 224191, upload-time = "2026-09-27T12:25:36.476Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/c737b73bac9bf5034f5ff45a4237b7a189d417a4e36751faf5d10c082cbe/coverage-7.16.2-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a4624f80732f6b427ac58f1f59c577a0994a12e8174b5af6a027b4b58795d4c3", size = 250921, upload-time = "2026-09-27T12:25:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/29/4e/e1d38d27817d91776ca543d01276a67eace3df675a673cac32b12c167d58/coverage-7.16.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:191803c4996b499fcd78c2ad5e5f767dcc53cb4dc6de6d6a741b443a1821ef02", size = 252751, upload-time = "2026-09-27T12:25:39.291Z" }, + { url = "https://files.pythonhosted.org/packages/0e/19/14a8e44cbb2ad36ae62aaa03c5eec4a06a7f7df220ea3776af26927acbe2/coverage-7.16.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fd670ac43b709c575aefc25bf52d8a598a3bc5017bddfd0a179152ab06a2deb", size = 254616, upload-time = "2026-09-27T12:25:40.506Z" }, + { url = "https://files.pythonhosted.org/packages/f9/72/f5bcad0d9a9b450080032344fbff7ec60c1b0e3d38019f7735dee2b69645/coverage-7.16.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:705e5af11d34647efdc170c7840b6857c81cf74be96419a553f237e68e62cb72", size = 256541, upload-time = "2026-09-27T12:25:41.785Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9a/8c735234e8abb52bf5d063f98c780fa942e77c4c0225f9fcb9b33effc346/coverage-7.16.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8afd9bf35cc6a1f22eb3634808fa8e0b91902459c5721ef2e4461dfe771d7f08", size = 251589, upload-time = "2026-09-27T12:25:43.269Z" }, + { url = "https://files.pythonhosted.org/packages/35/6a/1bf6d32e55642d6972aa842640e0a8850e612d19ac7d05e2c32fa59dfd34/coverage-7.16.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3f43bac1856ba269b905302778d4df433d6006489a192174ad77ac528e395032", size = 252660, upload-time = "2026-09-27T12:25:45.051Z" }, + { url = "https://files.pythonhosted.org/packages/53/5b/05b1c0d0e9495cb056155a16066259adc1e935e7411a82a62779bf729975/coverage-7.16.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f8475460aa33ee28ac896ab1156d0bb3b6c639f7f8383c2677d3359eb35f8205", size = 250660, upload-time = "2026-09-27T12:25:46.51Z" }, + { url = "https://files.pythonhosted.org/packages/05/33/5bc3db57fc9c56b4ef055725c38c34faa818a27d17d65e251371f7f0e3a0/coverage-7.16.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d6276d78f6fca7d0ac066d5da4165c5acd07829e8305c2cb900b738fb3a75a72", size = 254479, upload-time = "2026-09-27T12:25:47.819Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0f/4a5de66daef26eb919213f63b84f45d5065ab2df94cd767fc7c1174ac5e2/coverage-7.16.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:736fde09ea39646d11f8e3b76bd3425c075aa4dd45f24891970bb77c14ff20f5", size = 250940, upload-time = "2026-09-27T12:25:49.227Z" }, + { url = "https://files.pythonhosted.org/packages/57/d3/84cd6a11e707e422194739e9734d947746efa5e5b3358ff2d601f823ee45/coverage-7.16.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c85d54e7e8a2ca932fe8399301af9b8d5907ea2a455ffaff6e7d1208db83b943", size = 251543, upload-time = "2026-09-27T12:25:50.554Z" }, + { url = "https://files.pythonhosted.org/packages/ab/bc/3d84c2e2a95f38e346ce3730d9f0e53114f4e8aace72e6e51e014afb9992/coverage-7.16.2-cp310-cp310-win32.whl", hash = "sha256:5139009b5efd2194fc168ee9362f0e191ba612ef5d29242f9269c22f9b8f80c7", size = 225732, upload-time = "2026-09-27T12:25:51.945Z" }, + { url = "https://files.pythonhosted.org/packages/07/86/31f1f3170571a345ab9d8361a7b2f8e0c5698173fbecd1f7149b6e2089ee/coverage-7.16.2-cp310-cp310-win_amd64.whl", hash = "sha256:c3305c38a2fa21a4254f2ace7dd9ef5fc569c9a558b66e7017650b3d637fb95e", size = 226357, upload-time = "2026-09-27T12:25:53.272Z" }, + { url = "https://files.pythonhosted.org/packages/58/fa/ce3baf63d85b730398d92a7162f486f3a5e4e2cc3382a02488b3943725ba/coverage-7.16.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:732d950e51f3ba4fb6209c73250f3e8924fefca42953ee04a9e65d8c02414d7d", size = 223797, upload-time = "2026-09-27T12:25:54.756Z" }, + { url = "https://files.pythonhosted.org/packages/7a/57/9ba29c2aac7f756d479f03d45762120060f0f988788001001bf36e0e6fca/coverage-7.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5dca0bb66b4c3d624ba047887bf70270030c150692d543cb501293dc38a9f4b5", size = 224303, upload-time = "2026-09-27T12:25:56.214Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7b/0d6d60906dca7d28cc1e3fce12a9861801c4fbb6cbf220ad78cd059c9467/coverage-7.16.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:af2a2a8c7c74de0559e0c368d94c8def9e16c58faaee33a0bf081057c4227e3b", size = 254715, upload-time = "2026-09-27T12:25:57.755Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b8/9198b865679379fb165c689c64f6e11105ef380f6bd1c7673e83f73d9f5c/coverage-7.16.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db5f8394e17f877a625b257f2ba0ce8e728a499c2c1579ad66220272cd3df510", size = 256627, upload-time = "2026-09-27T12:25:59.131Z" }, + { url = "https://files.pythonhosted.org/packages/98/79/9521462cb6072fe394701bc8974b74afd576c9c9355156c7844e1a86a42b/coverage-7.16.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5b3146d2317c75f70df2509066d979dadd941f7021cdf9b5db4bcd8568258e25", size = 258724, upload-time = "2026-09-27T12:26:00.691Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/8d7d5d633db9fe0f3182fedc731bf09f9bcf2366055735152504ad614677/coverage-7.16.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e1d0ced76318bab499693ff25f64faa343415187cb2e4d7befdfdd391a1cf6a", size = 260692, upload-time = "2026-09-27T12:26:02.083Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a7/76cb09c89ba46d74d37428bf93251fc14fb0bbe9e05cc2a5ef61773d318a/coverage-7.16.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:af98ad5ed9d6daaca956201e00bb429a7eb2b080426686f70a20353e0f9839f5", size = 254800, upload-time = "2026-09-27T12:26:03.369Z" }, + { url = "https://files.pythonhosted.org/packages/72/b6/2351c1979aaeb5b4a8091a75b90ca997ad60de36e181ddba267cf61dac97/coverage-7.16.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d56e4d21c56d2046447733f8b118409597db48c01efe898ee9ac24e858ec2d6", size = 256423, upload-time = "2026-09-27T12:26:04.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/f4/ad9a4f8b5cb2d494fa9452b546fe742ed2f9d3847cc14c05e36279a3e649/coverage-7.16.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1d5d0e3b660506fb84f995814e3118a21efdc0c8eb80127da1be627d90093c17", size = 254487, upload-time = "2026-09-27T12:26:06.082Z" }, + { url = "https://files.pythonhosted.org/packages/6c/1f/a520470472f3e8b01169bf42162b1470c9ba992230432f62ca36269bf3a0/coverage-7.16.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:17228fbca0f22976f797be94e975dcd237799c657d49551c7de1e0654d1202e9", size = 258259, upload-time = "2026-09-27T12:26:07.513Z" }, + { url = "https://files.pythonhosted.org/packages/09/d2/ff26d5938274745855fa61cfcba0245c88ccc10d98d2cbd96064f16cd5a7/coverage-7.16.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:bc0b0ac781d489304b741269857f1f8338b7a26b1b89c06c0344658001ec0035", size = 254216, upload-time = "2026-09-27T12:26:08.982Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1d/5d832d3b06785d9f53267e4f2724a9f60c312eee6ebed9063a461d0d3b45/coverage-7.16.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bf1bd822ec4e387ed245bed0d71151582cf7be9e5309bc4145eefe36083d5878", size = 255125, upload-time = "2026-09-27T12:26:10.35Z" }, + { url = "https://files.pythonhosted.org/packages/55/4d/1d33edbc2fcf7d99e384e393e712aa5a2ebbbd8409825357815982207976/coverage-7.16.2-cp311-cp311-win32.whl", hash = "sha256:7ed238d227e23cc300c3d464babdaf9f6ddc740aa1b15a77ae96136e6a7c4516", size = 225898, upload-time = "2026-09-27T12:26:11.7Z" }, + { url = "https://files.pythonhosted.org/packages/6f/7c/676df4882118756c4f8f560c954eddb93e166d84dda8c5f0b6a829689bde/coverage-7.16.2-cp311-cp311-win_amd64.whl", hash = "sha256:a90700f743e29aa3d75a6ff5f01953176a889c00e526194bc4d281731b88d99d", size = 226369, upload-time = "2026-09-27T12:26:13.375Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0e/a457f4a461b3c5610d845137fdd45fa465e011a64c25af440518ab1f4e41/coverage-7.16.2-cp311-cp311-win_arm64.whl", hash = "sha256:a336eec40e3520d369b8a6cdabb4f596e69a8b42927ca074aa1452fed943238a", size = 226344, upload-time = "2026-09-27T12:26:15.127Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2c/f8296c63c5d542f3d21aed685e56b7031a419037d155bb3382fc0940d249/coverage-7.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:218d742afca2b5ad5ca759e93eddedfbcc6eadf8322f080dcefc40b7bd4e2d48", size = 223969, upload-time = "2026-09-27T12:26:16.753Z" }, + { url = "https://files.pythonhosted.org/packages/90/23/6f3dcb1423a0d43216e402ea1746e4a7c7c44f38896b97dd573790f56a40/coverage-7.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9a638be322a8d76a41cdb17781c7f82aaee6a66493d8ffb7e2c09ee22423d99", size = 224328, upload-time = "2026-09-27T12:26:18.15Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7d/8f3b6dc920e3fc6732f7678785a2091db439f186afbec30dbf2214d9b1f7/coverage-7.16.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808", size = 255832, upload-time = "2026-09-27T12:26:19.799Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/6c45f15be4eca4ac1062c6a55a323286494c99726a7e58951fe85967ac08/coverage-7.16.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d", size = 258571, upload-time = "2026-09-27T12:26:21.199Z" }, + { url = "https://files.pythonhosted.org/packages/34/fb/b54cbeba3ad89082c2e441278681859e538322cc34b84b2af7ebff00080f/coverage-7.16.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c", size = 259680, upload-time = "2026-09-27T12:26:22.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a2/0dc65ec3d61930e1e4c2e371763b15eb4290896eb343a12d5d3091308116/coverage-7.16.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800", size = 261945, upload-time = "2026-09-27T12:26:24.336Z" }, + { url = "https://files.pythonhosted.org/packages/d6/93/5fad7a61f2c14e08e98946fc31c1c7ffc1195061bf3fdc351db3be77a863/coverage-7.16.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148", size = 256191, upload-time = "2026-09-27T12:26:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/2d/47/74e5de9227b939ece9f64e729645ddc4296bea10dbfa98721c1333c8be2e/coverage-7.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7", size = 257610, upload-time = "2026-09-27T12:26:27.35Z" }, + { url = "https://files.pythonhosted.org/packages/13/fe/2cf28d40b43645d1b72388fe3ee7f7c747533a6a9557bb8c24a7ae74fe1a/coverage-7.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad", size = 255757, upload-time = "2026-09-27T12:26:28.949Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3d/7c149fd99fc8bbc39c80db5e688d1d39fd040be2ecb78b8335a51a55b9c0/coverage-7.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6", size = 259822, upload-time = "2026-09-27T12:26:30.515Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3f/b283fce09d5995e227bd8e513358dd7471bedc0f78abc85a925ebdb0a2f6/coverage-7.16.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd", size = 255325, upload-time = "2026-09-27T12:26:32.037Z" }, + { url = "https://files.pythonhosted.org/packages/bf/91/f3325edf0c4223fb1fe1532b8dbef2a1d2f729459a9a7d1a44d073bae534/coverage-7.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206", size = 257191, upload-time = "2026-09-27T12:26:33.525Z" }, + { url = "https://files.pythonhosted.org/packages/c4/89/21eb5e83ecf2eed523c4eb3d65ae513cd082c8fd1b6deb34c4cb6c332f97/coverage-7.16.2-cp312-cp312-win32.whl", hash = "sha256:152877cdc8a07264882cfcd503ba56a3ef6cba56a70e8c70f6eb8ffd7384789a", size = 226034, upload-time = "2026-09-27T12:26:35.021Z" }, + { url = "https://files.pythonhosted.org/packages/db/de/e3ad6d864c0833624b4f1f9b53f9e58e116c945e5e965c3f1e172c5e84cd/coverage-7.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:e6c52d3307824ff93b39efd99e4185d557db40bd841452abfb32e5d9151ca162", size = 226569, upload-time = "2026-09-27T12:26:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c1/bccc58ebe5489cc70628f635c1932fd371f5d7da850dbcf960f95f4c4afc/coverage-7.16.2-cp312-cp312-win_arm64.whl", hash = "sha256:a678c0b6b22086ec2427359d22e37445d4a792f5fdbbc744112c7dade65cad02", size = 226372, upload-time = "2026-09-27T12:26:38.406Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, + { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/577fc0803dab4155dcf808faffbdd7b159256781c0874a8586e17b81b149/coverage-7.16.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a", size = 224138, upload-time = "2026-09-27T12:27:05.141Z" }, + { url = "https://files.pythonhosted.org/packages/75/9e/e3785ba3ecba2bd11efc74bfe2801ca4b78c4480b15a375648d809a59da3/coverage-7.16.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee", size = 224445, upload-time = "2026-09-27T12:27:06.805Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/963ff22d3fd27117da3b8cc442f5bdc91196f783321e1a8ff0ec43476772/coverage-7.16.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b", size = 256112, upload-time = "2026-09-27T12:27:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d4/a306940c81c6ae759e82fff27d20b7fdc6896e422b821f51313cce212b6c/coverage-7.16.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03", size = 258727, upload-time = "2026-09-27T12:27:09.927Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/d3d99d93b02517087aa05bc0cf2d04d372956b849e5443e059079901429b/coverage-7.16.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda", size = 259909, upload-time = "2026-09-27T12:27:11.55Z" }, + { url = "https://files.pythonhosted.org/packages/08/44/39dd599181726758dd185ae4dc0c0ab3aeabf7ca70e68e145060feeaaa16/coverage-7.16.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c", size = 262481, upload-time = "2026-09-27T12:27:13.17Z" }, + { url = "https://files.pythonhosted.org/packages/99/e8/91ee43f6ded411460c359d7e1aebde4d6fd8f00a2e5394182d9d212eb23c/coverage-7.16.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9", size = 256012, upload-time = "2026-09-27T12:27:14.91Z" }, + { url = "https://files.pythonhosted.org/packages/11/8c/e9499ddc33197bd7eabcb1118ca81756fc874457b324e2b479a4804b2ad2/coverage-7.16.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4", size = 257898, upload-time = "2026-09-27T12:27:16.588Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6e/c081cb5991a0afba99f9c4ad6c74a5fce9513a38ddc64e3e6680c6fed9af/coverage-7.16.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90", size = 255938, upload-time = "2026-09-27T12:27:18.19Z" }, + { url = "https://files.pythonhosted.org/packages/b2/42/1c3d819e8f9b6eb01c2fe90874d67a8882adb9507e0bbb09361ed131ea89/coverage-7.16.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95", size = 260368, upload-time = "2026-09-27T12:27:19.903Z" }, + { url = "https://files.pythonhosted.org/packages/19/4f/d70eac07901fd587b6ab05e659b52afe13959992aa5113bf6cce059cc572/coverage-7.16.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8", size = 255620, upload-time = "2026-09-27T12:27:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/34/5e/6d87af88317d3d9a9b18a9ca1bc1673eb516917f296e579d0d4a55cb3490/coverage-7.16.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6", size = 257521, upload-time = "2026-09-27T12:27:23.358Z" }, + { url = "https://files.pythonhosted.org/packages/79/bb/90c2641170d2fa1a6757b3f8450ba2740197317b0ddd749e9604b914e886/coverage-7.16.2-cp314-cp314-win32.whl", hash = "sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011", size = 226248, upload-time = "2026-09-27T12:27:25.153Z" }, + { url = "https://files.pythonhosted.org/packages/30/08/d8d0478bb02c8eb0ae20a496fc80c40fcf4d3450bd184300d682ba2d28a6/coverage-7.16.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b", size = 226732, upload-time = "2026-09-27T12:27:26.907Z" }, + { url = "https://files.pythonhosted.org/packages/32/3f/0001da22155b0a8ce063ec0f7e64ecbe17b373f306e7a74435f6d6accb72/coverage-7.16.2-cp314-cp314-win_arm64.whl", hash = "sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb", size = 226645, upload-time = "2026-09-27T12:27:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/d7/85/6d8813aff9b8b8586691a9d33c43c5604f7227622574da7cdc3d91a86861/coverage-7.16.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf", size = 224871, upload-time = "2026-09-27T12:27:30.32Z" }, + { url = "https://files.pythonhosted.org/packages/5c/70/444f3a4981ac2cda40fdcf4cc9b56a4e1a33c222abeb33e51ed3e3eb2a6b/coverage-7.16.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036", size = 225123, upload-time = "2026-09-27T12:27:32.33Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c1/980681cd7b33eb66ac835044116ef0a92e11fcc7bdd866cc89d10b1130b9/coverage-7.16.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3", size = 265049, upload-time = "2026-09-27T12:27:34.085Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e3/87679875c33bb2191f0f05544a1cc9adcc940fe0c35443a10f2df753dde5/coverage-7.16.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831", size = 267717, upload-time = "2026-09-27T12:27:36.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/64/5d372776d6eb523d4e93bafba2253f96984e3b18261c4cc56a50863c6d0d/coverage-7.16.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf", size = 270047, upload-time = "2026-09-27T12:27:37.96Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/44082ff0cbf9f97d0043f57970a71204097ec7ba606361a9fd2065393669/coverage-7.16.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681", size = 271360, upload-time = "2026-09-27T12:27:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/b8/17/9a215efe25b5e0ecc87c89dbe525c4a87d14d87c8c0c7316ef140a5f6f3e/coverage-7.16.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0", size = 264736, upload-time = "2026-09-27T12:27:42.072Z" }, + { url = "https://files.pythonhosted.org/packages/a2/da/7f0a31af8e448107d4d32844bd684757f51ea907bc0c68c8fd537b2123ff/coverage-7.16.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb", size = 267600, upload-time = "2026-09-27T12:27:43.85Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/3bfecbd3366b775bacdcb3330394d356cf384b5d8f5b2146ac4b14b252b5/coverage-7.16.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7", size = 264767, upload-time = "2026-09-27T12:27:45.768Z" }, + { url = "https://files.pythonhosted.org/packages/b8/3f/5d62163732d87e4a0c4710a0eab30f0fd6a2d480112abe2029f014fe8c9d/coverage-7.16.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64", size = 269026, upload-time = "2026-09-27T12:27:47.787Z" }, + { url = "https://files.pythonhosted.org/packages/49/4d/8e4579f225426535085a9be371cc75e3b026d058d679b80affbdfb4c3ef0/coverage-7.16.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20", size = 264147, upload-time = "2026-09-27T12:27:49.681Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/ef1f77e2c3f7bb03c2b13b9a2006f88700fdd75535ef158d70049f425c1c/coverage-7.16.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce", size = 266376, upload-time = "2026-09-27T12:27:51.551Z" }, + { url = "https://files.pythonhosted.org/packages/be/79/0cb2bf4428830dec971c718c2c841a039c084415c99e67281f5a72841aab/coverage-7.16.2-cp314-cp314t-win32.whl", hash = "sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46", size = 226585, upload-time = "2026-09-27T12:27:53.945Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f9/da17121c16667fd84998e972200ae226a41540f6ea4795776c6d99e8976f/coverage-7.16.2-cp314-cp314t-win_amd64.whl", hash = "sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c", size = 227378, upload-time = "2026-09-27T12:27:55.778Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/01179c62d1b7e6e33bd5001566b02d7f778cf33d3ec1e81e94ca170c517f/coverage-7.16.2-cp314-cp314t-win_arm64.whl", hash = "sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91", size = 227064, upload-time = "2026-09-27T12:27:57.496Z" }, + { url = "https://files.pythonhosted.org/packages/4c/57/52935003c3f627ba6e5203d7179aad32448c10899663a30336aba8e81a2c/coverage-7.16.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8", size = 224140, upload-time = "2026-09-27T12:27:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/31/38/df472520f3e626524d7e2fc9d6da0afe7895a2f1489d36b48af8ca40bb41/coverage-7.16.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24", size = 224449, upload-time = "2026-09-27T12:28:01.299Z" }, + { url = "https://files.pythonhosted.org/packages/0c/aa/3be084d5b82e63ccdad4ed751e4acbae294673573e30481d29f8b7402eec/coverage-7.16.2-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09", size = 256060, upload-time = "2026-09-27T12:28:03.185Z" }, + { url = "https://files.pythonhosted.org/packages/de/29/48fca82a7ebf7ff7b2e35019cc9537e7f65e4d2aa1215cc5a8792c989251/coverage-7.16.2-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843", size = 259106, upload-time = "2026-09-27T12:28:05.15Z" }, + { url = "https://files.pythonhosted.org/packages/06/3d/b2d5986f2dd53fe201aa1be2e4ab204fa1aed5101e67c0dbbb419b850aee/coverage-7.16.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1", size = 260674, upload-time = "2026-09-27T12:28:06.868Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7e/b50160be3506ead12e6480d14279af7f0f17627694300a2d1fd2c42d2ff5/coverage-7.16.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982", size = 263139, upload-time = "2026-09-27T12:28:08.771Z" }, + { url = "https://files.pythonhosted.org/packages/14/5e/7c805ac9a32606de1399bd7e9bd375aa2f973dc61b12680d9e6403c2e891/coverage-7.16.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34", size = 256591, upload-time = "2026-09-27T12:28:10.842Z" }, + { url = "https://files.pythonhosted.org/packages/ab/9e/76f1ed129a2daf658a3ea17122824cf2e3b91fea0460d8d3664fc5a61018/coverage-7.16.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6", size = 258708, upload-time = "2026-09-27T12:28:12.962Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b7/8d62e75f48b527619239a65294f842d4b7fd02a0839d43ae1de80184e2df/coverage-7.16.2-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4", size = 256562, upload-time = "2026-09-27T12:28:14.934Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8d/0a15f95c3afb78e947c52644786ba4bc9de259905687dd720d5e6fae2e76/coverage-7.16.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0", size = 261077, upload-time = "2026-09-27T12:28:17.215Z" }, + { url = "https://files.pythonhosted.org/packages/25/00/88389987305a47d732866c07c8a500000ab574df9505e3114ac69c8d027f/coverage-7.16.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a", size = 256034, upload-time = "2026-09-27T12:28:19.08Z" }, + { url = "https://files.pythonhosted.org/packages/92/02/34d079d4952ad461bde037d353f9a6e037a7edc45fe0f9ee8781ff73f028/coverage-7.16.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63", size = 258034, upload-time = "2026-09-27T12:28:21.242Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d8/3e59a62879285b464ec1b10fd824fbc1af9ce66e842cd39974f80a0becc4/coverage-7.16.2-cp315-cp315-win32.whl", hash = "sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e", size = 226252, upload-time = "2026-09-27T12:28:23.102Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e1/128026e1b2836e9ad6b219207ba9edf1c5e0088a7869e23088aee7fbbe7a/coverage-7.16.2-cp315-cp315-win_amd64.whl", hash = "sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae", size = 226728, upload-time = "2026-09-27T12:28:25.21Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f4/c9fa8e7cf525ca7748ac52b0ee89331d13fe09808e45c679830708782e90/coverage-7.16.2-cp315-cp315-win_arm64.whl", hash = "sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495", size = 226648, upload-time = "2026-09-27T12:28:27.136Z" }, + { url = "https://files.pythonhosted.org/packages/a2/13/e96b045447a856666f36f9c653e2a80bdaa732aaaf72412b19aa2c26a473/coverage-7.16.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75", size = 224858, upload-time = "2026-09-27T12:28:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/23/90/087f6ad1bd3df059632ca3407a4e6552ed1053ee35354de0a771acf35423/coverage-7.16.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55", size = 225139, upload-time = "2026-09-27T12:28:31.131Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8e/285dcef0184358044e7cbcd810a1bdc9566bc620f54702d605477155df4a/coverage-7.16.2-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2", size = 265090, upload-time = "2026-09-27T12:28:33.04Z" }, + { url = "https://files.pythonhosted.org/packages/06/b2/cc83f3a6e5789a4e89059c69555bc641c2efcde568405a1c06fc702951ab/coverage-7.16.2-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2", size = 268254, upload-time = "2026-09-27T12:28:35.135Z" }, + { url = "https://files.pythonhosted.org/packages/ac/41/f548c19530f5d66ac6e3c92bbcbc49da7261de3a458b9f3e54a3efb1a0b2/coverage-7.16.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1", size = 270695, upload-time = "2026-09-27T12:28:36.959Z" }, + { url = "https://files.pythonhosted.org/packages/94/61/4dc27cf82ef96434d2874110ad0cc10ea4621025705dc5049862bd3bd181/coverage-7.16.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f", size = 271853, upload-time = "2026-09-27T12:28:38.821Z" }, + { url = "https://files.pythonhosted.org/packages/38/29/bf8072b1b8bd5f2de8b21460a404460b1a2b97e80a9464c78ec0271f6199/coverage-7.16.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d", size = 265644, upload-time = "2026-09-27T12:28:40.815Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2f/0aecb8721be5cdeb8afd9d6d9f6b463f074e4d8d37f00f4c42442522709f/coverage-7.16.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0", size = 268530, upload-time = "2026-09-27T12:28:42.725Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0b/92b4b7628268ee711249958e68fc0328779bd3d9a7ab4715379465aedb84/coverage-7.16.2-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283", size = 265154, upload-time = "2026-09-27T12:28:44.684Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/41c95c1ab29b3dcd357cd1227181d1c98185632aca41ce670ce671b23a43/coverage-7.16.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec", size = 269799, upload-time = "2026-09-27T12:28:46.672Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/ebeb259aa5362b033a137b86d7274ff4b109d59be8cc9913889b783bf75a/coverage-7.16.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5", size = 265196, upload-time = "2026-09-27T12:28:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/b2/18/8437620f90d023680a072eee02f968055f3658bbfb7d386d0ea34cfb7f30/coverage-7.16.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c", size = 267266, upload-time = "2026-09-27T12:28:51.361Z" }, + { url = "https://files.pythonhosted.org/packages/28/6c/f08e8ee4293e6434035424180bef4d45e028e8ecc006c61bf9453e74405e/coverage-7.16.2-cp315-cp315t-win32.whl", hash = "sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35", size = 226583, upload-time = "2026-09-27T12:28:53.33Z" }, + { url = "https://files.pythonhosted.org/packages/f7/fd/3f939c2847f4a72c20cff8b1ac33da78ea91a2d38d9b43336e60db719103/coverage-7.16.2-cp315-cp315t-win_amd64.whl", hash = "sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5", size = 227374, upload-time = "2026-09-27T12:28:55.158Z" }, + { url = "https://files.pythonhosted.org/packages/5a/35/b98cdc354c952402132e675a87f2cc3227fb68f959c84aaa491fbe15933d/coverage-7.16.2-cp315-cp315t-win_arm64.whl", hash = "sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8", size = 227064, upload-time = "2026-09-27T12:28:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + [[package]] name = "deepmerge" version = "3.0" @@ -697,6 +836,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -739,7 +887,8 @@ name = "markdown-it-py" version = "4.2.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10' and python_full_version < '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", ] dependencies = [ { name = "mdurl" }, @@ -1132,6 +1281,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl", hash = "sha256:7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4", size = 23361, upload-time = "2026-08-10T15:48:04.855Z" }, ] +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + [[package]] name = "prek" version = "0.5.3" @@ -1414,6 +1572,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From c8e904a47a0ea75b0e8964a0b9cc75956f82e8f0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:16:21 +1000 Subject: [PATCH 107/110] say in the cli reference that a by run --coverage run ended by os.exec* or a signal leaves no report either --- docs/basedpython/cli-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index e73947e604..92963753c7 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -137,7 +137,7 @@ compiled itself, which runs as the staged copy it compiled: a `source` or an `omit` naming it does not reach it. a `.by` module under `source` that never ran is reported as not run, as coverage.py reports a `.py` that never ran. a program that -leaves without returning to `by run` — `os._exit` — leaves no report, and +leaves without returning to `by run` — on `os._exit`, an `os.exec*` or a signal — leaves no report, and `--compiled` cannot be combined with it, since a native module runs no python lines to measure From e84c6f2e98c8bec9acfc079fdff7ab79d4713961 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:19:49 +1000 Subject: [PATCH 108/110] correct what dedent-strings.md says a dedented string is: no bytes or implicit concatenation, a syntax error the checker reports, and not textwrap.dedent's result --- docs/basedpython/features/dedent-strings.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/basedpython/features/dedent-strings.md b/docs/basedpython/features/dedent-strings.md index afc21cd5d2..36a0674847 100644 --- a/docs/basedpython/features/dedent-strings.md +++ b/docs/basedpython/features/dedent-strings.md @@ -26,18 +26,19 @@ dedenting fires when *all* of these hold: - the string is triple-quoted (`"""` or `'''`) - the opening quote is immediately followed by a newline (`"""\n`) -- the final line before the closing quote is whitespace-only +- the closing quotes sit on a line of their own, after nothing but whitespace inline triple-quoted strings (`"""one line"""`) and strings without a newline after the opening quote are left untouched. the rule applies -uniformly to plain strings, raw (`r"""..."""`), f-strings (`f"""..."""`), -t-strings (`t"""..."""`), and prefixed combinations +to plain strings, raw (`r"""..."""`), f-strings (`f"""..."""`) and +t-strings (`t"""..."""`). a bytes literal is never dedented, and nor is a +string written beside another in an implicit concatenation ## closing-quote indent rule the closing `"""` must be indented **no more** than the content's common leading indent. otherwise the rewrite has no consistent dedent depth and -the transpiler raises an error: +the transpiler raises an error, and the checker reports a syntax error: ```by text = """ @@ -53,8 +54,11 @@ case ## relationship to PEP 750 / `textwrap.dedent` -the resulting string is identical to what `textwrap.dedent` would produce, -but there is no runtime call. the rewrite preserves embedded f-string +there is no runtime call, and the result is not what `textwrap.dedent` would +produce: the opening newline and the line the closing quotes sit on are dropped, +so `textwrap.dedent("\n a\n ")` is `"\na\n"` and the same text written +this way is `"a"`. a `\` that continues a line joins the next line after that +line's indent has been stripped. the rewrite preserves embedded f-string interpolations: ```by From 230c8be0d9734560e40a947db2499982727a030a Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:26:33 +1000 Subject: [PATCH 109/110] drop hawk.toml's expectation for TargetVersion::linter_version, which ty_ide now calls, and expect HierarchicalSymbols::get as unnecessarily public now ty_ide uses it --- hawk.toml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/hawk.toml b/hawk.toml index 41c98b80a0..8d9f9e1e8d 100644 --- a/hawk.toml +++ b/hawk.toml @@ -118,14 +118,6 @@ kind = "inherent_method" level = "expect" reason = "public line-ending settings documentation links to this representation accessor" -[[override]] -lint = "hawk::unnecessary_public" -crate = "ruff_linter" -item = "settings::TargetVersion::linter_version" -kind = "inherent_method" -level = "expect" -reason = "preserves the public parser and linter target-version accessor pair" - [[override]] lint = "hawk::unnecessary_public" crate = "ruff_python_parser" @@ -608,7 +600,7 @@ level = "expect" reason = "flat and hierarchical symbol collections intentionally share collection APIs" [[override]] -lint = "hawk::dead_public" +lint = "hawk::unnecessary_public" crate = "ty_ide" item = "symbols::HierarchicalSymbols::get" kind = "inherent_method" From ecde89605875f9672e08e491927e91efa5a2296f Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:54:42 +1000 Subject: [PATCH 110/110] test by run --coverage's report of a module that never ran on each core coverage.py can use on the interpreter, rather than on ctrace and sysmon wherever it runs, since python 3.11 has no sys.monitoring and coverage.py's build for free-threaded 3.13 has no C tracer --- crates/ty/tests/by_e2e.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 91925850d5..6505201ad0 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1274,6 +1274,31 @@ fn coverage_interpreter() -> (String, bool) { (python, importable) } +/// The cores coverage.py can measure with on `python`, decided as coverage.py +/// decides it: `ctrace` needs its C tracer, which its build for free-threaded +/// python 3.13 lacks, and `sysmon` needs `sys.monitoring`, which python 3.12 added +fn coverage_cores(python: &str) -> Vec { + let output = Command::new(python) + .args([ + "-c", + "import coverage.env\n\ + try:\n from coverage.tracer import CTracer\nexcept ImportError:\n pass\n\ + else:\n print('ctrace')\n\ + if coverage.env.PYBEHAVIOR.pep669:\n print('sysmon')", + ]) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_owned) + .collect() +} + /// A project with a transpiled module that runs only partly, a hand-written /// `.py` the build carries over, and an entry point calling into both. fn project_with_partly_covered_modules(dir: &Path) { @@ -1498,7 +1523,12 @@ fn run_coverage_reports_a_by_module_that_never_ran() { .unwrap(); let project = dir.path().canonicalize().unwrap(); - for core in ["ctrace", "sysmon"] { + let cores = coverage_cores(&python); + assert!( + !cores.is_empty(), + "coverage.py has no core to test on {python}" + ); + for core in &cores { let output = Command::new(env!("CARGO_BIN_EXE_by")) .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .env("COVERAGE_CORE", core)