From 70978702432f2f7bec97e932cd10208dfe192dc0 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 12 Oct 2026 17:36:19 -0400 Subject: [PATCH 1/7] Bind extension methods to the extended type so they are callable from the instance C# extension methods were only reachable from Python through their static class, so instance-style calls such as order_status.is_open() failed with AttributeError. Extension methods declared in non-framework assemblies are now indexed during the assembly scan and bound as members of the type they extend, including interface targets on implementing classes. The binder passes the instance as the first argument when an extension method is called through it. Real members with the same name win, and open generic targets are skipped since the binder cannot infer them from the instance. --- CHANGELOG.md | 1 + src/embed_tests/TestExtensionMethods.cs | 193 ++++++++++++++++++++++++ src/runtime/AssemblyManager.cs | 6 +- src/runtime/ClassManager.cs | 11 ++ src/runtime/ExtensionMethods.cs | 114 ++++++++++++++ src/runtime/MethodBinder.cs | 23 +++ 6 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 src/embed_tests/TestExtensionMethods.cs create mode 100644 src/runtime/ExtensionMethods.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d72a3a0..4c2fe6fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`) - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs new file mode 100644 index 000000000..316b49bf6 --- /dev/null +++ b/src/embed_tests/TestExtensionMethods.cs @@ -0,0 +1,193 @@ +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public enum ExtendedStatus { Open, Closed } + + public interface IExtendedShape + { + double Side { get; } + } + + public class ExtendedModel + { + public string Name { get; set; } = "model"; + + public string Existing() => "member"; + } + + public class DerivedExtendedModel : ExtendedModel + { + } + + public class ExtendedSquare : IExtendedShape + { + public double Side => 2; + } + + public static class ExtendedStatusExtensions + { + public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open; + + public static string Describe(this ExtendedStatus status, string prefix, int times = 1) + => string.Concat(Enumerable.Repeat(prefix + status, times)); + } + + public static class ExtendedModelExtensions + { + public static string Describe(this ExtendedModel model, string prefix = "") => prefix + model.Name; + + // collides with the real instance method, which must win + public static string Existing(this ExtendedModel model) => "extension"; + + public static double Area(this IExtendedShape shape) => shape.Side * shape.Side; + + public static int Total(this List values) => values.Sum(); + + public static T Identity(this T value) => value; + + public static string Twice(this string value) => value + value; + } + + public class TestExtensionMethods + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from System.Collections.Generic import List +from Python.EmbeddingTest import * + +def enum_instance(): + return ExtendedStatus.Open.IsOpen(), ExtendedStatus.Open.is_open(), ExtendedStatus.Closed.is_open() + +def enum_with_args(): + return ExtendedStatus.Open.describe('s-', times=2) + +def enum_static_forms(): + return ExtendedStatusExtensions.is_open(ExtendedStatus.Open), ExtendedStatus.is_open(ExtendedStatus.Closed) + +def class_instance(): + return ExtendedModel().describe(), ExtendedModel().describe(prefix='p-') + +def derived_inherits(): + return DerivedExtendedModel().describe('d-') + +def interface_target(): + return ExtendedSquare().area() + +def member_wins(): + return ExtendedModel().existing() + +def closed_generic(): + values = List[int]() + values.Add(1) + values.Add(2) + return values.total() + +def open_generic_not_bound(): + return hasattr(ExtendedModel(), 'identity') + +def python_native_not_bound(): + return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab') +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("extension_methods", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void EnumExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.enum_instance(); + Assert.IsTrue((bool)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsFalse((bool)result[2]); + } + + [Test] + public void ExtensionArgumentsFollowTheInstance() + { + using var _ = Py.GIL(); + Assert.AreEqual("s-Opens-Open", (string)module.enum_with_args()); + } + + [Test] + public void StaticFormsKeepWorking() + { + using var _ = Py.GIL(); + var result = module.enum_static_forms(); + Assert.IsTrue((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + + [Test] + public void ClassExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.class_instance(); + Assert.AreEqual("model", (string)result[0]); + Assert.AreEqual("p-model", (string)result[1]); + } + + [Test] + public void DerivedClassInheritsBaseClassExtensions() + { + using var _ = Py.GIL(); + Assert.AreEqual("d-model", (string)module.derived_inherits()); + } + + [Test] + public void InterfaceExtensionIsBoundToImplementingClass() + { + using var _ = Py.GIL(); + Assert.AreEqual(4.0, (double)module.interface_target()); + } + + [Test] + public void RealMemberWinsOverExtension() + { + using var _ = Py.GIL(); + Assert.AreEqual("member", (string)module.member_wins()); + } + + [Test] + public void ClosedGenericTargetIsBound() + { + using var _ = Py.GIL(); + Assert.AreEqual(3, (int)module.closed_generic()); + } + + [Test] + public void OpenGenericTargetIsNotBound() + { + using var _ = Py.GIL(); + Assert.IsFalse((bool)module.open_generic_not_bound()); + } + + [Test] + public void PythonNativeTargetKeepsTheStaticForm() + { + using var _ = Py.GIL(); + var result = module.python_native_not_bound(); + Assert.IsFalse((bool)result[0]); + Assert.AreEqual("abab", (string)result[1]); + } + } +} diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index 3370e4410..76b252652 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -62,6 +62,7 @@ internal static void Initialize() // test/usage after the first init cycle. Clear so the scan runs fresh. assembliesNamesCache.Clear(); assemblies.Clear(); + ExtensionMethods.Clear(); AppDomain domain = AppDomain.CurrentDomain; @@ -356,7 +357,8 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + var types = GetTypes(assembly); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -383,6 +385,8 @@ internal static void ScanAssembly(Assembly assembly) GenericUtil.Register(t); } } + + ExtensionMethods.Register(assembly, types); } public static AssemblyName[] ListAssemblies() diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b88a6a6b6..835f0ec82 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -416,6 +416,17 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) } } + // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C# + var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray(); + if (extensionMethods.Length > 0) + { + foreach (var em in extensionMethods) + { + local.Add(em.Name); + } + info = info.Concat(extensionMethods).ToArray(); + } + // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) { diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs new file mode 100644 index 000000000..2458057c3 --- /dev/null +++ b/src/runtime/ExtensionMethods.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Python.Runtime +{ + /// + /// Index of the C# extension methods found in the scanned assemblies, keyed by the type they extend. + /// Classes bind them as members so they are callable from an instance, like C# callers do. + /// + internal static class ExtensionMethods + { + private static readonly ConcurrentDictionary> _methodsByExtendedType = new(); + + // The framework ships thousands of extension methods (LINQ above all); binding them onto + // every collection would bloat each class, so only user assemblies are indexed. + private static readonly string[] _frameworkAssemblyPrefixes = { "System", "Microsoft", "mscorlib", "netstandard", "WindowsBase", "Python.Runtime" }; + + /// + /// Drops the index so a re-initialized runtime rebuilds it while re-scanning the assemblies + /// + internal static void Clear() + { + _methodsByExtendedType.Clear(); + } + + internal static bool IsExtensionMethod(MethodBase method) + { + return method.IsStatic && method.IsDefined(typeof(ExtensionAttribute), false); + } + + /// + /// Indexes the extension methods declared by the given exported types of an assembly + /// + internal static void Register(Assembly assembly, IEnumerable exportedTypes) + { + if (IsFrameworkAssembly(assembly)) + { + return; + } + + foreach (var type in exportedTypes) + { + // extension methods can only live in non-generic static classes + if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false)) + { + continue; + } + + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (!IsExtensionMethod(method)) + { + continue; + } + + var extendedType = method.GetParameters()[0].ParameterType; + // The binder cannot infer open generic targets (this T, this IEnumerable) from the instance + if (extendedType.ContainsGenericParameters || extendedType.IsByRef) + { + continue; + } + + var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List()); + lock (methods) + { + methods.Add(method); + } + } + } + } + + /// + /// Gets the extension methods to bind to the given type: those extending the type itself and the + /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy. + /// + internal static List GetMethodsToBind(Type type) + { + var result = new List(); + AddMethods(type, result); + + if (!type.IsInterface) + { + var baseInterfaces = type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; + foreach (var @interface in type.GetInterfaces().Except(baseInterfaces)) + { + AddMethods(@interface, result); + } + } + + return result; + } + + private static void AddMethods(Type extendedType, List result) + { + if (_methodsByExtendedType.TryGetValue(extendedType, out var methods)) + { + lock (methods) + { + result.AddRange(methods); + } + } + } + + private static bool IsFrameworkAssembly(Assembly assembly) + { + var name = assembly.GetName().Name ?? string.Empty; + return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + } + } +} diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 6deec74f8..803ee2f23 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -534,6 +534,17 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe pi = pi.Take(1).ToArray(); } + // An extension method called through an instance gets the instance as its first CLR argument + bool isExtension = inst != null && methodInformation.IsExtension; + if (isExtension) + { + pi = pi.Skip(1).ToArray(); + if (hasNamedArgs) + { + paramNames = paramNames.Skip(1).ToArray(); + } + } + // Must be done after IsOperator section int clrArgCount = pi.Length; @@ -775,6 +786,15 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe } } + if (isExtension) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + margs = new[] { co.inst }.Concat(margs).ToArray(); + } + else continue; + } + var match = new MatchedMethod(kwargsMatched, margs, outs, methodInformation, implicitConversions); if (implicitConversions > 0) { @@ -1238,9 +1258,12 @@ internal class MethodInformation { private ParameterInfo[] _parameterInfo; private string[] _parametersNames; + private bool? _isExtension; public MethodBase MethodBase { get; } + public bool IsExtension => _isExtension ??= ExtensionMethods.IsExtensionMethod(MethodBase); + public bool IsOriginal { get; set; } public ParameterInfo[] ParameterInfo From da8b3d51219990a479b89ef696e1bbce808fb57b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 12 Oct 2026 17:36:34 -0400 Subject: [PATCH 2/7] Update version to 2.0.67 --- src/perf_tests/Python.PerformanceTests.csproj | 4 ++-- src/runtime/Properties/AssemblyInfo.cs | 4 ++-- src/runtime/Python.Runtime.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index dd7061c2b..5efef4906 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 0a24781b8..dacadc64d 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.66")] -[assembly: AssemblyFileVersion("2.0.66")] +[assembly: AssemblyVersion("2.0.67")] +[assembly: AssemblyFileVersion("2.0.67")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 765a0dd2c..e290143d9 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.66 + 2.0.67 false LICENSE https://github.com/pythonnet/pythonnet From 2056895e0c71ec6c82d2167459c0800a9f9b9b5f Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 14 Oct 2026 14:41:42 -0400 Subject: [PATCH 3/7] Give types with an implicit numeric conversion Python's number protocol Python operators never consult C# implicit conversions, so a type convertible to decimal or double still raised TypeError on addition, comparison and float(). Such types now get arithmetic, comparison, __float__ and __int__ methods delegating to the conversion, grafted at class binding the same way EnumOps is for enums. Operators the type declares itself are kept. --- CHANGELOG.md | 1 + .../TestImplicitNumericOperators.cs | 162 ++++++++++++++++++ src/runtime/ClassManager.cs | 10 ++ src/runtime/Native/ITypeOffsets.cs | 1 + src/runtime/Native/TypeOffset.cs | 1 + src/runtime/Types/OperatorMethod.cs | 6 + src/runtime/Util/ImplicitNumericOps.cs | 95 ++++++++++ 7 files changed, 276 insertions(+) create mode 100644 src/embed_tests/TestImplicitNumericOperators.cs create mode 100644 src/runtime/Util/ImplicitNumericOps.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c2fe6fbd..e444838c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added - Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`) +- Types with an implicit conversion to a numeric type support Python arithmetic, comparisons, `float()` and `int()` through that conversion, unless they declare the operator themselves - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestImplicitNumericOperators.cs b/src/embed_tests/TestImplicitNumericOperators.cs new file mode 100644 index 000000000..3b580adaf --- /dev/null +++ b/src/embed_tests/TestImplicitNumericOperators.cs @@ -0,0 +1,162 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class ConvertibleFee + { + public decimal Amount { get; } + + public ConvertibleFee(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleFee fee) => fee.Amount; + } + + public class ConvertibleRatio + { + public double Value { get; } + + public ConvertibleRatio(double value) + { + Value = value; + } + + public static implicit operator double(ConvertibleRatio ratio) => ratio.Value; + } + + public class ConvertibleMoney + { + public decimal Amount { get; } + + public ConvertibleMoney(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleMoney money) => money.Amount; + + // the type's own operator must keep winning over the synthesized one + public static ConvertibleMoney operator +(ConvertibleMoney a, ConvertibleMoney b) => new(a.Amount + b.Amount); + } + + public class NotConvertible + { + } + + public class TestImplicitNumericOperators + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from Python.EmbeddingTest import * + +fee = ConvertibleFee(2) +other = ConvertibleFee(3) + +def arithmetic(): + return (fee + other, fee + 1, 1 + fee, fee - other, fee - 1, 10 - fee, + fee * other, fee * 2, 2 * fee, other / fee, fee / 2, 10 / fee, -fee) + +def aggregates(): + return sum([fee, other]), max(fee, other) is other, sorted([other, fee])[0] is fee + +def comparisons(): + return (fee < other, fee > other, fee <= other, fee >= other, + fee > 0, fee < 2, fee <= 2, fee >= 2, 1 < fee, 2 >= fee) + +def conversions(): + return float(fee), int(ConvertibleRatio(1.75)), float(ConvertibleRatio(1.75)) + +def own_operator_wins(): + total = ConvertibleMoney(2) + ConvertibleMoney(3) + return type(total).__name__, total.Amount, ConvertibleMoney(2) - ConvertibleMoney(1) + +def not_convertible(): + return hasattr(NotConvertible(), '__add__'), hasattr(NotConvertible(), '__float__') +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("implicit_numeric_operators", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void ArithmeticDelegatesToTheImplicitConversion() + { + using var _ = Py.GIL(); + var result = module.arithmetic(); + var expected = new[] { 5.0, 3.0, 3.0, -1.0, 1.0, 8.0, 6.0, 4.0, 4.0, 1.5, 1.0, 5.0, -2.0 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (double)result[i], $"index {i}"); + } + } + + [Test] + public void AggregatesWork() + { + using var _ = Py.GIL(); + var result = module.aggregates(); + Assert.AreEqual(5.0, (double)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsTrue((bool)result[2]); + } + + [Test] + public void ComparisonsDelegateToTheImplicitConversion() + { + using var _ = Py.GIL(); + var result = module.comparisons(); + var expected = new[] { true, false, true, false, true, false, true, true, true, true }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (bool)result[i], $"index {i}"); + } + } + + [Test] + public void FloatAndIntConversions() + { + using var _ = Py.GIL(); + var result = module.conversions(); + Assert.AreEqual(2.0, (double)result[0]); + Assert.AreEqual(1, (int)result[1]); + Assert.AreEqual(1.75, (double)result[2]); + } + + [Test] + public void TheTypesOwnOperatorsKeepWinning() + { + using var _ = Py.GIL(); + var result = module.own_operator_wins(); + Assert.AreEqual("ConvertibleMoney", (string)result[0]); + Assert.AreEqual(5m, (decimal)result[1]); + // subtraction is not declared by the type, so the synthesized one applies + Assert.AreEqual(1.0, (double)result[2]); + } + + [Test] + public void TypesWithoutAConversionAreUntouched() + { + using var _ = Py.GIL(); + var result = module.not_convertible(); + Assert.IsFalse((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + } +} diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index 835f0ec82..6cc8886af 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -415,6 +415,16 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) info = info.Concat(opsImpl.GetMethods(OpsHelper.BindingFlags)).ToArray(); } } + else if (ImplicitNumericOps.IsSupported(type)) + { + // Types with an implicit numeric conversion get Python's number protocol, keeping the operators they declare themselves + var ops = ImplicitNumericOps.GetMethods(type).Where(op => !local.Contains(op.Name)).ToArray(); + foreach (var op in ops) + { + local.Add(op.Name); + } + info = info.Concat(ops).ToArray(); + } // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C# var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray(); diff --git a/src/runtime/Native/ITypeOffsets.cs b/src/runtime/Native/ITypeOffsets.cs index fb65e76f8..bae8636c5 100644 --- a/src/runtime/Native/ITypeOffsets.cs +++ b/src/runtime/Native/ITypeOffsets.cs @@ -22,6 +22,7 @@ interface ITypeOffsets int nb_true_divide { get; } int nb_and { get; } int nb_int { get; } + int nb_float { get; } int nb_or { get; } int nb_xor { get; } int nb_lshift { get; } diff --git a/src/runtime/Native/TypeOffset.cs b/src/runtime/Native/TypeOffset.cs index c94447f37..76196bcad 100644 --- a/src/runtime/Native/TypeOffset.cs +++ b/src/runtime/Native/TypeOffset.cs @@ -31,6 +31,7 @@ static partial class TypeOffset internal static int nb_or { get; private set; } internal static int nb_xor { get; private set; } internal static int nb_int { get; private set; } + internal static int nb_float { get; private set; } internal static int nb_lshift { get; private set; } internal static int nb_rshift { get; private set; } internal static int nb_remainder { get; private set; } diff --git a/src/runtime/Types/OperatorMethod.cs b/src/runtime/Types/OperatorMethod.cs index e905375e0..ae214b05c 100644 --- a/src/runtime/Types/OperatorMethod.cs +++ b/src/runtime/Types/OperatorMethod.cs @@ -56,6 +56,7 @@ static OperatorMethod() ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), ["__int__"] = new SlotDefinition("__int__", TypeOffset.nb_int), + ["__float__"] = new SlotDefinition("__float__", TypeOffset.nb_float), }; ComparisonOpMap = new Dictionary { @@ -136,6 +137,11 @@ static IEnumerable GetOperatorCandidates(Type clrType) .Concat(typeof(FlagEnumOps<>).MakeGenericType(clrType).GetMethods(flags)); } + if (ImplicitNumericOps.IsSupported(clrType)) + { + return clrType.GetMethods(flags).Concat(ImplicitNumericOps.GetMethods(clrType)); + } + return clrType.GetMethods(flags); } diff --git a/src/runtime/Util/ImplicitNumericOps.cs b/src/runtime/Util/ImplicitNumericOps.cs new file mode 100644 index 000000000..8f4146a7a --- /dev/null +++ b/src/runtime/Util/ImplicitNumericOps.cs @@ -0,0 +1,95 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; + +namespace Python.Runtime +{ + /// + /// Locates the implicit numeric conversion of a CLR type. Python operators never consult implicit + /// conversions, so types declaring one get the number protocol through . + /// + internal static class ImplicitNumericOps + { + private static readonly Type[] NumericTypes = + { + typeof(decimal), typeof(double), typeof(float), typeof(long), typeof(int), typeof(short), typeof(sbyte), + typeof(ulong), typeof(uint), typeof(ushort), typeof(byte) + }; + + internal static MethodInfo? GetConversion(Type type) + { + if (type.IsEnum || type.IsPrimitive || type == typeof(decimal) || type.ContainsGenericParameters) + { + return null; + } + + return type.GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(method => method.Name == "op_Implicit" + && NumericTypes.Contains(method.ReturnType) + && method.GetParameters().Length == 1 + && method.GetParameters()[0].ParameterType == type); + } + + internal static bool IsSupported(Type type) => GetConversion(type) != null; + + internal static MethodInfo[] GetMethods(Type type) + { + return typeof(ImplicitNumericOps<>).MakeGenericType(type).GetMethods(OpsHelper.BindingFlags); + } + } + + [Ops] + internal static class ImplicitNumericOps + { + private static readonly Func ToDouble = BuildConversion(); + +#pragma warning disable IDE1006 // Naming Styles - must match Python + [ForbidPythonThreads] + public static double __float__(T value) => ToDouble(value); + + [ForbidPythonThreads] + public static long __int__(T value) => (long)Math.Truncate(ToDouble(value)); +#pragma warning restore IDE1006 + + public static double op_UnaryNegation(T value) => -ToDouble(value); + + public static double op_Addition(T a, T b) => ToDouble(a) + ToDouble(b); + public static double op_Addition(T a, double b) => ToDouble(a) + b; + public static double op_Addition(double a, T b) => a + ToDouble(b); + + public static double op_Subtraction(T a, T b) => ToDouble(a) - ToDouble(b); + public static double op_Subtraction(T a, double b) => ToDouble(a) - b; + public static double op_Subtraction(double a, T b) => a - ToDouble(b); + + public static double op_Multiply(T a, T b) => ToDouble(a) * ToDouble(b); + public static double op_Multiply(T a, double b) => ToDouble(a) * b; + public static double op_Multiply(double a, T b) => a * ToDouble(b); + + public static double op_Division(T a, T b) => ToDouble(a) / ToDouble(b); + public static double op_Division(T a, double b) => ToDouble(a) / b; + public static double op_Division(double a, T b) => a / ToDouble(b); + + // Python reflects a reversed comparison onto the other operand, so only the forward overloads are needed + public static bool op_LessThan(T a, T b) => ToDouble(a) < ToDouble(b); + public static bool op_LessThan(T a, double b) => ToDouble(a) < b; + + public static bool op_GreaterThan(T a, T b) => ToDouble(a) > ToDouble(b); + public static bool op_GreaterThan(T a, double b) => ToDouble(a) > b; + + public static bool op_LessThanOrEqual(T a, T b) => ToDouble(a) <= ToDouble(b); + public static bool op_LessThanOrEqual(T a, double b) => ToDouble(a) <= b; + + public static bool op_GreaterThanOrEqual(T a, T b) => ToDouble(a) >= ToDouble(b); + public static bool op_GreaterThanOrEqual(T a, double b) => ToDouble(a) >= b; + + private static Func BuildConversion() + { + var conversion = ImplicitNumericOps.GetConversion(typeof(T)) + ?? throw new InvalidOperationException($"{typeof(T)} has no implicit numeric conversion"); + var value = Expression.Parameter(typeof(T), "value"); + var body = Expression.Convert(Expression.Call(conversion, value), typeof(double)); + return Expression.Lambda>(body, value).Compile(); + } + } +} From 9d331428de911d317de787f6199fd573c4675d8d Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 14 Oct 2026 14:41:42 -0400 Subject: [PATCH 4/7] Update version to 2.0.68 --- src/perf_tests/Python.PerformanceTests.csproj | 4 ++-- src/runtime/Properties/AssemblyInfo.cs | 4 ++-- src/runtime/Python.Runtime.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index 5efef4906..da5140d91 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index dacadc64d..f5b913a33 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.67")] -[assembly: AssemblyFileVersion("2.0.67")] +[assembly: AssemblyVersion("2.0.68")] +[assembly: AssemblyFileVersion("2.0.68")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index e290143d9..57dde5de7 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.67 + 2.0.68 false LICENSE https://github.com/pythonnet/pythonnet From 4444105f93b6eea757aaa9100ab805002f521559 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 16 Oct 2026 09:58:10 -0300 Subject: [PATCH 5/7] Bind extension methods only to targets involving a non-framework type Targets made only of framework types are shared by countless classes: extensions on object reached every CLR object, and Accord.Math alone added ~600 methods to each primitive array type. Across Lean's assemblies the index shrinks from 16487 to 653 methods. Such extensions keep their static form. Assemblies without the compiler's assembly-level [Extension] marker are skipped up front, as C# does, and binding a class only walks its interfaces when an interface target was indexed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/embed_tests/TestExtensionMethods.cs | 63 ++++++++++++++++++++- src/runtime/ExtensionMethods.cs | 75 ++++++++++++++++++++----- 2 files changed, 120 insertions(+), 18 deletions(-) diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs index 316b49bf6..74906e4d2 100644 --- a/src/embed_tests/TestExtensionMethods.cs +++ b/src/embed_tests/TestExtensionMethods.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using NUnit.Framework; @@ -28,6 +30,13 @@ public class ExtendedSquare : IExtendedShape public double Side => 2; } + public class DisposableExtendedModel : ExtendedModel, IDisposable + { + public void Dispose() + { + } + } + public static class ExtendedStatusExtensions { public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open; @@ -45,8 +54,21 @@ public static class ExtendedModelExtensions public static double Area(this IExtendedShape shape) => shape.Side * shape.Side; + public static int CountModels(this List models) => models.Count; + + public static string Names(this IEnumerable models) => string.Join(",", models.Select(model => model.Name)); + + // targets made only of framework types are not bound, they would bloat every class sharing them public static int Total(this List values) => values.Sum(); + public static double SumAll(this double[] values) => values.Sum(); + + public static string Tag(this object value) => "tag"; + + public static int CountItems(this IEnumerable values) => values.Cast().Count(); + + public static bool Release(this IDisposable disposable) => true; + public static T Identity(this T value) => value; public static string Twice(this string value) => value + value; @@ -60,7 +82,9 @@ public class TestExtensionMethods from clr import AddReference AddReference(""System"") AddReference(""Python.EmbeddingTest"") +from System import Array from System.Collections.Generic import List +from System.Text import StringBuilder from Python.EmbeddingTest import * def enum_instance(): @@ -85,10 +109,22 @@ def member_wins(): return ExtendedModel().existing() def closed_generic(): + models = List[ExtendedModel]() + models.Add(ExtendedModel()) + models.Add(ExtendedModel()) + return models.count_models(), models.names() + +def framework_targets(): values = List[int]() values.Add(1) values.Add(2) - return values.total() + array = Array[float]([1.5, 2.5]) + bound = (hasattr(values, 'total'), hasattr(array, 'sum_all'), hasattr(ExtendedModel(), 'tag'), + hasattr(StringBuilder(), 'tag'), hasattr(values, 'count_items'), hasattr(DisposableExtendedModel(), 'release')) + static = (ExtendedModelExtensions.total(values), ExtendedModelExtensions.sum_all(array), + ExtendedModelExtensions.tag(ExtendedModel()), ExtendedModelExtensions.count_items(values), + ExtendedModelExtensions.release(DisposableExtendedModel())) + return bound, static def open_generic_not_bound(): return hasattr(ExtendedModel(), 'identity') @@ -168,10 +204,31 @@ public void RealMemberWinsOverExtension() } [Test] - public void ClosedGenericTargetIsBound() + public void ClosedGenericTargetOverAUserTypeIsBound() + { + using var _ = Py.GIL(); + var result = module.closed_generic(); + Assert.AreEqual(2, (int)result[0]); + Assert.AreEqual("model,model", (string)result[1]); + } + + [Test] + public void FrameworkOnlyTargetsKeepTheStaticFormOnly() { using var _ = Py.GIL(); - Assert.AreEqual(3, (int)module.closed_generic()); + var result = module.framework_targets(); + var bound = result[0]; + for (var i = 0; i < 6; i++) + { + Assert.IsFalse((bool)bound[i], $"index {i}"); + } + + var staticForms = result[1]; + Assert.AreEqual(3, (int)staticForms[0]); + Assert.AreEqual(4.0, (double)staticForms[1]); + Assert.AreEqual("tag", (string)staticForms[2]); + Assert.AreEqual(2, (int)staticForms[3]); + Assert.IsTrue((bool)staticForms[4]); } [Test] diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs index 2458057c3..4782161b6 100644 --- a/src/runtime/ExtensionMethods.cs +++ b/src/runtime/ExtensionMethods.cs @@ -14,6 +14,8 @@ namespace Python.Runtime internal static class ExtensionMethods { private static readonly ConcurrentDictionary> _methodsByExtendedType = new(); + private static readonly ConcurrentDictionary _isFrameworkAssembly = new(); + private static volatile bool _hasInterfaceTargets; // The framework ships thousands of extension methods (LINQ above all); binding them onto // every collection would bloat each class, so only user assemblies are indexed. @@ -25,6 +27,8 @@ internal static class ExtensionMethods internal static void Clear() { _methodsByExtendedType.Clear(); + _isFrameworkAssembly.Clear(); + _hasInterfaceTargets = false; } internal static bool IsExtensionMethod(MethodBase method) @@ -37,7 +41,8 @@ internal static bool IsExtensionMethod(MethodBase method) /// internal static void Register(Assembly assembly, IEnumerable exportedTypes) { - if (IsFrameworkAssembly(assembly)) + // The compiler marks the assemblies declaring extension methods, C# itself skips the unmarked ones + if (IsFrameworkAssembly(assembly) || !assembly.IsDefined(typeof(ExtensionAttribute), false)) { return; } @@ -58,12 +63,16 @@ internal static void Register(Assembly assembly, IEnumerable exportedTypes } var extendedType = method.GetParameters()[0].ParameterType; - // The binder cannot infer open generic targets (this T, this IEnumerable) from the instance - if (extendedType.ContainsGenericParameters || extendedType.IsByRef) + if (!IsBindableTarget(extendedType)) { continue; } + if (extendedType.IsInterface) + { + _hasInterfaceTargets = true; + } + var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List()); lock (methods) { @@ -77,38 +86,74 @@ internal static void Register(Assembly assembly, IEnumerable exportedTypes /// Gets the extension methods to bind to the given type: those extending the type itself and the /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy. /// - internal static List GetMethodsToBind(Type type) + internal static IReadOnlyList GetMethodsToBind(Type type) { - var result = new List(); - AddMethods(type, result); + List? result = null; + AddMethods(type, ref result); - if (!type.IsInterface) + if (_hasInterfaceTargets && !type.IsInterface) { - var baseInterfaces = type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; - foreach (var @interface in type.GetInterfaces().Except(baseInterfaces)) + Type[]? baseInterfaces = null; + foreach (var @interface in type.GetInterfaces()) { - AddMethods(@interface, result); + if (!_methodsByExtendedType.ContainsKey(@interface)) + { + continue; + } + + baseInterfaces ??= type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; + if (Array.IndexOf(baseInterfaces, @interface) < 0) + { + AddMethods(@interface, ref result); + } } } - return result; + return result ?? (IReadOnlyList)Array.Empty(); } - private static void AddMethods(Type extendedType, List result) + private static void AddMethods(Type extendedType, ref List? result) { if (_methodsByExtendedType.TryGetValue(extendedType, out var methods)) { lock (methods) { - result.AddRange(methods); + (result ??= new List()).AddRange(methods); } } } + /// + /// Whether the extension methods of the given type should be bound to its instances. The binder cannot + /// infer open generic targets (this T, this IEnumerable<T>) from the instance. Targets made only of + /// framework types (object, IEnumerable, double[], List<int>) are shared by countless classes, which + /// would all be bloated with them (Accord.Math alone extends each primitive array with ~600 methods), + /// so those keep their static form only. + /// + private static bool IsBindableTarget(Type type) + { + if (type.IsByRef || type.ContainsGenericParameters) + { + return false; + } + if (type.HasElementType) + { + return IsBindableTarget(type.GetElementType()!); + } + if (!IsFrameworkAssembly(type.Assembly)) + { + return true; + } + return type.IsGenericType && type.GetGenericArguments().Any(IsBindableTarget); + } + private static bool IsFrameworkAssembly(Assembly assembly) { - var name = assembly.GetName().Name ?? string.Empty; - return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + return _isFrameworkAssembly.GetOrAdd(assembly, a => + { + var name = a.GetName().Name ?? string.Empty; + return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + }); } } } From 62447d58803dcdfce449caca79f9326b0e320b1f Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 16 Oct 2026 10:05:31 -0300 Subject: [PATCH 6/7] Make implicit numeric operators follow Python's int and float semantics Everything went through double, so integral conversions lost precision past 2**53 and returned floats where Python gives ints. Values now behave like the Python number their conversion maps to: exact ints for integral conversions (unbounded, like Python), floats for floating point and decimal ones. - operands arrive as PyObject: ints beyond Int32 failed the binder's object conversion, and numbers implementing __index__ or __float__ (numpy scalars, other CLR types with a conversion) are read too; anything else returns NotImplemented so Python raises its usual TypeError - division by zero raises ZeroDivisionError instead of returning infinity - int() truncates decimals exactly and raises ValueError/OverflowError on NaN/infinity - comparisons between instances of an IComparable type use its own ordering, as they did before the number protocol was added - inherited operators keep winning over the synthesized ones, as in C#; OperatorMethod shares the filtered list, or the inherited slot recursed Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- .../TestImplicitNumericOperators.cs | 205 ++++++++++++ src/runtime/ClassManager.cs | 4 +- src/runtime/Util/ImplicitNumericOps.cs | 297 ++++++++++++++++-- 4 files changed, 476 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e444838c4..c191805f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added - Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`) -- Types with an implicit conversion to a numeric type support Python arithmetic, comparisons, `float()` and `int()` through that conversion, unless they declare the operator themselves +- Types with an implicit conversion to a numeric type support Python arithmetic, comparisons, `float()` and `int()` through that conversion, behaving like a Python `int` for integral conversions and a `float` otherwise, unless they declare or inherit the operator - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestImplicitNumericOperators.cs b/src/embed_tests/TestImplicitNumericOperators.cs index 3b580adaf..cd9de7470 100644 --- a/src/embed_tests/TestImplicitNumericOperators.cs +++ b/src/embed_tests/TestImplicitNumericOperators.cs @@ -1,3 +1,4 @@ +using System; using NUnit.Framework; using Python.Runtime; @@ -42,6 +43,53 @@ public ConvertibleMoney(decimal amount) public static ConvertibleMoney operator +(ConvertibleMoney a, ConvertibleMoney b) => new(a.Amount + b.Amount); } + public class ConvertibleCount + { + public long Value { get; } + + public ConvertibleCount(long value) + { + Value = value; + } + + public static implicit operator long(ConvertibleCount count) => count.Value; + } + + public class ConvertibleBase + { + // inherited by ConvertibleDerived, it must keep winning over the synthesized one as in C# + public static string operator +(ConvertibleBase a, ConvertibleBase b) => "base"; + } + + public class ConvertibleDerived : ConvertibleBase + { + public decimal Amount { get; } + + public ConvertibleDerived(decimal amount) + { + Amount = amount; + } + + public static implicit operator decimal(ConvertibleDerived derived) => derived.Amount; + } + + public class PreciseQuantity : IComparable + { + public decimal Value { get; } + + public PreciseQuantity(decimal value) + { + Value = value; + } + + // equal to 1 once converted to double, only the type's own ordering tells them apart + public static PreciseQuantity AboveOne() => new(1.0000000000000000000000000001m); + + public int CompareTo(PreciseQuantity other) => Value.CompareTo(other.Value); + + public static implicit operator decimal(PreciseQuantity quantity) => quantity.Value; + } + public class NotConvertible { } @@ -79,6 +127,62 @@ def own_operator_wins(): def not_convertible(): return hasattr(NotConvertible(), '__add__'), hasattr(NotConvertible(), '__float__') + +count = ConvertibleCount(2) +big = ConvertibleCount(2**53 + 1) + +def integer_arithmetic(): + results = (count + 1, 1 + count, count + count, count - 5, 10 - count, count * 3, 3 * count, -count, count + True) + return all(type(result) is int for result in results), results + +def integer_with_floats(): + results = (count + 0.5, 0.5 + count, count * 1.5, count / 2, count / count, 3 / count) + return all(type(result) is float for result in results), results + +def large_integers(): + exact = 2**53 + 1 + return (int(big) == exact, big > 2**53, big - 1 == 2**53, big * big == exact * exact, + -big == -exact, big >= exact, big + 2**70 == exact + 2**70) + +def int_conversions(): + errors = [] + for value in (float('nan'), float('inf')): + try: + int(ConvertibleRatio(value)) + except Exception as e: + errors.append(type(e).__name__) + return errors, int(ConvertibleRatio(-1.75)), int(ConvertibleFee(12345678901234567891)) == 12345678901234567891 + +def division_by_zero(): + errors = [] + for operation in (lambda: fee / 0, lambda: 1 / ConvertibleFee(0), lambda: count / 0.0, lambda: fee / ConvertibleFee(0)): + try: + operation() + except ZeroDivisionError: + errors.append(True) + return len(errors) + +def other_numeric_operands(): + margin = ConvertibleRatio(0.5) + return fee + 2**40, count + 2**70, fee + margin, count * margin, count < 2**64 + +def unsupported_operands(): + messages = [] + for operation in (lambda: count + 'x', lambda: fee * None, lambda: fee < 'x', lambda: NotConvertible() - fee): + try: + operation() + messages.append(None) + except TypeError as e: + messages.append(str(e)) + return messages + +def inherited_operator_wins(): + derived = ConvertibleDerived(2) + return derived + derived, derived - ConvertibleDerived(1), float(derived) + +def own_ordering_wins(): + one, above = PreciseQuantity(1), PreciseQuantity.AboveOne() + return above > one, one < above, above >= one, one <= above, above < one, sorted([above, one])[0] is one "; [OneTimeSetUp] @@ -150,6 +254,107 @@ public void TheTypesOwnOperatorsKeepWinning() Assert.AreEqual(1.0, (double)result[2]); } + [Test] + public void IntegralConversionsBehaveLikeInts() + { + using var _ = Py.GIL(); + var result = module.integer_arithmetic(); + Assert.IsTrue((bool)result[0], "every result must be an int"); + var expected = new long[] { 3, 3, 4, -3, 8, 6, 6, -2, 3 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (long)result[1][i], $"index {i}"); + } + } + + [Test] + public void IntegralConversionsGiveFloatsWherePythonWould() + { + using var _ = Py.GIL(); + var result = module.integer_with_floats(); + Assert.IsTrue((bool)result[0], "every result must be a float"); + var expected = new[] { 2.5, 2.5, 3.0, 1.0, 1.0, 1.5 }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (double)result[1][i], $"index {i}"); + } + } + + [Test] + public void LargeIntegralValuesStayExact() + { + using var _ = Py.GIL(); + var result = module.large_integers(); + for (var i = 0; i < 7; i++) + { + Assert.IsTrue((bool)result[i], $"index {i}"); + } + } + + [Test] + public void IntConversionFollowsPython() + { + using var _ = Py.GIL(); + var result = module.int_conversions(); + CollectionAssert.AreEqual(new[] { "ValueError", "OverflowError" }, ((PyObject)result[0]).As()); + Assert.AreEqual(-1, (int)result[1]); + Assert.IsTrue((bool)result[2], "decimal conversions must truncate exactly"); + } + + [Test] + public void DivisionByZeroRaisesLikePython() + { + using var _ = Py.GIL(); + Assert.AreEqual(4, (int)module.division_by_zero()); + } + + [Test] + public void LargeIntsAndOtherNumbersAreOperands() + { + using var _ = Py.GIL(); + var result = module.other_numeric_operands(); + Assert.AreEqual(2.0 + Math.Pow(2, 40), (double)result[0]); + Assert.AreEqual("1180591620717411303426", result[1].ToString()); + // another type with an implicit numeric conversion is read through its __float__ + Assert.AreEqual(2.5, (double)result[2]); + Assert.AreEqual(1.0, (double)result[3]); + Assert.IsTrue((bool)result[4]); + } + + [Test] + public void UnsupportedOperandsRaisePythonsTypeError() + { + using var _ = Py.GIL(); + var messages = ((PyObject)module.unsupported_operands()).As(); + StringAssert.Contains("unsupported operand type(s) for +", messages[0]); + StringAssert.Contains("unsupported operand type(s) for *", messages[1]); + StringAssert.Contains("'<' not supported", messages[2]); + StringAssert.Contains("unsupported operand type(s) for -", messages[3]); + } + + [Test] + public void InheritedOperatorsKeepWinning() + { + using var _ = Py.GIL(); + var result = module.inherited_operator_wins(); + Assert.AreEqual("base", (string)result[0]); + // subtraction is declared nowhere in the hierarchy, so the synthesized one applies + Assert.AreEqual(1.0, (double)result[1]); + Assert.AreEqual(2.0, (double)result[2]); + } + + [Test] + public void TheTypesOwnOrderingWinsBetweenItsInstances() + { + using var _ = Py.GIL(); + var result = module.own_ordering_wins(); + var expected = new[] { true, true, true, true, false, true }; + for (var i = 0; i < expected.Length; i++) + { + Assert.AreEqual(expected[i], (bool)result[i], $"index {i}"); + } + } + [Test] public void TypesWithoutAConversionAreUntouched() { diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index 6cc8886af..b4aefacc2 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -417,8 +417,8 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) } else if (ImplicitNumericOps.IsSupported(type)) { - // Types with an implicit numeric conversion get Python's number protocol, keeping the operators they declare themselves - var ops = ImplicitNumericOps.GetMethods(type).Where(op => !local.Contains(op.Name)).ToArray(); + // Types with an implicit numeric conversion get Python's number protocol, keeping the operators they declare or inherit + var ops = ImplicitNumericOps.GetMethods(type); foreach (var op in ops) { local.Add(op.Name); diff --git a/src/runtime/Util/ImplicitNumericOps.cs b/src/runtime/Util/ImplicitNumericOps.cs index 8f4146a7a..d53259f14 100644 --- a/src/runtime/Util/ImplicitNumericOps.cs +++ b/src/runtime/Util/ImplicitNumericOps.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Linq.Expressions; +using System.Numerics; using System.Reflection; namespace Python.Runtime @@ -33,63 +36,299 @@ internal static class ImplicitNumericOps internal static bool IsSupported(Type type) => GetConversion(type) != null; + /// + /// Gets the operators to graft onto the type. The ones it declares or inherits keep winning, as in C# + /// internal static MethodInfo[] GetMethods(Type type) { - return typeof(ImplicitNumericOps<>).MakeGenericType(type).GetMethods(OpsHelper.BindingFlags); + var existing = new HashSet(type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy).Select(method => method.Name)); + return typeof(ImplicitNumericOps<>).MakeGenericType(type).GetMethods(OpsHelper.BindingFlags) + .Where(method => !existing.Contains(method.Name)) + .ToArray(); } } + /// + /// The number protocol of a type with an implicit numeric conversion, behaving like the Python number the + /// conversion maps to: an int for integral conversions (exact, results are ints unless Python would give a + /// float) and a float for floating point and decimal ones. Operands that are not numbers return + /// NotImplemented, so Python can try the other operand and raise its usual TypeError. + /// [Ops] internal static class ImplicitNumericOps { - private static readonly Func ToDouble = BuildConversion(); + private static readonly MethodInfo Conversion = ImplicitNumericOps.GetConversion(typeof(T)) + ?? throw new InvalidOperationException($"{typeof(T)} has no implicit numeric conversion"); + private static readonly bool IntegralConversion = Type.GetTypeCode(Conversion.ReturnType) is not (TypeCode.Double or TypeCode.Single or TypeCode.Decimal); + private static readonly Func ToDouble = BuildConversion(); + // double to BigInteger throws on NaN and infinity, so floating point conversions go through ToDouble + private static readonly Func? ToInteger = IntegralConversion || Conversion.ReturnType == typeof(decimal) + ? BuildConversion() + : null; + private static readonly bool Comparable = typeof(IComparable).IsAssignableFrom(typeof(T)) || typeof(IComparable).IsAssignableFrom(typeof(T)); #pragma warning disable IDE1006 // Naming Styles - must match Python [ForbidPythonThreads] public static double __float__(T value) => ToDouble(value); [ForbidPythonThreads] - public static long __int__(T value) => (long)Math.Truncate(ToDouble(value)); + public static object __int__(T value) + { + using (Py.GIL()) + { + if (ToInteger != null) + { + return ToPython(ToInteger(value)); + } + + var number = ToDouble(value); + if (double.IsNaN(number)) + { + throw RaisePython(Exceptions.ValueError, "cannot convert float NaN to integer"); + } + if (double.IsInfinity(number)) + { + throw RaisePython(Exceptions.OverflowError, "cannot convert float infinity to integer"); + } + return ToPython(new BigInteger(number)); + } + } #pragma warning restore IDE1006 - public static double op_UnaryNegation(T value) => -ToDouble(value); + public static object op_UnaryNegation(T value) + { + using (Py.GIL()) + { + var number = ToNumber(value); + return number.IsInteger ? ToPython(-number.Integer) : -number.Float; + } + } - public static double op_Addition(T a, T b) => ToDouble(a) + ToDouble(b); - public static double op_Addition(T a, double b) => ToDouble(a) + b; - public static double op_Addition(double a, T b) => a + ToDouble(b); + public static object op_Addition(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Add, (x, y) => x + y); + public static object op_Addition(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Add, (x, y) => x + y); - public static double op_Subtraction(T a, T b) => ToDouble(a) - ToDouble(b); - public static double op_Subtraction(T a, double b) => ToDouble(a) - b; - public static double op_Subtraction(double a, T b) => a - ToDouble(b); + public static object op_Subtraction(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Subtract, (x, y) => x - y); + public static object op_Subtraction(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Subtract, (x, y) => x - y); - public static double op_Multiply(T a, T b) => ToDouble(a) * ToDouble(b); - public static double op_Multiply(T a, double b) => ToDouble(a) * b; - public static double op_Multiply(double a, T b) => a * ToDouble(b); + public static object op_Multiply(T a, PyObject b) => Arithmetic(a, b, reverse: false, BigInteger.Multiply, (x, y) => x * y); + public static object op_Multiply(PyObject a, T b) => Arithmetic(b, a, reverse: true, BigInteger.Multiply, (x, y) => x * y); - public static double op_Division(T a, T b) => ToDouble(a) / ToDouble(b); - public static double op_Division(T a, double b) => ToDouble(a) / b; - public static double op_Division(double a, T b) => a / ToDouble(b); + // true division: a float even between ints, and like Python it raises ZeroDivisionError instead of returning infinity + public static object op_Division(T a, PyObject b) => Arithmetic(a, b, reverse: false, null, Divide); + public static object op_Division(PyObject a, T b) => Arithmetic(b, a, reverse: true, null, Divide); // Python reflects a reversed comparison onto the other operand, so only the forward overloads are needed - public static bool op_LessThan(T a, T b) => ToDouble(a) < ToDouble(b); - public static bool op_LessThan(T a, double b) => ToDouble(a) < b; + public static object op_LessThan(T a, PyObject b) => Compare(a, b, Runtime.Py_LT); + public static object op_GreaterThan(T a, PyObject b) => Compare(a, b, Runtime.Py_GT); + public static object op_LessThanOrEqual(T a, PyObject b) => Compare(a, b, Runtime.Py_LE); + public static object op_GreaterThanOrEqual(T a, PyObject b) => Compare(a, b, Runtime.Py_GE); + + private static object Arithmetic(T self, PyObject? other, bool reverse, + Func? integerOperation, Func floatOperation) + { + // the binder released the GIL to call us + using (Py.GIL()) + { + if (!TryGetOperand(other, out var right)) + { + return Runtime.PyNotImplemented; + } + + var left = ToNumber(self); + if (reverse) + { + (left, right) = (right, left); + } + + if (integerOperation != null && left.IsInteger && right.IsInteger) + { + return ToPython(integerOperation(left.Integer, right.Integer)); + } + return floatOperation(left.AsDouble, right.AsDouble); + } + } + + private static double Divide(double dividend, double divisor) + { + if (divisor == 0) + { + throw RaisePython(Exceptions.ZeroDivisionError, "division by zero"); + } + return dividend / divisor; + } + + private static object Compare(T self, PyObject? other, int op) + { + int order; + using (Py.GIL()) + { + // the type's own ordering wins between its instances, as it did before the number protocol was added + if (Comparable && TryGetInstance(other, out var same)) + { + order = Comparer.Default.Compare(self, same); + } + else + { + if (!TryGetOperand(other, out var right)) + { + return Runtime.PyNotImplemented; + } - public static bool op_GreaterThan(T a, T b) => ToDouble(a) > ToDouble(b); - public static bool op_GreaterThan(T a, double b) => ToDouble(a) > b; + var left = ToNumber(self); + if (!left.IsInteger || !right.IsInteger) + { + // IEEE comparisons, false against NaN as in Python + double x = left.AsDouble, y = right.AsDouble; + return op switch + { + Runtime.Py_LT => x < y, + Runtime.Py_LE => x <= y, + Runtime.Py_GT => x > y, + _ => x >= y, + }; + } + order = left.Integer.CompareTo(right.Integer); + } + } - public static bool op_LessThanOrEqual(T a, T b) => ToDouble(a) <= ToDouble(b); - public static bool op_LessThanOrEqual(T a, double b) => ToDouble(a) <= b; + return op switch + { + Runtime.Py_LT => order < 0, + Runtime.Py_LE => order <= 0, + Runtime.Py_GT => order > 0, + _ => order >= 0, + }; + } - public static bool op_GreaterThanOrEqual(T a, T b) => ToDouble(a) >= ToDouble(b); - public static bool op_GreaterThanOrEqual(T a, double b) => ToDouble(a) >= b; + private static Number ToNumber(T value) => IntegralConversion ? new Number(ToInteger!(value)) : new Number(ToDouble(value)); - private static Func BuildConversion() + private static bool TryGetInstance(PyObject? value, out T instance) + { + if (value != null && ManagedType.GetManagedObject(value.Reference) is CLRObject { inst: T clrInstance }) + { + instance = clrInstance; + return true; + } + instance = default!; + return false; + } + + /// + /// Reads a Python operand as a number: instances of , ints (bool included) and + /// floats, and any other object implementing __index__ or __float__ (numpy scalars, other CLR types with + /// an implicit numeric conversion). Requires the GIL. + /// + private static bool TryGetOperand(PyObject? value, out Number number) + { + number = default; + if (value == null) + { + return false; + } + if (TryGetInstance(value, out var instance)) + { + number = ToNumber(instance); + return true; + } + + var reference = value.Reference; + if (Runtime.PyInt_Check(reference)) + { + number = new Number(ToBigInteger(reference)); + return true; + } + if (Runtime.PyObject_TypeCheck(reference, Runtime.PyFloatType)) + { + number = new Number(Runtime.PyFloat_AsDouble(reference)); + return true; + } + if (Runtime.PyObject_HasAttrString(reference, "__index__") != 0) + { + using var integer = Runtime.PyNumber_Long(reference); + if (!integer.IsNull()) + { + number = new Number(ToBigInteger(integer.Borrow())); + return true; + } + } + else if (Runtime.PyObject_HasAttrString(reference, "__float__") != 0) + { + var real = Runtime.PyFloat_AsDouble(reference); + if (real != -1 || !Exceptions.ErrorOccurred()) + { + number = new Number(real); + return true; + } + } + + // a failed probe must not leak, the operand is just not a number + Exceptions.Clear(); + return false; + } + + private static BigInteger ToBigInteger(BorrowedReference integer) + { + var value = Runtime.PyLong_AsLongLong(integer); + if (value.HasValue) + { + return value.Value; + } + + // beyond long, parse its digits + Exceptions.Clear(); + using var digits = Runtime.PyObject_Str(integer); + return BigInteger.Parse(Runtime.GetManagedString(digits.Borrow())!, CultureInfo.InvariantCulture); + } + + /// + /// Converts an integer result to a Python int, which is unbounded. Requires the GIL beyond long. + /// + private static object ToPython(BigInteger value) + { + if (value >= long.MinValue && value <= long.MaxValue) + { + return (long)value; + } + return new PyInt(value.ToString(CultureInfo.InvariantCulture)); + } + + /// + /// Raises a Python exception from a call made by the binder, which restores it to the caller. Requires the GIL. + /// + private static Exception RaisePython(PyObject type, string message) + { + Exceptions.SetError(type.Reference, message); + return PythonException.ThrowLastAsClrException(); + } + + private static Func BuildConversion() { - var conversion = ImplicitNumericOps.GetConversion(typeof(T)) - ?? throw new InvalidOperationException($"{typeof(T)} has no implicit numeric conversion"); var value = Expression.Parameter(typeof(T), "value"); - var body = Expression.Convert(Expression.Call(conversion, value), typeof(double)); - return Expression.Lambda>(body, value).Compile(); + var body = Expression.Convert(Expression.Call(Conversion, value), typeof(TResult)); + return Expression.Lambda>(body, value).Compile(); + } + + private readonly struct Number + { + public readonly bool IsInteger; + public readonly BigInteger Integer; + public readonly double Float; + + public Number(BigInteger value) + { + IsInteger = true; + Integer = value; + Float = 0; + } + + public Number(double value) + { + IsInteger = false; + Integer = default; + Float = value; + } + + public double AsDouble => IsInteger ? (double)Integer : Float; } } } From ab9cc61a5283e4b76b866746b63ccc9bad2128ad Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 16 Oct 2026 11:51:46 -0300 Subject: [PATCH 7/7] Keep extension methods marked EditorBrowsable(Never) in their static form [EditorBrowsable(EditorBrowsableState.Never)] hides a member from IntelliSense while keeping it callable. Binding such extension methods to the instance would advertise them anyway, so those methods, and all the extension methods of a class marked so, keep their static form only. It lets a library hide plumbing helpers, like Lean's IAlgorithm extensions, from the instance with a standard attribute. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- src/embed_tests/TestExtensionMethods.cs | 32 +++++++++++++++++++++++++ src/runtime/ExtensionMethods.cs | 15 ++++++++++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c191805f8..416b47c53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added -- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`) +- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`). Those marked `[EditorBrowsable(EditorBrowsableState.Never)]`, or declared in a class marked so, keep their static form only - Types with an implicit conversion to a numeric type support Python arithmetic, comparisons, `float()` and `int()` through that conversion, behaving like a Python `int` for integral conversions and a `float` otherwise, unless they declare or inherit the operator - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs index 74906e4d2..d9257ca11 100644 --- a/src/embed_tests/TestExtensionMethods.cs +++ b/src/embed_tests/TestExtensionMethods.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; using NUnit.Framework; using Python.Runtime; @@ -72,6 +73,19 @@ public static class ExtendedModelExtensions public static T Identity(this T value) => value; public static string Twice(this string value) => value + value; + + // hidden from IntelliSense, so not advertised on the instance either + [EditorBrowsable(EditorBrowsableState.Never)] + public static string Hidden(this ExtendedModel model) => "hidden"; + + [EditorBrowsable(EditorBrowsableState.Advanced)] + public static string Advanced(this ExtendedModel model) => "advanced"; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public static class HiddenExtendedModelExtensions + { + public static string HiddenByClass(this ExtendedModel model) => "hidden by class"; } public class TestExtensionMethods @@ -129,6 +143,12 @@ def framework_targets(): def open_generic_not_bound(): return hasattr(ExtendedModel(), 'identity') +def editor_browsable(): + model = ExtendedModel() + bound = hasattr(model, 'hidden'), hasattr(model, 'hidden_by_class'), model.advanced() + static = ExtendedModelExtensions.hidden(model), HiddenExtendedModelExtensions.hidden_by_class(model) + return bound, static + def python_native_not_bound(): return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab') "; @@ -238,6 +258,18 @@ public void OpenGenericTargetIsNotBound() Assert.IsFalse((bool)module.open_generic_not_bound()); } + [Test] + public void EditorBrowsableNeverKeepsTheStaticFormOnly() + { + using var _ = Py.GIL(); + var result = module.editor_browsable(); + Assert.IsFalse((bool)result[0][0]); + Assert.IsFalse((bool)result[0][1]); + Assert.AreEqual("advanced", (string)result[0][2]); + Assert.AreEqual("hidden", (string)result[1][0]); + Assert.AreEqual("hidden by class", (string)result[1][1]); + } + [Test] public void PythonNativeTargetKeepsTheStaticForm() { diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs index 4782161b6..dea19f5a0 100644 --- a/src/runtime/ExtensionMethods.cs +++ b/src/runtime/ExtensionMethods.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; @@ -50,7 +51,7 @@ internal static void Register(Assembly assembly, IEnumerable exportedTypes foreach (var type in exportedTypes) { // extension methods can only live in non-generic static classes - if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false)) + if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false) || IsHidden(type)) { continue; } @@ -63,7 +64,7 @@ internal static void Register(Assembly assembly, IEnumerable exportedTypes } var extendedType = method.GetParameters()[0].ParameterType; - if (!IsBindableTarget(extendedType)) + if (!IsBindableTarget(extendedType) || IsHidden(method)) { continue; } @@ -147,6 +148,16 @@ private static bool IsBindableTarget(Type type) return type.IsGenericType && type.GetGenericArguments().Any(IsBindableTarget); } + /// + /// [EditorBrowsable(Never)] hides a member from IntelliSense while keeping it callable, so it is not advertised + /// on the instances either: hidden extension methods, or all those of a hidden class, keep their static form only + /// + private static bool IsHidden(MemberInfo member) + { + return member.IsDefined(typeof(EditorBrowsableAttribute), false) + && member.GetCustomAttribute(false)!.State == EditorBrowsableState.Never; + } + private static bool IsFrameworkAssembly(Assembly assembly) { return _isFrameworkAssembly.GetOrAdd(assembly, a =>