diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d72a3a0..ee61e8f72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`). Those marked `[EditorBrowsable(EditorBrowsableState.Never)]`, or declared in a class marked so, keep their static form only - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs new file mode 100644 index 000000000..d9257ca11 --- /dev/null +++ b/src/embed_tests/TestExtensionMethods.cs @@ -0,0 +1,282 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public enum ExtendedStatus { Open, Closed } + + public interface IExtendedShape + { + double Side { get; } + } + + public class ExtendedModel + { + public string Name { get; set; } = "model"; + + public string Existing() => "member"; + } + + public class DerivedExtendedModel : ExtendedModel + { + } + + public class ExtendedSquare : IExtendedShape + { + public double Side => 2; + } + + public class DisposableExtendedModel : ExtendedModel, IDisposable + { + public void Dispose() + { + } + } + + public static class ExtendedStatusExtensions + { + public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open; + + public static string Describe(this ExtendedStatus status, string prefix, int times = 1) + => string.Concat(Enumerable.Repeat(prefix + status, times)); + } + + public static class ExtendedModelExtensions + { + public static string Describe(this ExtendedModel model, string prefix = "") => prefix + model.Name; + + // collides with the real instance method, which must win + public static string Existing(this ExtendedModel model) => "extension"; + + public static double Area(this IExtendedShape shape) => shape.Side * shape.Side; + + public static int CountModels(this List models) => models.Count; + + public static string Names(this IEnumerable models) => string.Join(",", models.Select(model => model.Name)); + + // targets made only of framework types are not bound, they would bloat every class sharing them + public static int Total(this List values) => values.Sum(); + + public static double SumAll(this double[] values) => values.Sum(); + + public static string Tag(this object value) => "tag"; + + public static int CountItems(this IEnumerable values) => values.Cast().Count(); + + public static bool Release(this IDisposable disposable) => true; + + public static T Identity(this T value) => value; + + public static string Twice(this string value) => value + value; + + // hidden from IntelliSense, so not advertised on the instance either + [EditorBrowsable(EditorBrowsableState.Never)] + public static string Hidden(this ExtendedModel model) => "hidden"; + + [EditorBrowsable(EditorBrowsableState.Advanced)] + public static string Advanced(this ExtendedModel model) => "advanced"; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public static class HiddenExtendedModelExtensions + { + public static string HiddenByClass(this ExtendedModel model) => "hidden by class"; + } + + public class TestExtensionMethods + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from System import Array +from System.Collections.Generic import List +from System.Text import StringBuilder +from Python.EmbeddingTest import * + +def enum_instance(): + return ExtendedStatus.Open.IsOpen(), ExtendedStatus.Open.is_open(), ExtendedStatus.Closed.is_open() + +def enum_with_args(): + return ExtendedStatus.Open.describe('s-', times=2) + +def enum_static_forms(): + return ExtendedStatusExtensions.is_open(ExtendedStatus.Open), ExtendedStatus.is_open(ExtendedStatus.Closed) + +def class_instance(): + return ExtendedModel().describe(), ExtendedModel().describe(prefix='p-') + +def derived_inherits(): + return DerivedExtendedModel().describe('d-') + +def interface_target(): + return ExtendedSquare().area() + +def member_wins(): + return ExtendedModel().existing() + +def closed_generic(): + models = List[ExtendedModel]() + models.Add(ExtendedModel()) + models.Add(ExtendedModel()) + return models.count_models(), models.names() + +def framework_targets(): + values = List[int]() + values.Add(1) + values.Add(2) + array = Array[float]([1.5, 2.5]) + bound = (hasattr(values, 'total'), hasattr(array, 'sum_all'), hasattr(ExtendedModel(), 'tag'), + hasattr(StringBuilder(), 'tag'), hasattr(values, 'count_items'), hasattr(DisposableExtendedModel(), 'release')) + static = (ExtendedModelExtensions.total(values), ExtendedModelExtensions.sum_all(array), + ExtendedModelExtensions.tag(ExtendedModel()), ExtendedModelExtensions.count_items(values), + ExtendedModelExtensions.release(DisposableExtendedModel())) + return bound, static + +def open_generic_not_bound(): + return hasattr(ExtendedModel(), 'identity') + +def editor_browsable(): + model = ExtendedModel() + bound = hasattr(model, 'hidden'), hasattr(model, 'hidden_by_class'), model.advanced() + static = ExtendedModelExtensions.hidden(model), HiddenExtendedModelExtensions.hidden_by_class(model) + return bound, static + +def python_native_not_bound(): + return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab') +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("extension_methods", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void EnumExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.enum_instance(); + Assert.IsTrue((bool)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsFalse((bool)result[2]); + } + + [Test] + public void ExtensionArgumentsFollowTheInstance() + { + using var _ = Py.GIL(); + Assert.AreEqual("s-Opens-Open", (string)module.enum_with_args()); + } + + [Test] + public void StaticFormsKeepWorking() + { + using var _ = Py.GIL(); + var result = module.enum_static_forms(); + Assert.IsTrue((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + + [Test] + public void ClassExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.class_instance(); + Assert.AreEqual("model", (string)result[0]); + Assert.AreEqual("p-model", (string)result[1]); + } + + [Test] + public void DerivedClassInheritsBaseClassExtensions() + { + using var _ = Py.GIL(); + Assert.AreEqual("d-model", (string)module.derived_inherits()); + } + + [Test] + public void InterfaceExtensionIsBoundToImplementingClass() + { + using var _ = Py.GIL(); + Assert.AreEqual(4.0, (double)module.interface_target()); + } + + [Test] + public void RealMemberWinsOverExtension() + { + using var _ = Py.GIL(); + Assert.AreEqual("member", (string)module.member_wins()); + } + + [Test] + public void ClosedGenericTargetOverAUserTypeIsBound() + { + using var _ = Py.GIL(); + var result = module.closed_generic(); + Assert.AreEqual(2, (int)result[0]); + Assert.AreEqual("model,model", (string)result[1]); + } + + [Test] + public void FrameworkOnlyTargetsKeepTheStaticFormOnly() + { + using var _ = Py.GIL(); + var result = module.framework_targets(); + var bound = result[0]; + for (var i = 0; i < 6; i++) + { + Assert.IsFalse((bool)bound[i], $"index {i}"); + } + + var staticForms = result[1]; + Assert.AreEqual(3, (int)staticForms[0]); + Assert.AreEqual(4.0, (double)staticForms[1]); + Assert.AreEqual("tag", (string)staticForms[2]); + Assert.AreEqual(2, (int)staticForms[3]); + Assert.IsTrue((bool)staticForms[4]); + } + + [Test] + public void OpenGenericTargetIsNotBound() + { + using var _ = Py.GIL(); + Assert.IsFalse((bool)module.open_generic_not_bound()); + } + + [Test] + public void EditorBrowsableNeverKeepsTheStaticFormOnly() + { + using var _ = Py.GIL(); + var result = module.editor_browsable(); + Assert.IsFalse((bool)result[0][0]); + Assert.IsFalse((bool)result[0][1]); + Assert.AreEqual("advanced", (string)result[0][2]); + Assert.AreEqual("hidden", (string)result[1][0]); + Assert.AreEqual("hidden by class", (string)result[1][1]); + } + + [Test] + public void PythonNativeTargetKeepsTheStaticForm() + { + using var _ = Py.GIL(); + var result = module.python_native_not_bound(); + Assert.IsFalse((bool)result[0]); + Assert.AreEqual("abab", (string)result[1]); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index dd7061c2b..5efef4906 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index 3370e4410..76b252652 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -62,6 +62,7 @@ internal static void Initialize() // test/usage after the first init cycle. Clear so the scan runs fresh. assembliesNamesCache.Clear(); assemblies.Clear(); + ExtensionMethods.Clear(); AppDomain domain = AppDomain.CurrentDomain; @@ -356,7 +357,8 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + var types = GetTypes(assembly); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -383,6 +385,8 @@ internal static void ScanAssembly(Assembly assembly) GenericUtil.Register(t); } } + + ExtensionMethods.Register(assembly, types); } public static AssemblyName[] ListAssemblies() diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b88a6a6b6..835f0ec82 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -416,6 +416,17 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) } } + // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C# + var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray(); + if (extensionMethods.Length > 0) + { + foreach (var em in extensionMethods) + { + local.Add(em.Name); + } + info = info.Concat(extensionMethods).ToArray(); + } + // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) { diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs new file mode 100644 index 000000000..dea19f5a0 --- /dev/null +++ b/src/runtime/ExtensionMethods.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Python.Runtime +{ + /// + /// Index of the C# extension methods found in the scanned assemblies, keyed by the type they extend. + /// Classes bind them as members so they are callable from an instance, like C# callers do. + /// + internal static class ExtensionMethods + { + private static readonly ConcurrentDictionary> _methodsByExtendedType = new(); + private static readonly ConcurrentDictionary _isFrameworkAssembly = new(); + private static volatile bool _hasInterfaceTargets; + + // The framework ships thousands of extension methods (LINQ above all); binding them onto + // every collection would bloat each class, so only user assemblies are indexed. + private static readonly string[] _frameworkAssemblyPrefixes = { "System", "Microsoft", "mscorlib", "netstandard", "WindowsBase", "Python.Runtime" }; + + /// + /// Drops the index so a re-initialized runtime rebuilds it while re-scanning the assemblies + /// + internal static void Clear() + { + _methodsByExtendedType.Clear(); + _isFrameworkAssembly.Clear(); + _hasInterfaceTargets = false; + } + + internal static bool IsExtensionMethod(MethodBase method) + { + return method.IsStatic && method.IsDefined(typeof(ExtensionAttribute), false); + } + + /// + /// Indexes the extension methods declared by the given exported types of an assembly + /// + internal static void Register(Assembly assembly, IEnumerable exportedTypes) + { + // The compiler marks the assemblies declaring extension methods, C# itself skips the unmarked ones + if (IsFrameworkAssembly(assembly) || !assembly.IsDefined(typeof(ExtensionAttribute), false)) + { + return; + } + + foreach (var type in exportedTypes) + { + // extension methods can only live in non-generic static classes + if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false) || IsHidden(type)) + { + continue; + } + + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (!IsExtensionMethod(method)) + { + continue; + } + + var extendedType = method.GetParameters()[0].ParameterType; + if (!IsBindableTarget(extendedType) || IsHidden(method)) + { + continue; + } + + if (extendedType.IsInterface) + { + _hasInterfaceTargets = true; + } + + var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List()); + lock (methods) + { + methods.Add(method); + } + } + } + } + + /// + /// Gets the extension methods to bind to the given type: those extending the type itself and the + /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy. + /// + internal static IReadOnlyList GetMethodsToBind(Type type) + { + List? result = null; + AddMethods(type, ref result); + + if (_hasInterfaceTargets && !type.IsInterface) + { + Type[]? baseInterfaces = null; + foreach (var @interface in type.GetInterfaces()) + { + if (!_methodsByExtendedType.ContainsKey(@interface)) + { + continue; + } + + baseInterfaces ??= type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; + if (Array.IndexOf(baseInterfaces, @interface) < 0) + { + AddMethods(@interface, ref result); + } + } + } + + return result ?? (IReadOnlyList)Array.Empty(); + } + + private static void AddMethods(Type extendedType, ref List? result) + { + if (_methodsByExtendedType.TryGetValue(extendedType, out var methods)) + { + lock (methods) + { + (result ??= new List()).AddRange(methods); + } + } + } + + /// + /// Whether the extension methods of the given type should be bound to its instances. The binder cannot + /// infer open generic targets (this T, this IEnumerable<T>) from the instance. Targets made only of + /// framework types (object, IEnumerable, double[], List<int>) are shared by countless classes, which + /// would all be bloated with them (Accord.Math alone extends each primitive array with ~600 methods), + /// so those keep their static form only. + /// + private static bool IsBindableTarget(Type type) + { + if (type.IsByRef || type.ContainsGenericParameters) + { + return false; + } + if (type.HasElementType) + { + return IsBindableTarget(type.GetElementType()!); + } + if (!IsFrameworkAssembly(type.Assembly)) + { + return true; + } + return type.IsGenericType && type.GetGenericArguments().Any(IsBindableTarget); + } + + /// + /// [EditorBrowsable(Never)] hides a member from IntelliSense while keeping it callable, so it is not advertised + /// on the instances either: hidden extension methods, or all those of a hidden class, keep their static form only + /// + private static bool IsHidden(MemberInfo member) + { + return member.IsDefined(typeof(EditorBrowsableAttribute), false) + && member.GetCustomAttribute(false)!.State == EditorBrowsableState.Never; + } + + private static bool IsFrameworkAssembly(Assembly assembly) + { + return _isFrameworkAssembly.GetOrAdd(assembly, a => + { + var name = a.GetName().Name ?? string.Empty; + return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + }); + } + } +} diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 6deec74f8..803ee2f23 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -534,6 +534,17 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe pi = pi.Take(1).ToArray(); } + // An extension method called through an instance gets the instance as its first CLR argument + bool isExtension = inst != null && methodInformation.IsExtension; + if (isExtension) + { + pi = pi.Skip(1).ToArray(); + if (hasNamedArgs) + { + paramNames = paramNames.Skip(1).ToArray(); + } + } + // Must be done after IsOperator section int clrArgCount = pi.Length; @@ -775,6 +786,15 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe } } + if (isExtension) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + margs = new[] { co.inst }.Concat(margs).ToArray(); + } + else continue; + } + var match = new MatchedMethod(kwargsMatched, margs, outs, methodInformation, implicitConversions); if (implicitConversions > 0) { @@ -1238,9 +1258,12 @@ internal class MethodInformation { private ParameterInfo[] _parameterInfo; private string[] _parametersNames; + private bool? _isExtension; public MethodBase MethodBase { get; } + public bool IsExtension => _isExtension ??= ExtensionMethods.IsExtensionMethod(MethodBase); + public bool IsOriginal { get; set; } public ParameterInfo[] ParameterInfo diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 0a24781b8..dacadc64d 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.66")] -[assembly: AssemblyFileVersion("2.0.66")] +[assembly: AssemblyVersion("2.0.67")] +[assembly: AssemblyFileVersion("2.0.67")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 765a0dd2c..e290143d9 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.66 + 2.0.67 false LICENSE https://github.com/pythonnet/pythonnet