Preserve annotation, lambda and comprehension source bindings

CHANGELOG.md

@@ -1,5 +1,12 @@
 # **Upcoming release**
 
+- Preserve source bindings in function annotations, lambdas and comprehensions
+  during renaming, and keep call-time lambda values separate from source names.
+- Keep initialized instance lambda values separate from same-named class values
+  when resolving attribute calls and renaming fields.
+- Refuse edits to ordinary positional-only and keyword-only parameter bindings
+  that the current function argument protocol cannot represent safely.
+
 - #895 patchedast Starred and keyword now consumes their syntactically expected and ** (@lieryan)
 - #896 patchedast cleanup and refactoring (@lieryan)
 

docs/library.rst

@@ -875,6 +875,53 @@ It provides two new search methods:
  -  search() - simpler wrapper around search_full with a basic sorting algorithm
 
 
+Source binding environments
+---------------------------
+
+The structural scope of an expression and the environment in which its names
+are resolved can differ. Function annotations use their declaration or
+annotation environment; lambda parameters and comprehension targets have their
+own bindings. The existing ``GlobalScope.get_inner_scope_for_offset(offset)``
+continues to return the structural scope. Its explicit
+``get_structural_scope_for_offset(offset)`` alias has the same behavior.
+
+A module's ``get_source_binding_index()`` provides source identity queries:
+
+.. code-block:: python
+
+  pymodule = project.get_pymodule(resource)
+  index = pymodule.get_source_binding_index()
+  context = index.context_at(offset)
+  environment = context.environment
+  structural_scope = context.structural_scope
+  binding = index.binding_for_token(offset)
+
+``context_for_node(node)`` resolves a node from that module's AST. Source
+queries in ``rope.base.evaluate`` use these environments while keeping source
+identities separate from call-time values. A lambda's private argument matcher
+handles its own positional, keyword and creation-time default values; it does
+not change the ordinary ``Arguments`` or ``PyFunction`` parameter protocols.
+Unknown argument expansion or conflicting values remain unknown.
+
+When a directly assigned class attribute and a directly assigned instance
+attribute in ``__init__`` hold different lambdas, attribute value views select
+the class or initialized instance value independently. This selection does not
+replace their source names or change the ordinary class attribute protocol.
+
+Ordinary positional-only and keyword-only parameters are represented by
+distinct readable source names, so their bodies do not fall back to same-named
+outer variables. Their call inference and signature rewriting are outside this
+model's supported range. Rename refuses edits to these parameter bindings, and
+introduce-parameter refuses to rewrite such a signature. Queries still return
+definition locations and an unknown value. An unresolved generic annotation
+binding likewise refuses a potentially affected edit before any source changes.
+
+The model does not provide arbitrary dynamic class namespace inference. For
+example, assigning a lambda to ``Owner.callback`` after the class definition
+can leave attribute inference unknown; source binding identity is not a promise
+of complete dynamic value inference. Inline safety for deferred annotations and
+initializer removal is outside the source model's new consumer scope.
+
 Cross-Project Refactorings
 --------------------------
 

rope/base/evaluate.py

@@ -7,16 +7,38 @@
 from rope.base import (
     arguments,
     ast,
+    codeanalyze,
     exceptions,
     nameanalyze,
     pyobjects,
     pyobjectsdef,
+    pyscopes,
     worder,
 )
 
 BadIdentifierError = exceptions.BadIdentifierError
 
 
+class SourceValueScope(pyscopes.TemporaryScope):
+    """A call's value overlay, without replacing canonical source binders."""
+
+    def __init__(self, canonical_scope, capture_scope, names):
+        super().__init__(canonical_scope.pycore, canonical_scope, names)
+        self.pyobject = canonical_scope.pyobject
+        self.canonical_scope = canonical_scope
+        self.capture_scope = capture_scope
+
+    def lookup(self, name):
+        if name in self.names:
+            return self.names[name]
+        if name in self.canonical_scope.get_names():
+            return self.canonical_scope.get_names()[name]
+        return self.capture_scope.lookup(name)
+
+    def _propagated_lookup(self, name):
+        return self.lookup(name)
+
+
 def eval_location(pymodule, offset):
     """Find the pyname at the offset"""
     return eval_location2(pymodule, offset)[1]
@@ -37,6 +59,10 @@ def eval_node(scope, node):
 
 
 def eval_node2(scope, node):
+    module = scope.pyobject.get_module()
+    context = module.get_source_binding_index().context_for_node(node)
+    if context is not None and scope.get_kind() != "Temporary":
+        scope = context.environment
     evaluator = StatementEvaluator(scope)
     evaluator.visit(node)
     return evaluator.old_result, evaluator.result
@@ -92,31 +118,44 @@ def get_primary_and_pyname_at(
         offset: int,
     ) -> Tuple[Optional[rope.base.pynames.PyName], Optional[rope.base.pynames.PyName]]:
         lineno = self.lines.get_line_number(offset)
-        holding_scope = self.module_scope.get_inner_scope_for_offset(offset)
+        index = self.module_scope.pyobject.get_source_binding_index()
+        context = index.context_at(offset)
+        holding_scope = context.environment
+        structural_scope = context.structural_scope
+        parameter = index.binding_for_token(offset)
+        if parameter is not None:
+            return (None, parameter)
         # function keyword parameter
         if self.worder.is_function_keyword_parameter(offset):
             keyword_name = self.worder.get_word_at(offset)
             pyobject = self.get_enclosing_function(offset)
-            if isinstance(pyobject, pyobjectsdef.PyFunction):
+            from rope.base.sourcebindings import SourceLambdaFunction
+
+            if isinstance(pyobject, SourceLambdaFunction):
+                parameter_name = None
+                if keyword_name in pyobject.get_keyword_param_names():
+                    parameter_name = pyobject.get_parameters().get(keyword_name)
+                return (None, parameter_name)
+            elif isinstance(pyobject, pyobjectsdef.PyFunction):
                 parameter_name = pyobject.get_parameters().get(keyword_name, None)
                 return (None, parameter_name)
             elif isinstance(pyobject, pyobjects.AbstractFunction):
                 parameter_name = rope.base.pynames.ParameterName()
                 return (None, parameter_name)
         # class body
-        if self._is_defined_in_class_body(holding_scope, offset, lineno):
-            class_scope = holding_scope
-            if lineno == holding_scope.get_start():
-                class_scope = holding_scope.parent
+        if self._is_defined_in_class_body(structural_scope, offset, lineno):
+            class_scope = structural_scope
+            if lineno == structural_scope.get_start():
+                class_scope = structural_scope.parent
             name = self.worder.get_primary_at(offset).strip()
             try:
                 return (None, class_scope.pyobject[name])
             except rope.base.exceptions.AttributeNotFoundError:
                 return (None, None)
         # function header
-        if self._is_function_name_in_function_header(holding_scope, offset, lineno):
+        if self._is_function_name_in_function_header(structural_scope, offset, lineno):
             name = self.worder.get_primary_at(offset).strip()
-            return (None, holding_scope.parent[name])
+            return (None, structural_scope.parent[name])
         # module in a from statement or an imported name that is aliased
         if self.worder.is_from_statement_module(
             offset
@@ -128,14 +167,36 @@ def get_primary_and_pyname_at(
             name = self.worder.get_from_aliased(offset)
         else:
             name = self.worder.get_primary_at(offset)
-        return eval_str2(holding_scope, name)
+        primary, pyname = eval_str2(holding_scope, name)
+        from rope.base.sourcebindings import SourceValueName
+
+        if isinstance(pyname, SourceValueName):
+            pyname = pyname.source_pyname
+        return primary, pyname
 
     def get_enclosing_function(self, offset):
+        module = self.module_scope.pyobject
+        lines = codeanalyze.ASTLinesAdapter(module.source_code)
+        for node in ast.walk(module.get_ast()):
+            if not isinstance(node, ast.Call):
+                continue
+            for keyword in node.keywords:
+                if keyword.arg is None:
+                    continue
+                start, _ = lines[keyword]
+                if start <= offset < start + len(keyword.arg):
+                    return self._callable_object(
+                        eval_node(self.module_scope, node.func)
+                    )
         function_parens = self.worder.find_parens_start_from_inside(offset)
         try:
             function_pyname = self.get_pyname_at(function_parens - 1)
         except BadIdentifierError:
             function_pyname = None
+        return self._callable_object(function_pyname)
+
+    @staticmethod
+    def _callable_object(function_pyname):
         if function_pyname is not None:
             pyobject = function_pyname.get_object()
             if isinstance(pyobject, pyobjects.AbstractFunction):
@@ -174,6 +235,22 @@ def _Attribute(self, node):
         if pyname.get_object() != rope.base.pyobjects.get_unknown():
             try:
                 self.result = pyname.get_object()[node.attr]
+                from rope.base.sourcebindings import (
+                    SourceLambdaFunction,
+                    SourceValueName,
+                    source_lambda_attribute_value,
+                )
+
+                selected = source_lambda_attribute_value(pyname, self.result)
+                if selected is not None:
+                    self.result = selected
+                function = self.result.get_object()
+                if isinstance(function, SourceLambdaFunction):
+                    receiver = function.descriptor_receiver(pyname, self.result)
+                    if receiver is not None:
+                        self.result = SourceValueName(
+                            self.result, function.bind_receiver(receiver)
+                        )
             except exceptions.AttributeNotFoundError:
                 self.result = None
 
@@ -187,6 +264,21 @@ def _get_returned(pyobject):
             return pyobject.get_returned_object(args)
 
         if isinstance(pyobject, rope.base.pyobjects.AbstractClass):
+            from rope.base.sourcebindings import SourceLambdaFunction
+
+            if (
+                isinstance(pyobject, rope.base.builtins.BuiltinClass)
+                and pyobject.builtin in (staticmethod, classmethod)
+                and node.args
+            ):
+                function = eval_node(self.scope, node.args[0])
+                if function is not None and isinstance(
+                    function.get_object(), SourceLambdaFunction
+                ):
+                    self.result = rope.base.pynames.UnboundName(
+                        function.get_object().as_descriptor(pyobject.builtin.__name__)
+                    )
+                    return
             result = None
             if "__new__" in pyobject:
                 new_function = pyobject["__new__"].get_object()
@@ -201,6 +293,12 @@ def _get_returned(pyobject):
             pyfunction = pyobject
         elif "__call__" in pyobject:
             pyfunction = pyobject["__call__"].get_object()
+            from rope.base.sourcebindings import SourceLambdaFunction
+
+            if isinstance(pyfunction, SourceLambdaFunction):
+                pyfunction = pyfunction.bind_receiver(
+                    rope.base.pynames.UnboundName(pyobject)
+                )
         if pyfunction is not None:
             self.result = rope.base.pynames.UnboundName(
                 pyobject=_get_returned(pyfunction)
@@ -284,12 +382,76 @@ def _GeneratorExp(self, node):
             pyobject=rope.base.builtins.get_iterator(pyobject)
         )
 
+    def _SetComp(self, node):
+        pyobject = self._what_does_comprehension_hold(node)
+        self.result = rope.base.pynames.UnboundName(
+            pyobject=rope.base.builtins.get_set(pyobject)
+        )
+
+    def _DictComp(self, node):
+        scope = self._make_comprehension_scope(node)
+        key = eval_node(scope, node.key)
+        value = eval_node(scope, node.value)
+        self.result = rope.base.pynames.UnboundName(
+            pyobject=rope.base.builtins.get_dict(
+                key.get_object() if key is not None else None,
+                value.get_object() if value is not None else None,
+            )
+        )
+
     def _what_does_comprehension_hold(self, node):
         scope = self._make_comprehension_scope(node)
         pyname = eval_node(scope, node.elt)
         return pyname.get_object() if pyname is not None else None
 
     def _make_comprehension_scope(self, node):
+        module = self.scope.pyobject.get_module()
+        binding_scope = module.get_source_binding_index().comprehension_scope(node)
+        if binding_scope is not None:
+            if self.scope.get_kind() == "Temporary":
+                values = {
+                    name: rope.base.pynames.UnboundName()
+                    for name in binding_scope.get_names()
+                }
+                value_scope = SourceValueScope(binding_scope, self.scope, values)
+                for number, generator in enumerate(node.generators):
+                    entry = self.scope if number == 0 else value_scope
+                    iterator = eval_node(entry, generator.iter)
+                    if iterator is None:
+                        continue
+                    iterator_scope = pyscopes.TemporaryScope(
+                        entry.pycore, entry, {"__rope_iterable": iterator}
+                    )
+                    element = eval_str(
+                        iterator_scope, "__rope_iterable.__iter__().next()"
+                    )
+                    if element is None:
+                        continue
+                    for name, levels in nameanalyze.get_name_levels(generator.target):
+                        value = element
+                        for level in levels:
+                            element_type = value.get_object().get_type()
+                            if isinstance(element_type, rope.base.builtins.Tuple):
+                                holdings = element_type.get_holding_objects()
+                                value = (
+                                    rope.base.pynames.UnboundName(holdings[level])
+                                    if level < len(holdings)
+                                    and holdings[level] is not None
+                                    else None
+                                )
+                            else:
+                                item_scope = pyscopes.TemporaryScope(
+                                    entry.pycore, entry, {"__rope_element": value}
+                                )
+                                value = eval_str(
+                                    item_scope, "__rope_element[%s]" % level
+                                )
+                            if value is None:
+                                break
+                        if value is not None:
+                            values[name] = value
+                return value_scope
+            return binding_scope
         scope = self.scope
         module = scope.pyobject.get_module()
         names = {}
@@ -356,9 +518,13 @@ def _call_function(self, node, function_name, other_args=None):
             )
 
     def _Lambda(self, node):
-        self.result = rope.base.pynames.UnboundName(
-            pyobject=rope.base.builtins.Lambda(node, self.scope)
-        )
+        module = self.scope.pyobject.get_module()
+        function = module.get_source_binding_index().callable_for_node(node)
+        if function is None:
+            function = rope.base.builtins.Lambda(node, self.scope)
+        elif self.scope.get_kind() == "Temporary":
+            function = function.bind_creation_scope(self.scope)
+        self.result = rope.base.pynames.UnboundName(pyobject=function)
 
 
 def _get_evaluated_names(targets, assigned, module, evaluation, lineno):

rope/base/pyobjectsdef.py

@@ -214,6 +214,22 @@ def _create_concluded_attributes(self):
     def _create_scope(self):
         return rope.base.pyscopes.GlobalScope(self.pycore, self)
 
+    @utils.saveit
+    def get_source_binding_index(self):
+        from rope.base.sourcebindings import SourceBindingIndex
+
+        return SourceBindingIndex(self)
+
+    def get_attributes(self):
+        if ":=" in self.source_code:
+            self.get_source_binding_index().prepare_header_bindings()
+        return super().get_attributes()
+
+    def get_attribute(self, name):
+        if ":=" in self.source_code:
+            self.get_source_binding_index().prepare_header_bindings()
+        return super().get_attribute(name)
+
     @property
     @utils.saveit
     def lines(self):