diffcone 0.1.0__py3-none-any.whl

diffcone/indexer/symbols.py

@@ -0,0 +1,441 @@
+"""Pass 1: a module's statements and definitions become symbols, with
+their hashes, line spans and import tables."""
+
+from __future__ import annotations
+
+import ast
+from collections import defaultdict
+
+from diffcone.indexer.definitions import (
+    _annotated_args,
+    _canonical_imports,
+    _end_line,
+    _flatten_chain,
+    _future_annotations,
+    _has_annotations,
+    _import_layout,
+    _inert_def,
+    _is_inert_decorator,
+    _is_literal,
+    _is_special_method,
+    _start_line,
+    _variable_statements,
+)
+from diffcone.indexer.literals import (
+    _collect_literal_bindings,
+    _collect_store_names,
+    _module_mutations,
+)
+from diffcone.indexer.scopes import (
+    ClassScope,
+    ImportBinding,
+    ModuleScope,
+    VariableStatement,
+    _absolute_module,
+)
+from diffcone.indexer.state import IndexerState
+from diffcone.indexer.syntax import (
+    DEF_NODES,
+    FUNC_NODES,
+    _digest,
+    _docstring_hash,
+    _split_docstring,
+    hash_nodes,
+    hash_scope_body,
+    iter_scope_statements,
+)
+from diffcone.model import (
+    CLASS,
+    DEFINED_IN,
+    FUNCTION,
+    METHOD,
+    MODULE,
+    REFERENCES,
+    VARIABLE,
+    Edge,
+    Symbol,
+)
+from diffcone.snapshot import member_symbol_id
+
+
+def _doc_reads(body: list[ast.stmt]):
+    """``__doc__`` names and attributes, and ``getdoc(...)`` calls, in a
+    scope's own statements (not in nested definitions, which are symbols
+    of their own)."""
+    stack: list[ast.AST] = list(body)
+    while stack:
+        node = stack.pop()
+        if isinstance(node, DEF_NODES):
+            # Decorators, defaults and bases run in this scope; the body not.
+            stack.extend(node.decorator_list)
+            if isinstance(node, ast.ClassDef):
+                stack.extend([*node.bases, *node.keywords])
+            else:
+                stack.extend([*node.args.defaults, *(d for d in node.args.kw_defaults if d)])
+            continue
+        # Reads only: ``__doc__ = """..."""`` assigns a docstring, it reads none.
+        if isinstance(node, ast.Name) and node.id == "__doc__":
+            if isinstance(node.ctx, ast.Load):
+                yield node
+        elif isinstance(node, ast.Attribute) and node.attr == "__doc__":
+            if isinstance(node.ctx, ast.Load):
+                yield node
+        elif isinstance(node, ast.Call):
+            func = node.func
+            name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+            if name == "getdoc":
+                yield node
+        stack.extend(ast.iter_child_nodes(node))
+
+
+def _reads_docstrings_deep(node: ast.AST) -> bool:
+    """Like _reads_docstrings, nested definitions included."""
+    for inner in ast.walk(node):
+        if isinstance(inner, (ast.Name, ast.Attribute)) and isinstance(inner.ctx, ast.Load):
+            if getattr(inner, "id", None) == "__doc__" or getattr(inner, "attr", None) == "__doc__":
+                return True
+        if isinstance(inner, ast.Call):
+            func = inner.func
+            name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+            if name == "getdoc":
+                return True
+    return False
+
+
+def _reads_docstrings(body: list[ast.stmt]) -> bool:
+    return next(_doc_reads(body), None) is not None
+
+
+def _names_module_doc(body: list[ast.stmt]) -> bool:
+    return any(isinstance(n, ast.Name) for n in _doc_reads(body))
+
+
+class FirstPass(IndexerState):
+    """Pass 1: modules and their definitions into symbols."""
+
+    def _module_statements(
+        self, scope: ModuleScope, *, register_imports: bool
+    ) -> tuple[list[ast.stmt], list[ast.stmt], dict[str, VariableStatement]]:
+        """(all scope statements, body without docstring, variable statements)
+        of a parsed module; records the import statements and, unless the
+        import table was served by the cache, fills it."""
+        assert scope.tree is not None
+        stmts = list(iter_scope_statements(scope.tree.body))
+        scope.import_nodes = [s for s in stmts if isinstance(s, (ast.Import, ast.ImportFrom))]
+        if register_imports:
+            for node in scope.import_nodes:
+                self._register_imports(
+                    scope, node, scope.imports, scope.star_imports, scope.alt_imports
+                )
+        body = _split_docstring(scope.tree.body)[1]
+        return stmts, body, _variable_statements(scope, body)
+
+    def _index_module(self, scope: ModuleScope) -> None:
+        assert scope.tree is not None
+        stmts, body, variable_stmts = self._module_statements(scope, register_imports=True)
+        for stmt in stmts:
+            if not isinstance(stmt, DEF_NODES + (ast.Import, ast.ImportFrom)):
+                scope.bindings |= _collect_store_names(stmt)
+        scope.literal_names = _collect_literal_bindings(scope.tree, {})
+        scope.mutations = frozenset(_module_mutations(scope.tree))
+        imports = tuple(sorted(_canonical_imports(scope)))
+        layout = _import_layout(scope)
+        # A variable statement is its own symbol, so it leaves the module's
+        # body hash; one rebinding a def or class name (``helper = 3`` after
+        # ``def helper``) is not a variable symbol and stays in it.
+        defined = {s.name for s in stmts if isinstance(s, DEF_NODES)}
+        own_symbol = {id(stmt) for name, stmt in variable_stmts.items() if name not in defined}
+        module_body_hash = hash_scope_body(
+            [s for s in body if id(s) not in own_symbol], strip_imports=True
+        )
+        module_doc_hash = _docstring_hash([scope.tree.body])
+        module_reads_doc = _reads_docstrings(scope.tree.body)
+        if module_reads_doc and _names_module_doc(scope.tree.body):
+            # ``ArgumentParser(description=__doc__)``: import-time code reads
+            # the module's own docstring.
+            module_body_hash = _digest(module_body_hash + "|doc:" + module_doc_hash)
+        self._add_symbol(
+            Symbol(
+                id=scope.name,
+                kind=MODULE,
+                module=scope.name,
+                name=scope.name.rsplit(".", 1)[-1],
+                path=scope.path,
+                lineno=1,
+                body_hash=module_body_hash,
+                docstring_hash=module_doc_hash,
+                definition_hash=_digest("\n".join(imports) + "|layout|" + "\n".join(layout)),
+                container=None,
+                line_ranges=((1, _end_line(scope.tree)),),
+                imports=imports,
+                import_layout=layout,
+                reads_docstrings=module_reads_doc,
+            )
+        )
+        self._index_definitions(scope, scope.tree.body, scope.name, scope.members, None)
+        # Module-level statements that mention a variable may mutate it in place
+        # (``REGISTRY[k] = v``, ``NAMES.append(x)``, ``CONFIG.update(...)``), so
+        # they are part of that variable's body, not only of the module's.
+        mutators: dict[str, list[ast.stmt]] = defaultdict(list)
+        variable_ids = {id(s) for s in variable_stmts.values()}
+        for stmt in body:
+            if isinstance(stmt, DEF_NODES + (ast.Import, ast.ImportFrom)):
+                continue
+            if id(stmt) in variable_ids:
+                continue
+            mentioned = {
+                n.id for n in ast.walk(stmt) if isinstance(n, ast.Name) and n.id in variable_stmts
+            }
+            for name in mentioned:
+                mutators[name].append(stmt)
+        for name, stmt in variable_stmts.items():
+            if name in scope.members:
+                continue  # also a def/class: kept in the module body hash above
+            symbol_id = self._member_id(scope.name, name)
+            value = stmt.value
+            assert value is not None  # _variable_statements keeps assignments with a value
+            symbol = Symbol(
+                id=symbol_id,
+                kind=VARIABLE,
+                module=scope.name,
+                name=name,
+                path=scope.path,
+                lineno=stmt.lineno,
+                body_hash=hash_nodes([value, *mutators.get(name, [])]),
+                # An annotation is evaluated at import unless deferred: a
+                # change to it is a definition change (see classify).
+                annotation_hash=(
+                    _digest(hash_nodes([stmt.annotation]))
+                    if isinstance(stmt, ast.AnnAssign)
+                    else ""
+                ),
+                deferred_annotations=_future_annotations(scope),
+                definition_hash="",
+                container=scope.name,
+                line_ranges=tuple(
+                    (s.lineno, _end_line(s)) for s in (stmt, *mutators.get(name, []))
+                ),
+                # Binding a literal runs no code when the module is imported;
+                # only readers of the value can observe the change. ``__all__``
+                # is not inert: it decides what ``from m import *`` binds.
+                inert_definition=(
+                    name != "__all__" and not mutators.get(name) and _is_literal(value)
+                ),
+            )
+            if self._add_symbol(symbol):
+                scope.variables[name] = symbol_id
+                scope.variable_stmts[name] = stmt
+                self.out.edges.add(Edge(symbol_id, scope.name, DEFINED_IN))
+
+    def _register_imports(
+        self,
+        scope: ModuleScope,
+        node: ast.stmt,
+        table: dict[str, ImportBinding],
+        stars: list[str],
+        alternatives: dict[str, list[ImportBinding]] | None = None,
+    ) -> None:
+        def bind(name: str, binding: ImportBinding) -> None:
+            previous = table.get(name)
+            if alternatives is not None and previous is not None and previous != binding:
+                if previous not in alternatives.setdefault(name, []):
+                    alternatives[name].append(previous)
+            table[name] = binding
+
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                if alias.asname:
+                    bind(alias.asname, ImportBinding(alias.name, None))
+                else:
+                    bind(alias.name.split(".")[0], ImportBinding(alias.name.split(".")[0], None))
+        elif isinstance(node, ast.ImportFrom):
+            base = _absolute_module(scope, node.module, node.level)
+            for alias in node.names:
+                if alias.name == "*":
+                    stars.append(base)
+                else:
+                    bind(alias.asname or alias.name, ImportBinding(base, alias.name))
+
+    def _index_definitions(
+        self,
+        scope: ModuleScope,
+        body: list[ast.stmt],
+        container_id: str,
+        members: dict[str, str],
+        class_scope: ClassScope | None,
+    ) -> None:
+        funcs: dict[str, list[ast.FunctionDef | ast.AsyncFunctionDef]] = {}
+        classes: dict[str, list[ast.ClassDef]] = {}
+        order: list[str] = []
+        for stmt in iter_scope_statements(body):
+            if isinstance(stmt, FUNC_NODES):
+                funcs.setdefault(stmt.name, []).append(stmt)
+            elif isinstance(stmt, ast.ClassDef):
+                classes.setdefault(stmt.name, []).append(stmt)
+            else:
+                continue
+            if stmt.name not in order:
+                order.append(stmt.name)
+        for name in order:
+            if class_scope is None and container_id == scope.name:
+                symbol_id = self._member_id(scope.name, name)
+            else:
+                symbol_id = f"{container_id}.{name}"
+            if name in classes:
+                nodes = classes[name]
+                first = nodes[0]
+                member_names = sorted(
+                    {
+                        s.name
+                        for n in nodes
+                        for s in iter_scope_statements(n.body)
+                        if isinstance(s, DEF_NODES)
+                    }
+                )
+
+                def class_hashes(nodes=nodes, member_names=member_names):
+                    definition_parts: list[ast.AST] = []
+                    for n in nodes:
+                        definition_parts += (
+                            list(n.bases) + list(n.keywords) + list(n.decorator_list)
+                        )
+                    return (
+                        _digest(
+                            "\n".join(
+                                hash_scope_body(_split_docstring(n.body)[1], strip_imports=False)
+                                for n in nodes
+                            )
+                        ),
+                        _digest(hash_nodes(definition_parts) + "|" + ",".join(member_names)),
+                        _docstring_hash([n.body for n in nodes]),
+                    )
+
+                body_hash, definition_hash, doc_hash = class_hashes()
+                symbol = Symbol(
+                    id=symbol_id,
+                    kind=CLASS,
+                    module=scope.name,
+                    name=name,
+                    path=scope.path,
+                    lineno=first.lineno,
+                    body_hash=body_hash,
+                    definition_hash=definition_hash,
+                    container=container_id,
+                    line_ranges=tuple((_start_line(n), _end_line(n)) for n in nodes),
+                    docstring_hash=doc_hash,
+                    reads_docstrings=any(_reads_docstrings(n.body) for n in nodes),
+                )
+                if not self._add_symbol(symbol):
+                    continue
+                members[name] = symbol_id
+                self.out.edges.add(Edge(symbol_id, container_id, DEFINED_IN))
+                cscope = ClassScope(id=symbol_id, module=scope, enclosing=class_scope)
+                cscope.plain = len(nodes) == 1 and not (first.decorator_list or first.keywords)
+                for n in nodes:
+                    cscope.base_chains.extend(_flatten_chain(b) for b in n.bases)
+                    cscope.base_names.extend(
+                        _flatten_chain(b.value if isinstance(b, ast.Subscript) else b)
+                        for b in n.bases
+                    )
+                self.class_scopes[symbol_id] = cscope
+                self._added_classes.append(cscope)
+                for n in nodes:
+                    for stmt in iter_scope_statements(n.body):
+                        if not isinstance(stmt, DEF_NODES):
+                            cscope.bindings |= _collect_store_names(stmt)
+                # Every definition of the class (``if``/``else`` variants) is
+                # one symbol, so its members are indexed together: a method
+                # defined in several of them is one symbol too.
+                bodies = [stmt for n in nodes for stmt in n.body]
+                self._index_definitions(scope, bodies, symbol_id, cscope.members, cscope)
+                # Special methods run implicitly on instances (``==``, ``len()``,
+                # calling one, ``with``): whatever references the class may
+                # trigger them, so the class depends on them.
+                for member, member_id in sorted(cscope.members.items()):
+                    member_symbol = self.index.symbols.get(member_id)
+                    if (
+                        _is_special_method(member)
+                        and member_symbol is not None
+                        and member_symbol.kind == METHOD
+                    ):
+                        self.out.edges.add(Edge(symbol_id, member_id, REFERENCES, "special_method"))
+            else:
+                nodes = funcs[name]
+                first = nodes[0]
+
+                def function_hashes(nodes=nodes):
+                    definition_parts: list[ast.AST] = []
+                    annotation_parts: list[ast.AST] = []
+                    for n in nodes:
+                        definition_parts.append(n.args)
+                        definition_parts += list(n.decorator_list)
+                        if n.returns is not None:
+                            annotation_parts.append(n.returns)
+                    # The definition hash excludes annotations: detach them
+                    # while hashing (no copy of the tree) and restore them.
+                    detached = [a for n in nodes for a in _annotated_args(n.args)]
+                    annotations = [a.annotation for a in detached]
+                    for a in detached:
+                        a.annotation = None
+                    try:
+                        definition = hash_nodes(definition_parts)
+                    finally:
+                        for a, annotation in zip(detached, annotations, strict=True):
+                            a.annotation = annotation
+                    annotation_parts = [
+                        *(a for a in annotations if a is not None),
+                        *annotation_parts,
+                    ]
+                    return (
+                        _digest(hash_nodes(annotation_parts)),
+                        _digest(
+                            "\n".join(hash_nodes(list(_split_docstring(n.body)[1])) for n in nodes)
+                        ),
+                        _digest(definition + "|" + ",".join(type(n).__name__ for n in nodes)),
+                        _docstring_hash([n.body for n in nodes]),
+                    )
+
+                annotation_hash, body_hash, definition_hash, doc_hash = function_hashes()
+                deferred = (
+                    _future_annotations(scope)
+                    and all(_is_inert_decorator(d, scope) for n in nodes for d in n.decorator_list)
+                    and (class_scope is None or class_scope.plain)
+                )
+                header = all(_inert_def(n, scope) for n in nodes) and (
+                    class_scope is None or class_scope.plain
+                )
+                inert = header and (deferred or not any(_has_annotations(n) for n in nodes))
+                symbol = Symbol(
+                    id=symbol_id,
+                    kind=METHOD if class_scope is not None else FUNCTION,
+                    module=scope.name,
+                    name=name,
+                    path=scope.path,
+                    lineno=first.lineno,
+                    body_hash=body_hash,
+                    docstring_hash=doc_hash,
+                    definition_hash=definition_hash,
+                    container=container_id,
+                    line_ranges=tuple((_start_line(n), _end_line(n)) for n in nodes),
+                    annotation_hash=annotation_hash,
+                    deferred_annotations=deferred,
+                    inert_definition=inert,
+                    inert_header=header,
+                    # Nested functions are not symbols: a decorator factory's
+                    # wrapper reading ``f.__doc__`` counts for the factory.
+                    reads_docstrings=any(_reads_docstrings_deep(n) for n in nodes),
+                )
+                if not self._add_symbol(symbol):
+                    continue
+                members[name] = symbol_id
+                self.out.edges.add(Edge(symbol_id, container_id, DEFINED_IN))
+
+    def _member_id(self, module: str, name: str) -> str:
+        return member_symbol_id(module, name, self._children.get(module, frozenset()))
+
+    def modules_with_prefix(self, prefix: str) -> tuple[str, ...]:
+        return tuple(sorted(m for m in self.scopes if m.startswith(prefix)))
+
+    def symbol_names_with_prefix(self, prefix: str) -> tuple[str, ...]:
+        names = {s.name for s in self.index.symbols.values() if s.name.startswith(prefix)}
+        return tuple(sorted(names))

diffcone/indexer/syntax.py

@@ -0,0 +1,173 @@
+"""AST helpers the indexer and discovery share: name tables, source
+decoding, scope statements and the hashes of bodies and definitions."""
+
+from __future__ import annotations
+
+import ast
+import builtins
+import copy
+import hashlib
+import io
+import tokenize
+
+BUILTIN_NAMES = frozenset(dir(builtins))
+
+
+# Builtins whose argument nothing can bound: what they run is a string of
+# code or the module's own namespace. ``__import__`` is not among them -- it
+# names a module, and a name is exactly what the literal machinery bounds
+# (pandas imports its hard dependencies with ``__import__`` in a loop over a
+# literal tuple, and treating that as unbounded selected its whole suite).
+DYNAMIC_CALLS = frozenset({"eval", "exec", "globals", "vars"})
+
+
+# Reflection that observes names or signatures without naming them. Recorded
+# for evidence mode only (SourceIndex.reflection); static planning ignores it.
+REFLECTIVE_BUILTINS = frozenset({"dir", "hasattr", "vars"})
+
+
+REFLECTIVE_CALLS = frozenset(
+    {
+        "inspect.getmembers",
+        "inspect.getmembers_static",
+        "inspect.signature",
+        "inspect.getfullargspec",
+        "inspect.getcallargs",
+        "inspect.get_annotations",
+        "typing.get_type_hints",
+        "annotationlib.get_annotations",
+    }
+)
+
+
+REFLECTIVE_ATTRIBUTES = frozenset(
+    {
+        "__dict__",
+        "__annotations__",
+        "__signature__",
+        "__code__",
+        "__defaults__",
+        "__kwdefaults__",
+        "modules",  # sys.modules: any module, found by name
+    }
+)
+
+
+# How far to follow a module name passed from caller to caller before giving
+# up and leaving the dynamic reference where it is.
+IMPORT_ATTRIBUTION_DEPTH = 4
+
+
+DEF_NODES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+
+
+FUNC_NODES = (ast.FunctionDef, ast.AsyncFunctionDef)
+
+
+def decode_source(data: bytes) -> str:
+    """A source file as text, in the encoding it declares. Python reads a
+    coding cookie (PEP 263) or a BOM before it reads the source, and a file
+    that declares one is not UTF-8 (pip's latin-1 test package)."""
+    encoding, _ = tokenize.detect_encoding(io.BytesIO(data).readline)
+    return data.decode(encoding)
+
+
+def _digest(text: str) -> str:
+    return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
+
+
+class _StripDefs(ast.NodeTransformer):
+    """Remove nested definitions (and optionally imports) from a scope body.
+
+    Definitions are hashed as their own symbols and imports as the module's
+    definition hash, so leaving them here would double-count changes.
+    """
+
+    def __init__(self, strip_imports: bool) -> None:
+        self.strip_imports = strip_imports
+
+    def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> ast.stmt | None:
+        return None
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.stmt | None:
+        return self.visit_FunctionDef(node)
+
+    def visit_ClassDef(self, node: ast.ClassDef) -> ast.stmt | None:
+        return None
+
+    def visit_Import(self, node: ast.Import) -> ast.stmt | None:
+        return None if self.strip_imports else node
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.stmt | None:
+        return None if self.strip_imports else node
+
+
+def _dump(node: ast.AST) -> str:
+    return ast.dump(node, include_attributes=False)
+
+
+def hash_nodes(nodes: list[ast.AST]) -> str:
+    return _digest("\n".join(_dump(n) for n in nodes))
+
+
+def _contains_definition_or_import(stmt: ast.stmt, strip_imports: bool) -> bool:
+    for node in ast.walk(stmt):
+        if isinstance(node, DEF_NODES):
+            return True
+        if strip_imports and isinstance(node, (ast.Import, ast.ImportFrom)):
+            return True
+    return False
+
+
+def _split_docstring(stmts: list[ast.stmt]) -> tuple[str, list[ast.stmt]]:
+    """(docstring text, statements without it) for a module/class/function body."""
+    if (
+        stmts
+        and isinstance(stmts[0], ast.Expr)
+        and isinstance(stmts[0].value, ast.Constant)
+        and isinstance(stmts[0].value.value, str)
+    ):
+        return stmts[0].value.value, stmts[1:]
+    return "", stmts
+
+
+def _docstring_hash(bodies: list[list[ast.stmt]]) -> str:
+    docs = [_split_docstring(b)[0] for b in bodies]
+    return _digest("\n".join(docs)) if any(docs) else ""
+
+
+def hash_scope_body(stmts: list[ast.stmt], strip_imports: bool) -> str:
+    """Hash a scope's statements with nested definitions (and optionally
+    imports) removed. Only statements that actually contain one are copied
+    and stripped; the rest are dumped as they are, which avoids a deep copy
+    of every statement (the dominant cost of indexing)."""
+    kept: list[ast.AST] = []
+    for stmt in stmts:
+        if isinstance(stmt, DEF_NODES) or (
+            strip_imports and isinstance(stmt, (ast.Import, ast.ImportFrom))
+        ):
+            continue
+        if _contains_definition_or_import(stmt, strip_imports):
+            stripped = _StripDefs(strip_imports).visit(copy.deepcopy(stmt))
+            if stripped is not None:
+                kept.append(stripped)
+        else:
+            kept.append(stmt)
+    return hash_nodes(kept)
+
+
+def iter_scope_statements(body: list[ast.stmt]):
+    """Yield statements of a scope, descending into compound statements but
+    never into function or class definitions."""
+    for stmt in body:
+        yield stmt
+        if isinstance(stmt, DEF_NODES):
+            continue
+        for attr in ("body", "orelse", "finalbody"):
+            child = getattr(stmt, attr, None)
+            if isinstance(child, list):
+                yield from iter_scope_statements(child)
+        for handler in getattr(stmt, "handlers", []) or []:
+            yield from iter_scope_statements(handler.body)
+        for case in getattr(stmt, "cases", []) or []:
+            yield from iter_scope_statements(case.body)