diffcontext 0.5.1__py3-none-any.whl

diffcontext/resolver.py

@@ -0,0 +1,288 @@
+"""
+resolver.py — Resolve Python import statements to filesystem paths.
+"""
+
+import ast
+import logging
+import os
+from collections import deque
+from typing import Dict, List, Optional, Tuple
+
+from ._warn_once import warn_syntax_error_once, check_and_warn_encoding
+
+logger = logging.getLogger(__name__)
+
+
+# Import statements can only appear in statement blocks — never inside an
+# expression — so import scanning walks only these fields instead of every
+# AST node (expressions dominate node count ~10:1). Field order mirrors the
+# AST's own field order so import precedence matches a full BFS walk.
+_STMT_BLOCK_FIELDS = ("body", "handlers", "orelse", "finalbody", "cases")
+
+
+def _iter_import_nodes(tree: "ast.Module"):
+    """Yield every Import/ImportFrom in the tree, in BFS document order,
+    without descending into expression subtrees."""
+    queue = deque(tree.body)
+    while queue:
+        node = queue.popleft()
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            yield node
+            continue
+        for field in _STMT_BLOCK_FIELDS:
+            block = getattr(node, field, None)
+            if block:
+                queue.extend(block)
+
+
+def build_import_map(
+    filename: str,
+    repo_path: str,
+    tree: "Optional[ast.Module]" = None,
+) -> Dict[str, str]:
+    """
+    Parse imports in a file and resolve them to absolute paths.
+
+    Returns:
+        dict: local_name -> absolute_path_of_source_file
+        e.g. {"helper": "/repo/utils.py", "Session": "/repo/sessions.py"}
+
+    __init__.py transparency:
+        When an import resolves to a package __init__.py, we scan that
+        __init__.py for re-export statements (`from .sub import Name`) and
+        follow them to the actual definition file. This means:
+
+            from flask import Flask
+            # resolves to src/flask/__init__.py
+            # __init__.py has: from .app import Flask
+            # final result: src/flask/app.py   ← correct
+    """
+    # Accept a pre-parsed AST to avoid re-reading and re-parsing the file
+    # (the pipeline parses each file exactly once and shares the tree).
+    if tree is None:
+        with open(filename, "rb") as f:
+            raw = f.read()
+        check_and_warn_encoding(logger, filename, raw)
+        source = raw.decode("utf-8", errors="ignore")
+
+        try:
+            tree = ast.parse(source)
+        except SyntaxError as e:
+            warn_syntax_error_once(logger, filename, e)
+            return {}
+
+    imports: Dict[str, str] = {}
+    file_dir = os.path.dirname(os.path.abspath(filename))
+    repo_abs = os.path.abspath(repo_path)
+
+    for node in _iter_import_nodes(tree):
+
+        if isinstance(node, ast.ImportFrom):
+            module = node.module or ""
+            level = node.level
+
+            if level > 0:
+                # Relative import: from .utils import helper
+                base = file_dir
+                for _ in range(level - 1):
+                    base = os.path.dirname(base)
+                module_path = os.path.join(base, module.replace(".", os.sep))
+                resolved = _resolve_module_path(module_path)
+            else:
+                # Absolute import: from requests.utils import helper
+                # Resolved against every source root (repo root, then src/):
+                # black, flask, and most modern PyPI projects keep their
+                # packages under src/, where a repo-root-only lookup finds
+                # nothing and silently drops every edge into the package.
+                module_path, resolved = _resolve_absolute_module(
+                    module, repo_abs
+                )
+
+            for alias in node.names:
+                local_name = alias.asname or alias.name
+
+                # Check if this is a submodule import (from package import module)
+                submodule_path = os.path.join(module_path, alias.name)
+                submodule_resolved = _resolve_module_path(submodule_path)
+
+                if submodule_resolved:
+                    imports[local_name] = submodule_resolved
+                elif resolved:
+                    # __init__.py transparency: follow re-exports one level
+                    if resolved.endswith("__init__.py"):
+                        real = _follow_init_reexport(resolved, alias.name, repo_abs)
+                        imports[local_name] = real if real else resolved
+                    else:
+                        imports[local_name] = resolved
+
+        elif isinstance(node, ast.Import):
+            for alias in node.names:
+                top_module = alias.name.split(".")[0]
+
+                if alias.asname:
+                    # `import a.b as ab` — ab refers to the submodule a.b
+                    _mp, resolved = _resolve_absolute_module(alias.name, repo_abs)
+                    if not resolved:
+                        sibling = os.path.join(
+                            file_dir, alias.name.replace(".", os.sep)
+                        )
+                        resolved = _resolve_module_path(sibling)
+                    if resolved:
+                        imports[alias.asname] = resolved
+                    continue
+
+                # `import a` / `import a.b` — the bound local name is the TOP
+                # package `a` (binding it to a/b.py, as previously done, sent
+                # `a.other()` calls into the wrong file).
+                _mp, resolved = _resolve_absolute_module(top_module, repo_abs)
+                if not resolved:
+                    # Bare `import x` — try the importing file's own directory
+                    sibling_path = os.path.join(
+                        file_dir, top_module.replace(".", os.sep)
+                    )
+                    resolved = _resolve_module_path(sibling_path)
+                if resolved:
+                    imports[top_module] = resolved
+
+                # `import a.b` also makes `a.b.fn()` callable — record the
+                # full dotted path (dots can't collide with identifiers).
+                if "." in alias.name:
+                    _mp, sub_resolved = _resolve_absolute_module(
+                        alias.name, repo_abs
+                    )
+                    if sub_resolved:
+                        imports[alias.name] = sub_resolved
+
+    return imports
+
+
+# Conventional source roots tried, in order, when resolving absolute
+# imports. "" is the repo root itself (flat layout); "src" is the
+# setuptools src-layout used by black, flask, requests, and most modern
+# PyPI projects.
+_SOURCE_ROOT_NAMES = ("", "src")
+
+
+def _resolve_absolute_module(module: str, repo_abs: str):
+    """
+    Resolve a dotted absolute module name against each candidate source
+    root of the repository.
+
+    Returns (module_path, resolved_file):
+      module_path   — directory-ish path for the module under the root that
+                      matched (used for submodule probing by the caller);
+                      falls back to the repo-root join when nothing matched.
+      resolved_file — the module's .py / package __init__.py, or None.
+    """
+    rel = module.replace(".", os.sep)
+    fallback = os.path.join(repo_abs, rel)
+    for root_name in _SOURCE_ROOT_NAMES:
+        root = os.path.join(repo_abs, root_name) if root_name else repo_abs
+        candidate = os.path.join(root, rel)
+        resolved = _resolve_module_path(candidate)
+        if resolved:
+            return candidate, resolved
+        # Namespace package (no __init__.py): a real directory still lets
+        # the caller find `from pkg import submodule` targets inside it.
+        if os.path.isdir(candidate):
+            return candidate, None
+    return fallback, None
+
+
+# Session cache of parsed __init__.py re-export specs, keyed by absolute
+# path and validated by (mtime_ns, size) so an edited file is re-read. A
+# flagship package's __init__.py (django.db.models, flask) is consulted by
+# hundreds of importing files per cold index; without this each consult
+# re-read and re-parsed it from disk.
+_init_export_cache: "Dict[str, Tuple[Tuple[int, int], Optional[Dict[str, List[Tuple[str, int, str]]]]]]" = {}
+
+
+def _init_export_specs(
+    init_path: str,
+) -> "Optional[Dict[str, List[Tuple[str, int, str]]]]":
+    """
+    exported_name -> [(module, level, original_name), ...] for every
+    `from X import Y` in the file, in document order. None when the file
+    is unreadable or unparsable.
+    """
+    try:
+        st = os.stat(init_path)
+        stat_key = (st.st_mtime_ns, st.st_size)
+    except OSError:
+        return None
+    cached = _init_export_cache.get(init_path)
+    if cached is not None and cached[0] == stat_key:
+        return cached[1]
+
+    specs: "Optional[Dict[str, List[Tuple[str, int, str]]]]"
+    try:
+        with open(init_path, "rb") as f:
+            raw = f.read()
+        tree = ast.parse(raw.decode("utf-8", errors="ignore"))
+    except (OSError, SyntaxError):
+        specs = None
+    else:
+        specs = {}
+        for node in _iter_import_nodes(tree):
+            if not isinstance(node, ast.ImportFrom):
+                continue
+            for alias in node.names:
+                exported = alias.asname or alias.name
+                specs.setdefault(exported, []).append(
+                    (node.module or "", node.level, alias.name)
+                )
+    _init_export_cache[init_path] = (stat_key, specs)
+    return specs
+
+
+def _follow_init_reexport(init_path: str, name: str, repo_abs: str) -> Optional[str]:
+    """
+    Given a package __init__.py and a name imported from it, check whether
+    __init__.py re-exports that name from a submodule.
+
+    Example:
+        __init__.py contains:  from .app import Flask
+        name = "Flask"
+        → returns /abs/path/to/app.py
+
+    Both relative (`from .app import Flask`, flask style) and absolute
+    (`from black.parsing import parse_ast`, black style) re-exports are
+    followed. Only follows one level (no recursive re-export chasing) to
+    stay fast. Returns None if the name is not re-exported or the
+    submodule can't be found.
+    """
+    specs = _init_export_specs(init_path)
+    if not specs:
+        return None
+
+    init_dir = os.path.dirname(init_path)
+
+    for module, level, original_name in specs.get(name, ()):
+        if level > 0:
+            # Relative re-export: from .sub import X
+            base = init_dir
+            for _ in range(level - 1):
+                base = os.path.dirname(base)
+            sub_path = os.path.join(base, module.replace(".", os.sep))
+        else:
+            # Absolute re-export: from black.parsing import X
+            sub_path, _resolved = _resolve_absolute_module(module, repo_abs)
+        # Check if the original name itself is a submodule
+        submodule_path = os.path.join(sub_path, original_name)
+        resolved = _resolve_module_path(submodule_path) or _resolve_module_path(sub_path)
+        if resolved and resolved != os.path.normpath(init_path):
+            return resolved
+
+    return None
+
+
+def _resolve_module_path(module_path: str) -> Optional[str]:
+    """Try to find a .py file or package __init__.py for a given path."""
+    candidates = [
+        module_path + ".py",
+        os.path.join(module_path, "__init__.py"),
+    ]
+    for candidate in candidates:
+        if os.path.isfile(candidate):
+            return os.path.normpath(candidate)
+    return None

diffcontext/scanner.py

@@ -0,0 +1,153 @@
+"""
+scanner.py — Discover source files in a repository.
+
+Python always; other languages via the optional adapters in languages/
+(each adapter contributes its extensions to discovery only when its
+runtime deps are installed).
+"""
+
+import os
+import subprocess
+from typing import List, Optional, Set, Tuple
+
+EXCLUDED_DIRS: Set[str] = {
+    "__pycache__",
+    ".git",
+    ".tox",
+    ".mypy_cache",
+    ".pytest_cache",
+    "venv",
+    ".venv",
+    "env",
+    "node_modules",
+    "experimental",
+    "examples",
+    "docs",
+    "tests",
+    "test",
+    "benchmarks",
+    "datasets",
+    "dist",
+    "build",
+    "egg-info",
+}
+
+# Directories excluded from indexing by default. These are deliberately
+# NOT retrieval candidates: tests/, benchmarks/, docs/ are tracked in git
+# but rarely the "code that matters for a change", and indexing them would
+# both bloat the graph and drown real blast-radius signal in test scaffolding.
+#
+# This is the single biggest practical gotcha: a commit that spans an
+# excluded dir (e.g. benchmarks/) produces "was not found in the index"
+# warnings for every changed symbol in that dir, and the changed file is
+# omitted from the context entirely — the tool looks broken when it is
+# merely mis-scoped. Override with `--include <dir>...` (see
+# find_source_files / the CLI) to index an excluded dir; .gitignore still
+# applies on top (a gitignored dir is not indexed even with --include).
+
+
+def _is_excluded_dir(name: str, include: Optional[Set[str]] = None) -> bool:
+    """True if directory `name` should be pruned, unless it is in `include`
+    (a set of directory names to keep despite the default exclusions)."""
+    if include and name in include:
+        return False
+    return name in EXCLUDED_DIRS or name.endswith(".egg-info")
+
+
+def _excluded(rel_path: str, include: Optional[Set[str]] = None) -> bool:
+    """True if any directory component of rel_path is excluded (and not
+    overridden by `include`)."""
+    parts = rel_path.replace(os.sep, "/").split("/")[:-1]
+    return any(_is_excluded_dir(p, include) for p in parts)
+
+
+def first_excluded_dir(
+    rel_path: str, include: Optional[Set[str]] = None,
+) -> Optional[str]:
+    """Return the first directory component of `rel_path` that is excluded
+    (and not overridden by `include`), else None.
+
+    Used by warn_unknown_symbols to distinguish "your changed symbol's file
+    is outside the indexed tree" (actionable: re-run with --include) from
+    "typo / renamed / deleted" (a different kind of mistake)."""
+    parts = rel_path.replace(os.sep, "/").split("/")[:-1]
+    for p in parts:
+        if _is_excluded_dir(p, include):
+            return p
+    return None
+
+
+def _git_source_files(
+    root_dir: str, extensions: "Tuple[str, ...]",
+    include: Optional[Set[str]] = None,
+) -> Optional[List[str]]:
+    """
+    Enumerate matching files via git: tracked + untracked-but-not-ignored.
+
+    This makes indexing respect .gitignore, so vendored checkouts (e.g. a
+    cloned benchmark repo) never pollute the index — a hardcoded dir list
+    can't anticipate those. Returns None outside a git work tree or if git
+    is unavailable, so the caller falls back to the filesystem walk.
+
+    `include` overrides the hardcoded EXCLUDED_DIRS (e.g. {"benchmarks"}
+    keeps benchmarks/ even though it is excluded by default). It does NOT
+    override .gitignore — a gitignored dir is still omitted by git ls-files.
+    """
+    try:
+        out = subprocess.run(
+            ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+            cwd=root_dir, capture_output=True, timeout=15,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if out.returncode != 0:
+        return None
+
+    matched = []
+    for rel in out.stdout.decode("utf-8", "replace").split("\0"):
+        if not rel.endswith(extensions) or _excluded(rel, include):
+            continue
+        full = os.path.join(root_dir, rel)
+        # --cached lists tracked files even after deletion from disk
+        if os.path.isfile(full):
+            matched.append(full)
+    return matched
+
+
+def find_source_files(
+    root_dir: str, extensions: "Tuple[str, ...]",
+    include: Optional[Set[str]] = None,
+) -> List[str]:
+    """
+    Return paths of files matching `extensions`: .gitignore-aware via git
+    when root_dir is inside a git work tree, else a tree walk. Both paths
+    skip EXCLUDED_DIRS (deliberate exclusions like tests/ and docs/ that
+    are tracked in git but not useful retrieval candidates).
+
+    `include` is a set of directory names to KEEP despite the default
+    exclusions (e.g. {"benchmarks", "tests"} indexes those dirs too).
+    Matching is by directory-name component anywhere in the tree, so
+    `--include tests` un-excludes both top-level tests/ and any nested
+    dir named tests/. .gitignore still applies: a gitignored dir is not
+    indexed even when named in `include`.
+    """
+    git_files = _git_source_files(root_dir, extensions, include)
+    if git_files is not None:
+        return git_files
+
+    matched = []
+    for root, dirs, files in os.walk(root_dir):
+        dirs[:] = [d for d in dirs if not _is_excluded_dir(d, include)]
+
+        for f in files:
+            if f.endswith(extensions):
+                matched.append(os.path.join(root, f))
+
+    return matched
+
+
+def find_python_files(
+    root_dir: str, include: Optional[Set[str]] = None,
+) -> List[str]:
+    """Return list of .py file paths (see find_source_files)."""
+    return find_source_files(root_dir, (".py",), include)

diffcontext/symbols.py

@@ -0,0 +1,254 @@
+"""
+symbols.py — Attribute ownership extraction for resolving self.attr.method() chains.
+
+Given a class with `self.router = APIRouter()`, this module figures out that
+`self.router` has type `APIRouter`, enabling cross-file method resolution.
+"""
+
+import ast
+from typing import Dict, Optional, Tuple
+
+
+def _iter_statements(body):
+    """Yield statements in source order, recursing into if/for/while/with/try."""
+    for stmt in body:
+        yield stmt
+
+        if isinstance(stmt, (ast.If, ast.For, ast.AsyncFor, ast.While)):
+            yield from _iter_statements(stmt.body)
+            yield from _iter_statements(stmt.orelse)
+
+        elif isinstance(stmt, (ast.With, ast.AsyncWith)):
+            yield from _iter_statements(stmt.body)
+
+        elif isinstance(stmt, ast.Try):
+            yield from _iter_statements(stmt.body)
+            for handler in stmt.handlers:
+                yield from _iter_statements(handler.body)
+            yield from _iter_statements(stmt.orelse)
+            yield from _iter_statements(stmt.finalbody)
+
+
+def _extract_annotation_name(annotation) -> Optional[Tuple[Optional[str], str]]:
+    """
+    Returns (qualifier, bare_name) or None.
+
+    Router          ->  (None, "Router")
+    Optional[Router]->  (None, "Router")       <- unwrap subscript
+    routing.Router  ->  ("routing", "Router")
+    """
+    if isinstance(annotation, ast.Name):
+        return (None, annotation.id)
+
+    if isinstance(annotation, ast.Attribute) and isinstance(annotation.value, ast.Name):
+        return (annotation.value.id, annotation.attr)
+
+    # Optional[Router], List[Router], etc.  — unwrap one level
+    if isinstance(annotation, ast.Subscript):
+        return _extract_annotation_name(annotation.slice)
+
+    return None
+
+
+def _extract_call_owner(value) -> Optional[Tuple[Optional[str], str]]:
+    """
+    Returns (qualifier, bare_name) or None.
+
+    APIRouter()          ->  (None,      "APIRouter")
+    routing.APIRouter()  ->  ("routing", "APIRouter")
+    make_helper()        ->  (None,      "make_helper")
+    """
+    if not isinstance(value, ast.Call):
+        return None
+
+    func = value.func
+
+    if isinstance(func, ast.Name):
+        return (None, func.id)
+
+    if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
+        return (func.value.id, func.attr)
+
+    if isinstance(func, ast.Attribute):
+        return (None, func.attr)
+
+    return None
+
+
+def _extract_assign_owner(value, local_var_types=None, param_types=None):
+    """
+    Handles non-Call RHS cases that still indicate a type:
+      self.x = router or Router()   -> BoolOp -> try each operand
+      self.x = router               -> bare Name -> no type info (return None)
+    """
+    if local_var_types is None:
+        local_var_types = {}
+    if param_types is None:
+        param_types = {}
+
+    if isinstance(value, ast.Call):
+        return _extract_call_owner(value)
+
+    if isinstance(value, ast.Name):
+        if value.id in param_types:
+            return param_types[value.id]
+        if value.id in local_var_types:
+            return local_var_types[value.id]
+        return None
+
+    if isinstance(value, ast.BoolOp):
+        call_result = None
+        name_result = None
+        for operand in value.values:
+            ref = _extract_assign_owner(operand, local_var_types, param_types)
+            if ref is None:
+                continue
+            if isinstance(operand, ast.Call):
+                call_result = call_result or ref
+            else:
+                name_result = name_result or ref
+        return call_result or name_result
+
+    return None
+
+
+def extract_local_var_types(
+    fn_node,
+    param_types: Optional[Dict[str, Tuple[Optional[str], str]]] = None,
+) -> Dict[str, Tuple[Optional[str], str]]:
+    """
+    Track local variable -> type assignments within a SINGLE function body,
+    free function or method alike:
+
+        h = Helper()              -> {"h": (None, "Helper")}
+        r: routing.Router = ...   -> {"r": ("routing", "Router")}
+        x = make_helper()         -> chases factory functions the same way
+                                      attribute ownership tracking does
+                                      (resolution of the factory return type
+                                      itself happens later, in graph_builder)
+
+    This is the free-function counterpart to the self.attr tracking that
+    extract_attribute_ownerships does for class bodies. It exists because
+    a huge fraction of real code instantiates a class in a local variable
+    inside a plain function (not as a self.attr) and immediately calls a
+    method on it -- e.g.:
+
+        def run():
+            h = Handler()
+            return h.process()
+
+    Without this, `h.process()` can never resolve: there's nowhere that
+    records "h" has type "Handler". extract_attribute_ownerships alone
+    can't help, because it only ever looks inside ast.ClassDef bodies.
+
+    Returns: {local_var_name: (qualifier, bare_type_name)}
+    """
+    if param_types is None:
+        param_types = {}
+
+    local_var_types: Dict[str, Tuple[Optional[str], str]] = {}
+
+    for stmt in _iter_statements(fn_node.body):
+
+        if isinstance(stmt, ast.AnnAssign):
+            target = stmt.target
+            if isinstance(target, ast.Name):
+                ref = _extract_annotation_name(stmt.annotation)
+                if ref:
+                    local_var_types[target.id] = ref
+            continue
+
+        if isinstance(stmt, ast.Assign):
+            for target in stmt.targets:
+                if isinstance(target, ast.Name):
+                    ref = _extract_assign_owner(
+                        stmt.value, local_var_types, param_types
+                    )
+                    if ref:
+                        local_var_types[target.id] = ref
+            continue
+
+    return local_var_types
+
+
+def extract_param_types(fn_node) -> Dict[str, Tuple[Optional[str], str]]:
+    """
+    Map annotated parameter names to their (qualifier, bare_name) type, e.g.
+
+        def f(router: routing.Router, name: str): ...
+
+    -> {"router": ("routing", "Router")}   (unannotated/non-type params skipped)
+    """
+    param_types: Dict[str, Tuple[Optional[str], str]] = {}
+    for arg in fn_node.args.args:
+        if arg.annotation is not None:
+            ref = _extract_annotation_name(arg.annotation)
+            if ref:
+                param_types[arg.arg] = ref
+    return param_types
+
+
+def extract_attribute_ownerships(tree) -> Dict[str, Tuple[Optional[str], str]]:
+    """
+    Extracts mappings like:
+        self.router: routing.Router = routing.Router()
+        self.router = Router()
+        self.router2 = router            # typed constructor param
+        h = Helper(); self.helper = h    # local var tracking
+        self.router3 = router or Router()
+        self.opt_router: Optional[Router] = None
+
+    Returns: {"FastAPI.router": (qualifier, type_name), ...}
+    """
+    ownerships: Dict[str, Tuple[Optional[str], str]] = {}
+
+    for node in tree.body:
+        if not isinstance(node, ast.ClassDef):
+            continue
+
+        class_name = node.name
+
+        for item in node.body:
+            if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                continue
+
+            param_types = extract_param_types(item)
+
+            local_var_types = {}
+
+            for stmt in _iter_statements(item.body):
+
+                if isinstance(stmt, ast.AnnAssign):
+                    target = stmt.target
+                    if (
+                        isinstance(target, ast.Attribute)
+                        and isinstance(target.value, ast.Name)
+                        and target.value.id == "self"
+                    ):
+                        ref = _extract_annotation_name(stmt.annotation)
+                        if ref:
+                            ownerships[f"{class_name}.{target.attr}"] = ref
+                    continue
+
+                if isinstance(stmt, ast.Assign):
+                    for target in stmt.targets:
+                        if (
+                            isinstance(target, ast.Attribute)
+                            and isinstance(target.value, ast.Name)
+                            and target.value.id == "self"
+                        ):
+                            ref = _extract_assign_owner(
+                                stmt.value, local_var_types, param_types
+                            )
+                            if ref:
+                                ownerships[f"{class_name}.{target.attr}"] = ref
+
+                        elif isinstance(target, ast.Name):
+                            ref = _extract_assign_owner(
+                                stmt.value, local_var_types, param_types
+                            )
+                            if ref:
+                                local_var_types[target.id] = ref
+                    continue
+
+    return ownerships