loom-code 0.1.1__py3-none-any.whl

loom_code/loominit/schema.py

@@ -0,0 +1,242 @@
+"""Schema for ``.loom/index.json`` — the machine-readable index.
+
+This file is the CONTRACT between every other loominit module. The
+extractor produces it; the annotator consumes it; the staleness
+pipeline diffs against it; the refresh pass re-emits it. Lock the
+shape carefully — every field is hash-keyed where possible so the
+diff-aware refresh in :mod:`refresh` can identify what changed.
+
+We use Pydantic v2 because loomflow already pins it (``>=2.6``) —
+no new dep. Models are immutable (``frozen=True``) to surface
+accidental mutation as a TypeError; the refresh pass replaces, not
+edits.
+
+Versioning: ``LoomIndex.version`` is an integer. Bumping it
+invalidates older ``index.json`` files on read. Bump it any time
+the JSON shape changes incompatibly; for additive fields, the
+``model_config`` ``extra="ignore"`` policy keeps older readers
+working against newer writers.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Bump on incompatible schema changes. The reader (``persistence.py``)
+# refuses to load an index whose ``version`` differs from this — and
+# the REPL nudges the user to ``/loominit rebuild``.
+SCHEMA_VERSION = 1
+
+
+class _Immutable(BaseModel):
+    """Shared base — every loominit model is immutable + ignores
+    unknown fields so newer writers and older readers stay compatible
+    as long as we only add fields."""
+
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+
+# ---------------------------------------------------------------------------
+# Files
+# ---------------------------------------------------------------------------
+
+
+class FileEntry(_Immutable):
+    """One file in the repo. ``path`` is repo-relative POSIX.
+
+    ``sha256`` is the content hash the staleness pipeline diffs
+    against. ``in_api_surface`` is true when the file is reachable
+    from a package's ``__init__.py`` — agents over-read internals,
+    so this lets the annotator default to API-level descriptions.
+    """
+
+    path: str
+    lang: Literal["python", "markdown", "toml", "yaml", "json", "other"]
+    size_bytes: int
+    lines: int
+    sha256: str
+    mtime: datetime
+    # Number of commits touching this file in the last 90 days. None
+    # if the repo is not a git checkout. Used as a heat hint when
+    # ranking which files to annotate first.
+    git_changes_90d: int | None
+    is_test: bool
+    in_api_surface: bool
+
+
+# ---------------------------------------------------------------------------
+# Symbols (top-level class / function / module-level constant)
+# ---------------------------------------------------------------------------
+
+
+class SymbolEntry(_Immutable):
+    """One symbol — class, function, method, or module-level constant.
+
+    ``id`` is ``<path>:<qualified_name>`` (POSIX). The qualified name
+    is dotted for nested classes/methods (``Repl.run``); module
+    constants use just the bare name.
+
+    ``signature`` is the verbatim Python source for the
+    ``def …`` / ``class …`` line (single-line, no body). Useful in
+    LOOM.md as ground-truth — the annotator can quote it.
+
+    ``pagerank`` is the symbol's centrality in the import+call graph.
+    The annotator picks the top-K by this score plus everything in
+    ``in_api_surface=True``.
+
+    ``tests`` lists test-file callers (``tests/foo.py:42``). Built by
+    grepping test directories for the symbol's bare name — exact
+    matches only; the agent can verify if false-positive worries it.
+    """
+
+    id: str  # f"{path}:{qualified_name}"
+    name: str  # bare name ("login")
+    qualified_name: str  # dotted ("AuthManager.login")
+    kind: Literal["class", "function", "method", "constant"]
+    path: str
+    line: int
+    end_line: int
+    signature: str
+    docstring_first_line: str | None
+    decorators: list[str] = Field(default_factory=list)
+    is_public: bool  # not _-prefixed; in __all__ if defined
+    in_api_surface: bool  # reachable through package __init__.py
+    pagerank: float
+    n_callers: int
+    n_callees: int
+    tests: list[str] = Field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Graph edges
+# ---------------------------------------------------------------------------
+
+
+class ImportEdge(_Immutable):
+    """An ``import`` or ``from X import Y`` statement.
+
+    ``from_path`` and ``to_module`` are dotted Python module paths
+    when resolvable, otherwise the literal string from the source.
+    Unresolvable imports (third-party deps, stdlib) are still
+    recorded — they're a useful tech-stack signal — but never
+    contribute to the call graph."""
+
+    from_path: str
+    to_module: str
+    line: int
+    resolved: bool  # True if to_module maps to a file in this repo
+
+
+class CallEdge(_Immutable):
+    """A function-call edge ``caller -> callee``.
+
+    Both are symbol IDs (``<path>:<qualified_name>``). We only record
+    edges where BOTH ends resolve to symbols we know about — calls
+    into stdlib / third-party are dropped (would dominate PageRank
+    noise). Line is the call-site line in ``caller``'s file."""
+
+    caller: str
+    callee: str
+    line: int
+
+
+# ---------------------------------------------------------------------------
+# Landmarks — decorators / entry points
+# ---------------------------------------------------------------------------
+
+
+class DecoratorLandmark(_Immutable):
+    """A "landmark" decorator the annotator treats specially —
+    ``@app.route``, ``@click.command``, ``@tool``, ``@step``,
+    ``@pytest.fixture``, ``@dataclass``, ``@property``. These mark
+    entry points, tool definitions, fixtures, etc., and the
+    annotator uses them to populate the Entry Points section without
+    needing the LLM to discover them.
+
+    ``decorator`` is the source-form name (``"@app.route"`` —
+    keep the @ to disambiguate from regular calls). ``target`` is
+    the symbol ID being decorated."""
+
+    decorator: str
+    target: str  # symbol id
+    path: str
+    line: int
+
+
+class EntryPoint(_Immutable):
+    """A user-facing entry to the program.
+
+    Sources we mine:
+
+    * pyproject.toml ``[project.scripts]`` — ``kind="pyproject_script"``
+    * ``if __name__ == "__main__":`` blocks — ``kind="main_block"``
+    * Decorators on the landmark allow-list (``@click.command``,
+      ``@app.route``, etc.) — ``kind="decorated"``
+    """
+
+    kind: Literal["pyproject_script", "main_block", "decorated"]
+    name: str  # CLI name / route path / function name
+    path: str
+    line: int | None
+    callable_id: str | None  # symbol id, when known
+
+
+# ---------------------------------------------------------------------------
+# Clusters — file groups the annotator treats as one subsystem
+# ---------------------------------------------------------------------------
+
+
+class Cluster(_Immutable):
+    """A subsystem — group of files the annotator describes together.
+
+    Clustering signal: (1) path-prefix grouping (``loom_code/loominit/*``
+    is one cluster) takes precedence; (2) within larger packages, an
+    import-graph community-detection pass splits further.
+
+    ``hash_bucket`` is a hash over the SORTED list of file sha256s
+    in the cluster. When a file changes, the bucket changes; the
+    surgical refresh in :mod:`refresh` re-annotates only clusters
+    whose bucket moved.
+    """
+
+    id: str  # stable slug; used as the LOOM.md section anchor
+    title: str  # human-readable ("Loominit indexer")
+    paths: list[str]
+    centroid_symbols: list[str] = Field(default_factory=list)
+    centrality: float  # mean PageRank over symbols in the cluster
+    hash_bucket: str
+
+
+# ---------------------------------------------------------------------------
+# The top-level container
+# ---------------------------------------------------------------------------
+
+
+class LoomIndex(_Immutable):
+    """The whole ``.loom/index.json`` — produced by the structural
+    extractor, consumed by everything else.
+
+    Field ordering matches the conceptual layering: metadata first,
+    then files, then symbols, then graph edges, then landmarks /
+    clusters. Don't reorder — git diffs read more cleanly when the
+    JSON output preserves this shape across refreshes.
+    """
+
+    version: int = SCHEMA_VERSION
+    generated_at: datetime
+    repo_root: str  # absolute path on the generating machine
+    git_commit: str | None  # None if not a git repo
+    # Number of LLM calls the annotator made (recorded so /loominit
+    # status can report cost ballpark without re-reading workspace).
+    annotation_calls: int = 0
+
+    files: list[FileEntry] = Field(default_factory=list)
+    symbols: list[SymbolEntry] = Field(default_factory=list)
+    imports: list[ImportEdge] = Field(default_factory=list)
+    calls: list[CallEdge] = Field(default_factory=list)
+    decorators: list[DecoratorLandmark] = Field(default_factory=list)
+    entry_points: list[EntryPoint] = Field(default_factory=list)
+    clusters: list[Cluster] = Field(default_factory=list)

loom_code/lsp_tools.py

@@ -0,0 +1,396 @@
+"""LSP-backed navigation tools — go-to-definition, find-references, hover.
+
+The agent's answer to "where is X defined / who calls X / what is X" WITHOUT
+grep. grep finds the *string* ``login``; these resolve the *symbol* —
+through imports, across files, respecting scope — the way an IDE's
+"Go to Definition" does. Built on ``jedi`` (the pure-Python engine
+behind most editors' Python intelligence): no language-server process,
+no protocol, just in-process static analysis.
+
+Three tools, all keyed by a bare or dotted symbol name (the agent
+rarely knows exact line/columns, so we resolve the name across the
+project first, then run jedi at that location):
+
+* ``go_to_definition(symbol)`` — where it's defined (file:line + signature).
+* ``find_references(symbol)`` — every place it's used (file:line list).
+* ``hover(symbol)`` — its signature + docstring (the "what is this").
+
+Python-only (jedi is Python-only), matching the rest of loom-code's v1
+static-analysis surface (repomap, code index, AST walk). When jedi
+isn't installed, or a symbol can't be resolved, the tool returns a
+plain explanatory string and suggests grep — it never raises, so a
+navigation miss never aborts a turn.
+
+Why jedi over a real LSP server (pyright/pylsp): jedi is a single pure-
+Python dependency, starts instantly (no server lifecycle), and delivers
+>90% of the navigation value at a fraction of the weight. A real server
+is a future option if multi-language support is needed.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from loomflow import tool
+from loomflow.tools.registry import Tool
+
+# Directories we never scan for symbol resolution — vendored / generated
+# / VCS noise. Same spirit as the code index's skip set.
+_SKIP_DIRS: frozenset[str] = frozenset(
+    {
+        ".git",
+        ".loom",
+        ".venv",
+        "venv",
+        "node_modules",
+        "__pycache__",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        "dist",
+        "build",
+        ".tox",
+        "site-packages",
+    }
+)
+
+# Cap how many files we scan to resolve a bare symbol name to its
+# definition site. A monorepo with thousands of files would make the
+# resolve-by-name scan slow; the cap keeps the tool snappy. The agent
+# can pass a more specific dotted name or a path hint if a symbol is
+# missed in a huge tree.
+_MAX_SCAN_FILES = 400
+
+# Cap references returned — a heavily-used symbol could have hundreds of
+# call sites; the agent wants the map, not the phone book.
+_MAX_REFS = 40
+
+
+def _iter_py_files(root: Path) -> list[Path]:
+    out: list[Path] = []
+    for p in root.rglob("*.py"):
+        if any(part in _SKIP_DIRS for part in p.parts):
+            continue
+        out.append(p)
+        if len(out) >= _MAX_SCAN_FILES:
+            break
+    return out
+
+
+def _find_definition_site(
+    root: Path, symbol: str
+) -> tuple[Any, Path, int, int] | None:
+    """Resolve a bare/dotted ``symbol`` to its definition location by
+    scanning project files with jedi's name index.
+
+    Returns ``(jedi.Script, path, line, column)`` at the definition, or
+    None if not found. We match the LAST dotted segment against jedi's
+    definition names (so ``AuthManager.login`` matches a ``login``
+    method), preferring an exact qualified match when available.
+    """
+    import jedi  # local import — optional dep, surfaced as a tool message
+
+    bare = symbol.rsplit(".", 1)[-1]
+    fallback: tuple[Any, Path, int, int] | None = None
+    for fpath in _iter_py_files(root):
+        try:
+            source = fpath.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        try:
+            script = jedi.Script(code=source, path=str(fpath))
+            names = script.get_names(all_scopes=True, definitions=True)
+        except Exception:
+            continue
+        for n in names:
+            if n.name != bare:
+                continue
+            # Exact match on the qualified name wins immediately; a
+            # bare-name match is held as a fallback (first one found).
+            qual = _qualified(n)
+            if qual == symbol or n.name == symbol:
+                return script, fpath, n.line, n.column
+            if fallback is None:
+                fallback = (script, fpath, n.line, n.column)
+    return fallback
+
+
+def _qualified(name: Any) -> str:
+    """Best-effort dotted name for a jedi Name (full_name when jedi
+    resolved it, else the bare name)."""
+    full = getattr(name, "full_name", None)
+    return str(full) if full else str(getattr(name, "name", ""))
+
+
+def _rel(root: Path, p: Path | str | None) -> str:
+    if p is None:
+        return "<unknown>"
+    try:
+        return Path(p).resolve().relative_to(root).as_posix()
+    except (ValueError, OSError):
+        return str(p)
+
+
+# ---------------------------------------------------------------------------
+# go_to_definition
+# ---------------------------------------------------------------------------
+
+
+def go_to_definition_tool(workdir: Path | str) -> Tool:
+    """Build the ``go_to_definition`` tool for ``workdir``.
+
+    Model-facing: ``go_to_definition(symbol)`` — ``symbol`` is a
+    function/class/method name (bare ``login`` or dotted
+    ``AuthManager.login``). Returns the definition's file:line +
+    signature so the agent can ``read`` it directly instead of
+    grepping. Python-only; falls back to a grep suggestion when jedi
+    is unavailable or the symbol can't be resolved.
+    """
+    root = Path(workdir).resolve()
+
+    async def go_to_definition(symbol: str) -> str:
+        symbol = str(symbol).strip()
+        if not symbol:
+            return "go_to_definition: empty symbol"
+        try:
+            import jedi  # noqa: F401
+        except ImportError:
+            return (
+                "go_to_definition unavailable (jedi not installed). "
+                "Use grep to find the definition."
+            )
+        try:
+            found = _find_definition_site(root, symbol)
+            if found is None:
+                return (
+                    f"go_to_definition: no Python definition found for "
+                    f"'{symbol}'. Try grep, or a more specific name."
+                )
+            script, path, line, col = found
+            defs = script.goto(
+                line=line,
+                column=col,
+                follow_imports=True,
+                follow_builtin_imports=False,
+            )
+            if not defs:
+                # The name-index hit IS the definition site.
+                sig = _line_text(path, line)
+                return (
+                    f"{_rel(root, path)}:{line}  {sig}"
+                )
+            out: list[str] = []
+            for d in defs:
+                dpath = d.module_path
+                dline = d.line or line
+                sig = (d.description or "").strip()
+                loc = f"{_rel(root, dpath)}:{dline}"
+                out.append(f"  {d.type} {d.name}  ({loc})  {sig}")
+            return "\n".join(out)
+        except Exception as exc:  # never abort a turn on a nav failure
+            return (
+                f"go_to_definition unavailable "
+                f"({type(exc).__name__}: {exc}). Use grep."
+            )
+
+    return tool(
+        name="go_to_definition",
+        description=(
+            "Jump to where a Python symbol is DEFINED — resolves "
+            "through imports + scope like an IDE's Go-to-Definition, "
+            "unlike grep which only matches the string. Args: symbol "
+            "(function/class/method name, bare 'login' or dotted "
+            "'AuthManager.login'). Returns the definition's file:line "
+            "+ signature to read next. Python only; use grep for "
+            "other languages."
+        ),
+    )(go_to_definition)
+
+
+# ---------------------------------------------------------------------------
+# find_references
+# ---------------------------------------------------------------------------
+
+
+def find_references_tool(workdir: Path | str) -> Tool:
+    """Build the ``find_references`` tool for ``workdir``.
+
+    Model-facing: ``find_references(symbol)`` — every place ``symbol``
+    is used across the project (file:line list), scope-aware via jedi.
+    The answer to "what breaks if I change this?" — far more precise
+    than grepping the bare name (which matches comments, strings, and
+    unrelated same-named locals).
+    """
+    root = Path(workdir).resolve()
+
+    async def find_references(symbol: str) -> str:
+        symbol = str(symbol).strip()
+        if not symbol:
+            return "find_references: empty symbol"
+        try:
+            import jedi  # noqa: F401
+        except ImportError:
+            return (
+                "find_references unavailable (jedi not installed). "
+                "Use grep to find usages."
+            )
+        try:
+            found = _find_definition_site(root, symbol)
+            if found is None:
+                return (
+                    f"find_references: no Python definition found for "
+                    f"'{symbol}'. Try grep."
+                )
+            script, path, line, col = found
+            refs = script.get_references(
+                line=line, column=col, scope="project"
+            )
+            if not refs:
+                return f"find_references: no usages found for '{symbol}'"
+            seen: set[tuple[str, int]] = set()
+            lines: list[str] = []
+            for r in refs:
+                rel = _rel(root, r.module_path)
+                key = (rel, r.line or 0)
+                if key in seen:
+                    continue
+                seen.add(key)
+                text = _line_text(
+                    Path(r.module_path) if r.module_path else path,
+                    r.line or 0,
+                )
+                lines.append(f"  {rel}:{r.line}  {text}")
+                if len(lines) >= _MAX_REFS:
+                    lines.append(
+                        f"  … (+more; showing first {_MAX_REFS})"
+                    )
+                    break
+            return "\n".join(lines)
+        except Exception as exc:
+            return (
+                f"find_references unavailable "
+                f"({type(exc).__name__}: {exc}). Use grep."
+            )
+
+    return tool(
+        name="find_references",
+        description=(
+            "Find every place a Python symbol is USED across the "
+            "project — scope-aware (resolves through imports, ignores "
+            "same-named unrelated locals/strings/comments that grep "
+            "would falsely match). Args: symbol (function/class/method "
+            "name). Returns a file:line list of real usages — the "
+            "answer to 'what breaks if I change this?'. Python only."
+        ),
+    )(find_references)
+
+
+# ---------------------------------------------------------------------------
+# hover
+# ---------------------------------------------------------------------------
+
+
+def hover_tool(workdir: Path | str) -> Tool:
+    """Build the ``hover`` tool for ``workdir``.
+
+    Model-facing: ``hover(symbol)`` — the symbol's signature +
+    docstring (the IDE "what is this" tooltip), so the agent learns a
+    function's contract without reading its whole file.
+    """
+    root = Path(workdir).resolve()
+
+    async def hover(symbol: str) -> str:
+        symbol = str(symbol).strip()
+        if not symbol:
+            return "hover: empty symbol"
+        try:
+            import jedi  # noqa: F401
+        except ImportError:
+            return (
+                "hover unavailable (jedi not installed). "
+                "Use read to inspect the symbol."
+            )
+        try:
+            found = _find_definition_site(root, symbol)
+            if found is None:
+                return (
+                    f"hover: no Python definition found for '{symbol}'. "
+                    "Use grep / read."
+                )
+            script, path, line, col = found
+            helps = script.help(line=line, column=col)
+            if not helps:
+                return f"hover: nothing to show for '{symbol}'"
+            parts: list[str] = []
+            for h in helps:
+                loc = f"{_rel(root, path)}:{line}"
+                header = f"{h.type} {h.name}  ({loc})"
+                sigs = []
+                try:
+                    sigs = [s.to_string() for s in h.get_signatures()]
+                except Exception:
+                    sigs = []
+                doc = ""
+                try:
+                    doc = (h.docstring() or "").strip()
+                except Exception:
+                    doc = ""
+                block = header
+                if sigs:
+                    block += "\n  " + sigs[0]
+                if doc:
+                    # jedi's docstring() prepends the call signature as
+                    # the FIRST paragraph, then the real docstring. Drop
+                    # that leading signature paragraph so we show the
+                    # actual contract, not "verify_token(token)" twice.
+                    paras = doc.split("\n\n", 1)
+                    body = (
+                        paras[1] if len(paras) == 2 else paras[0]
+                    ).strip()
+                    if body:
+                        block += "\n  " + body.replace("\n", "\n  ")
+                parts.append(block)
+            return "\n\n".join(parts)
+        except Exception as exc:
+            return (
+                f"hover unavailable ({type(exc).__name__}: {exc}). "
+                "Use read."
+            )
+
+    return tool(
+        name="hover",
+        description=(
+            "Show a Python symbol's signature + docstring (the IDE "
+            "'what is this' tooltip) — learn a function's contract "
+            "without reading its whole file. Args: symbol "
+            "(function/class/method name). Python only; use read for "
+            "other languages or full bodies."
+        ),
+    )(hover)
+
+
+def _line_text(path: Path | str, line: int) -> str:
+    """The stripped source text at ``path:line`` — a one-line preview
+    so reference/definition lists show what's there. Best-effort."""
+    if line <= 0:
+        return ""
+    try:
+        text = Path(path).read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+    lines = text.splitlines()
+    if 0 < line <= len(lines):
+        return lines[line - 1].strip()[:100]
+    return ""
+
+
+def lsp_tools(workdir: Path | str) -> list[Tool]:
+    """All three navigation tools for ``workdir`` — the convenience
+    bundle the agent builder wires in one call (mirrors how the file
+    tools are grouped)."""
+    return [
+        go_to_definition_tool(workdir),
+        find_references_tool(workdir),
+        hover_tool(workdir),
+    ]