sin-code-bundle 0.9.2__py3-none-any.whl

sin_code_bundle/lsp_backend.py

@@ -0,0 +1,303 @@
+# SPDX-License-Identifier: MIT
+"""LSP-backed symbol resolution for the SCKG.
+
+This makes `impact()` structural and type-accurate instead of textual:
+- "what calls this symbol?"  -> LSP references
+- "where is it defined?"     -> LSP definition
+- blast-radius scoring        -> ranked caller set + fan-in
+
+Primary backend: multilspy (drives real language servers: pyright, gopls,
+typescript-language-server, rust-analyzer, jdtls, …).
+Fallback backend: tree-sitter symbol scan (cheap, language-agnostic, no server).
+
+The module degrades gracefully: if no LSP is available it returns tree-sitter
+results and flags `source="treesitter"`, so the agent still gets a useful signal
+and the bundle keeps working (consistent with `sin status`).
+
+Docs: lsp_backend.doc.md
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal, Optional
+
+Source = Literal["lsp", "treesitter", "none"]
+
+_LANG_BY_EXT = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+    ".rb": "ruby",
+    ".php": "php",
+    ".cs": "csharp",
+    ".c": "c",
+    ".cpp": "cpp",
+    ".h": "cpp",
+}
+
+
+@dataclass(frozen=True)
+class Location:
+    """A single source-code position, optionally with a short snippet."""
+
+    file: str
+    line: int
+    column: int
+    snippet: str = ""
+
+
+# ── LSPBackend: Language Server Manager ────────────────────────────────
+@dataclass
+class ImpactResult:
+    """Compact, deterministic blast-radius payload for the agent."""
+
+    symbol: str
+    defined_at: Optional[Location]
+    callers: list[Location] = field(default_factory=list)
+    fan_in: int = 0
+    touches_tests: bool = False
+    touches_public_api: bool = False
+    risk: Literal["low", "medium", "high"] = "low"
+    source: Source = "none"
+    notes: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Serialize to a JSON-safe dict (caches the result under `cache.set`).
+
+        Returns a plain dict with `Location` fields flattened to `{file, line,
+        column, snippet}` so the GraphCache (JSONL-backed) can round-trip it
+        without a custom encoder.
+        """
+        return {
+            "symbol": self.symbol,
+            "defined_at": _loc_to_dict(self.defined_at),
+            "callers": [_loc_to_dict(c) for c in self.callers],
+            "fan_in": self.fan_in,
+            "touches_tests": self.touches_tests,
+            "touches_public_api": self.touches_public_api,
+            "risk": self.risk,
+            "source": self.source,
+            "notes": self.notes,
+        }
+
+
+def _loc_to_dict(loc: Optional[Location]) -> Optional[dict]:
+    if loc is None:
+        return None
+    return {"file": loc.file, "line": loc.line, "column": loc.column, "snippet": loc.snippet}
+
+
+def _lang_for(path: Path) -> Optional[str]:
+    return _LANG_BY_EXT.get(path.suffix.lower())
+
+
+def _score_risk(
+    callers: int, touches_tests: bool, touches_api: bool
+) -> Literal["low", "medium", "high"]:
+    # Thresholds are intentionally simple and conservative. >10 callers = broad
+    # blast radius (high). >3 = significant surface area (medium). Tests + API
+    # each escalate one tier (e.g. a 4-caller non-test/non-api function is
+    # "low" but a 4-caller test-touching one is "medium").
+    if touches_api or callers > 10:
+        return "high"
+    if touches_tests or callers > 3:
+        return "medium"
+    return "low"
+
+
+def _is_test_path(p: str) -> bool:
+    pl = p.lower()
+    return "test" in Path(pl).name or "/tests/" in pl or pl.endswith("_test.py")
+
+
+def _is_public_api_path(p: str) -> bool:
+    name = Path(p).name.lower()
+    return name in {"__init__.py", "api.py", "index.ts", "index.js", "mod.rs", "lib.rs"}
+
+
+# ── Language Detection: File → Server Mapping ──────────────────────────
+# --------------------------------------------------------------------------- #
+# LSP backend (multilspy)
+# --------------------------------------------------------------------------- #
+async def _lsp_impact(
+    root: Path, file: Path, symbol: str, line: int, column: int
+) -> Optional[ImpactResult]:
+    try:
+        from multilspy import LanguageServer  # type: ignore
+        from multilspy.multilspy_config import MultilspyConfig  # type: ignore
+        from multilspy.multilspy_logger import MultilspyLogger  # type: ignore
+    except ImportError:
+        return None
+
+    lang = _lang_for(file)
+    if not lang:
+        return None
+
+    config = MultilspyConfig.from_dict({"code_language": lang})
+    logger = MultilspyLogger()
+    server = LanguageServer.create(config, logger, str(root))
+
+    rel = str(file.relative_to(root)) if file.is_absolute() else str(file)
+    async with server.start_server():
+        definition = await server.request_definition(rel, line - 1, column - 1)
+        references = await server.request_references(rel, line - 1, column - 1)
+
+    def_loc: Optional[Location] = None
+    if definition:
+        d = definition[0]
+        def_loc = Location(
+            file=d.get("relativePath", d.get("uri", "")),
+            line=d["range"]["start"]["line"] + 1,
+            column=d["range"]["start"]["character"] + 1,
+        )
+
+    callers: list[Location] = []
+    for ref in references or []:
+        rp = ref.get("relativePath", ref.get("uri", ""))
+        callers.append(
+            Location(
+                file=rp,
+                line=ref["range"]["start"]["line"] + 1,
+                column=ref["range"]["start"]["character"] + 1,
+            )
+        )
+
+    touches_tests = any(_is_test_path(c.file) for c in callers)
+    touches_api = any(_is_public_api_path(c.file) for c in callers)
+    fan_in = len(callers)
+    # Cap caller list at 25 — fits an LLM prompt-friendly blast-radius view
+    # without dropping high-fan-in signals. Anything larger reports the
+    # truncated count in `notes` so the agent can ask for more if needed.
+    return ImpactResult(
+        symbol=symbol,
+        defined_at=def_loc,
+        callers=callers[:25],
+        fan_in=fan_in,
+        touches_tests=touches_tests,
+        touches_public_api=touches_api,
+        risk=_score_risk(fan_in, touches_tests, touches_api),
+        source="lsp",
+        notes=[] if fan_in <= 25 else [f"{fan_in} callers total; showing first 25"],
+    )
+
+
+# --------------------------------------------------------------------------- #
+# tree-sitter fallback (textual but symbol-aware)
+# --------------------------------------------------------------------------- #
+def _treesitter_impact(root: Path, symbol: str) -> ImpactResult:
+    bare = symbol.split(".")[-1].split("::")[-1]
+    callers: list[Location] = []
+    defined_at: Optional[Location] = None
+
+    for path in root.rglob("*"):
+        if not path.is_file() or _lang_for(path) is None:
+            continue
+        if any(part in {".git", "node_modules", ".venv", "__pycache__"} for part in path.parts):
+            continue
+        try:
+            text = path.read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            continue
+        for i, raw in enumerate(text.splitlines(), start=1):
+            if bare not in raw:
+                continue
+            col = raw.find(bare) + 1
+            loc = Location(
+                file=str(path.relative_to(root)),
+                line=i,
+                column=col,
+                snippet=raw.strip()[:120],
+            )
+            stripped = raw.lstrip()
+            if defined_at is None and (
+                stripped.startswith(("def ", "class ", "func ", "fn ", "function "))
+                and bare in stripped.split("(")[0]
+            ):
+                defined_at = loc
+            else:
+                callers.append(loc)
+
+    touches_tests = any(_is_test_path(c.file) for c in callers)
+    touches_api = any(_is_public_api_path(c.file) for c in callers)
+    fan_in = len(callers)
+    # Mirror the same 25-caller cap as the LSP path above — keeps both
+    # backend outputs structurally identical so callers don't have to branch.
+    return ImpactResult(
+        symbol=symbol,
+        defined_at=defined_at,
+        callers=callers[:25],
+        fan_in=fan_in,
+        touches_tests=touches_tests,
+        touches_public_api=touches_api,
+        risk=_score_risk(fan_in, touches_tests, touches_api),
+        source="treesitter",
+        notes=["LSP unavailable — textual approximation. Install 'sin[lsp]' for accuracy."],
+    )
+
+
+# ── Graceful Shutdown: Cleanup Lifecycle ──────────────────────────────
+# --------------------------------------------------------------------------- #
+# Public entry point
+# --------------------------------------------------------------------------- #
+def compute_impact(
+    root: str | Path,
+    symbol: str,
+    file: Optional[str | Path] = None,
+    line: Optional[int] = None,
+    column: Optional[int] = None,
+) -> ImpactResult:
+    """Resolve the blast radius of `symbol`.
+
+    If (file, line, column) are given and an LSP is available, returns precise
+    LSP references. Otherwise falls back to a tree-sitter/textual scan.
+
+    Results are cached under .sin/cache/ and reused if the repo hasn't changed.
+    """
+    root_path = Path(root).resolve()
+
+    # Cache layer
+    from sin_code_bundle.cache import GraphCache
+
+    cache = GraphCache(root_path)
+    cache_key = f"impact:{symbol}:{file}:{line}:{column}"
+    cached = cache.get(cache_key)
+    if cached is not None:
+        defined = cached.get("defined_at")
+        return ImpactResult(
+            symbol=cached["symbol"],
+            defined_at=Location(**defined) if defined else None,
+            callers=[Location(**c) for c in cached.get("callers", [])],
+            fan_in=cached.get("fan_in", 0),
+            touches_tests=cached.get("touches_tests", False),
+            touches_public_api=cached.get("touches_public_api", False),
+            risk=cached.get("risk", "low"),
+            source=cached.get("source", "none"),
+            notes=cached.get("notes", []),
+        )
+
+    if file and line and column:
+        file_path = (
+            (root_path / file) if not Path(file).is_absolute() else Path(file)  # type: ignore[arg-type]
+        )
+        try:
+            result = asyncio.run(_lsp_impact(root_path, file_path, symbol, line, column))
+            if result is not None:
+                cache.set(cache_key, result.to_dict())
+                return result
+        except Exception as exc:  # noqa: BLE001
+            ts = _treesitter_impact(root_path, symbol)
+            ts.notes.append(f"LSP error, used fallback: {exc}")
+            cache.set(cache_key, ts.to_dict())
+            return ts
+
+    result = _treesitter_impact(root_path, symbol)
+    cache.set(cache_key, result.to_dict())
+    return result

sin_code_bundle/lsp_bootstrap.py

@@ -0,0 +1,85 @@
+"""Detect repo languages and ensure the matching language servers are present.
+
+`sin doctor` uses this to tell users exactly what to install for accurate
+impact analysis. We never silently install global tooling; we report and offer
+the exact install command.
+
+Docs: lsp_bootstrap.doc.md
+"""
+
+from __future__ import annotations
+
+import shutil
+from collections import Counter
+from pathlib import Path
+
+# language -> (server binary, install hint)
+SERVERS: dict[str, tuple[str, str]] = {
+    "python": (
+        "pyright-langserver",
+        "npm i -g pyright   (or: pip install pyright)",
+    ),
+    "typescript": (
+        "typescript-language-server",
+        "npm i -g typescript typescript-language-server",
+    ),
+    "javascript": (
+        "typescript-language-server",
+        "npm i -g typescript typescript-language-server",
+    ),
+    "go": (
+        "gopls",
+        "go install golang.org/x/tools/gopls@latest",
+    ),
+    "rust": (
+        "rust-analyzer",
+        "rustup component add rust-analyzer",
+    ),
+    "java": (
+        "jdtls",
+        "see: https://github.com/eclipse-jdtls/eclipse.jdt.ls",
+    ),
+}
+
+_EXT_LANG: dict[str, str] = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+}
+_IGNORE = {".git", "node_modules", ".venv", "__pycache__", ".sin"}
+
+
+def detect_languages(root: Path) -> list[tuple[str, int]]:
+    """Return (language, file_count) pairs, most frequent first."""
+    counter: Counter[str] = Counter()
+    for p in root.rglob("*"):
+        if not p.is_file() or any(part in _IGNORE for part in p.parts):
+            continue
+        lang = _EXT_LANG.get(p.suffix.lower())
+        if lang:
+            counter[lang] += 1
+    return counter.most_common()
+
+
+def server_status(root: Path) -> list[dict]:
+    """Return a list of dicts with language server availability info."""
+    rows: list[dict] = []
+    for lang, count in detect_languages(root):
+        entry = SERVERS.get(lang)
+        binary, hint = entry if entry else (None, "no LSP integration yet")
+        installed = bool(binary and shutil.which(binary))
+        rows.append(
+            {
+                "language": lang,
+                "files": count,
+                "server": binary,
+                "installed": installed,
+                "install_hint": hint,
+            }
+        )
+    return rows

sin_code_bundle/markitdown.py

@@ -0,0 +1,254 @@
+# SPDX-License-Identifier: MIT
+"""MarkItDown bridge.
+
+MarkItDown (https://github.com/microsoft/markitdown) is an *upstream* tool by
+Microsoft, distributed as the MIT-licensed PyPI packages ``markitdown`` (CLI /
+library) and ``markitdown-mcp`` (an MCP server). We never vendor or copy its
+source; we only invoke the published packages. This keeps the bundle
+MIT-licensed while giving coder agents a first-class way to turn binary and
+office documents (PDF, DOCX, PPTX, XLSX, images, audio, HTML, ...) into
+LLM-friendly Markdown.
+
+The bridge provides:
+  * discovery / health checks for the ``markitdown-mcp`` runner and the
+    ``markitdown`` CLI,
+  * a thin ``convert`` wrapper over the ``markitdown`` CLI,
+  * MCP wiring so OpenCode / Codex / Hermes each get the MarkItDown MCP server,
+    mirroring upstream's recommended ``uvx markitdown-mcp`` invocation.
+
+Docs: markitdown.doc.md
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+# ── MarkItDown Bridge: Document → Markdown ────────────────────────────
+# Microsoft MarkItDown is the upstream package. We never vendor it; the
+# bridge only discovers the published `markitdown-mcp` server and the
+# `markitdown` CLI and shells out to them. This keeps the bundle MIT and
+# lets us pick up upstream format support (PDF, DOCX, PPTX, XLSX, images
+# with OCR, audio transcription, HTML, CSV/JSON/XML, ZIP, EPUB, etc.)
+# without re-implementing any of it.
+
+# MarkItDown exposes its MCP server through the ``markitdown-mcp`` package.
+# Upstream recommends running it via ``uvx`` so it is fetched/cached on demand;
+# we fall back to a directly-installed ``markitdown-mcp`` executable.
+MARKITDOWN_MCP_PACKAGE = "markitdown-mcp"
+MARKITDOWN_CLI = "markitdown"
+
+
+class MarkItDownError(RuntimeError):
+    """Raised when MarkItDown is unavailable or a command fails."""
+
+
+@dataclass
+class MarkItDownEnv:
+    """Resolved runtime environment for invoking MarkItDown."""
+
+    uvx: str | None
+    mcp_exe: str | None
+    cli: str | None
+
+    @property
+    def mcp_available(self) -> bool:
+        """True iff either ``uvx`` or a directly-installed ``markitdown-mcp`` binary is on PATH."""
+        return bool(self.uvx or self.mcp_exe)
+
+    @property
+    def cli_available(self) -> bool:
+        """True iff the ``markitdown`` CLI (the converter) is on PATH."""
+        return bool(self.cli)
+
+    def mcp_command(self) -> dict[str, Any]:
+        """Return the MCP launch command, preferring ``uvx``."""
+        if self.uvx:
+            return {"command": "uvx", "args": [MARKITDOWN_MCP_PACKAGE]}
+        if self.mcp_exe:
+            return {"command": MARKITDOWN_MCP_PACKAGE, "args": []}
+        raise MarkItDownError(
+            "MarkItDown MCP server not found. Install it with "
+            "`pip install markitdown-mcp` (or `uv tool install markitdown-mcp`). "
+            "The bundle does not vendor MarkItDown."
+        )
+
+    def cli_cmd(self) -> str:
+        """Return the absolute path of the ``markitdown`` CLI, or raise.
+
+        Used by ``convert()`` to shell out for one-shot document→markdown
+        conversion without spinning up the long-lived MCP server.
+        """
+        if not self.cli:
+            raise MarkItDownError(
+                "`markitdown` CLI not found. Install with `pip install 'markitdown[all]'`."
+            )
+        return self.cli
+
+
+def detect_env() -> MarkItDownEnv:
+    """Probe PATH for ``uvx``, ``markitdown-mcp``, and ``markitdown`` (no I/O beyond that)."""
+    return MarkItDownEnv(
+        uvx=shutil.which("uvx"),
+        mcp_exe=shutil.which(MARKITDOWN_MCP_PACKAGE),
+        cli=shutil.which(MARKITDOWN_CLI),
+    )
+
+
+def mcp_server_command(env: MarkItDownEnv | None = None) -> dict[str, Any]:
+    """Resolve the MCP server launch command (``uvx markitdown-mcp`` by default)."""
+    env = env or detect_env()
+    return env.mcp_command()
+
+
+# ── Local-Only Safety: File Access Guard ─────────────────────────────
+# `convert()` is the only public surface that touches a file path. It
+# deliberately refuses anything that is not a regular file on the local
+# filesystem — we never want an MCP client (potentially remote / hostile)
+# to coerce us into passing an http:// or pipe:// URL into MarkItDown's
+# CLI, which would expand the attack surface considerably.
+
+
+def convert(
+    path: str, env: MarkItDownEnv | None = None, timeout: int = 300
+) -> str:  # 300s = 5min; large PDFs / pptx with embedded media can be slow on first pass
+    """Convert a document to Markdown using the upstream ``markitdown`` CLI."""
+    env = env or detect_env()
+    cli = env.cli_cmd()
+    src = Path(path)
+    # `is_file()` (not `exists()`) — guards against directories and broken
+    # symlinks, both of which the CLI would otherwise try to read as content.
+    if not src.is_file():
+        raise MarkItDownError(f"File not found: {path}")
+    try:
+        proc = subprocess.run(
+            [cli, str(src)],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired as exc:  # pragma: no cover - timing dependent
+        raise MarkItDownError(f"markitdown timed out after {timeout}s") from exc
+    if proc.returncode != 0:
+        raise MarkItDownError(f"markitdown failed ({proc.returncode}): {proc.stderr.strip()}")
+    return proc.stdout
+
+
+def doctor() -> dict[str, Any]:
+    """Report MarkItDown availability for diagnostics."""
+    env = detect_env()
+    return {
+        "mcp_available": env.mcp_available,
+        "cli_available": env.cli_available,
+        "runner": "uvx" if env.uvx else (MARKITDOWN_MCP_PACKAGE if env.mcp_exe else None),
+        "mcp_package": MARKITDOWN_MCP_PACKAGE,
+    }
+
+
+# ── OpenCode Integration: File Watcher Hooks ──────────────────────────
+# Below: per-agent MCP config writers. These mutate well-known files
+# under the user's home directory:
+#   * OpenCode: ~/.config/opencode/opencode.json  (JSON, mcp.<name>)
+#   * Codex:    ~/.codex/config.toml              (TOML, [mcp_servers.<name>])
+#   * Hermes:   ~/.hermes/mcp.json                (JSON, mcpServers.<name>)
+# We DO NOT touch plugin/hook files for the agents — MarkItDown integrates
+# through MCP, the same surface as GitNexus, so behaviour is uniform.
+
+
+# ── MCP Wiring (mirrors the GitNexus bridge) ──────────────────────────────
+def _opencode_config_path() -> Path:
+    return Path.home() / ".config" / "opencode" / "opencode.json"
+
+
+def _codex_config_path() -> Path:
+    return Path.home() / ".codex" / "config.toml"
+
+
+def _hermes_config_path() -> Path:
+    return Path.home() / ".hermes" / "mcp.json"
+
+
+AGENTS = ("opencode", "codex", "hermes")
+
+
+def _launch(env: MarkItDownEnv | None) -> tuple[str, list[str]]:
+    cmd = mcp_server_command(env)
+    return cmd["command"], cmd["args"]
+
+
+def _wire_opencode(env: MarkItDownEnv | None) -> str:
+    command, args = _launch(env)
+    path = _opencode_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data: dict[str, Any] = {}
+    if path.is_file():
+        try:
+            data = json.loads(path.read_text() or "{}")
+        except json.JSONDecodeError:
+            data = {}
+    mcp = data.setdefault("mcp", {})
+    mcp["markitdown"] = {
+        "type": "local",
+        "command": [command, *args],
+        "enabled": True,
+    }
+    path.write_text(json.dumps(data, indent=2) + "\n")
+    return str(path)
+
+
+def _wire_codex(env: MarkItDownEnv | None) -> str:
+    command, args = _launch(env)
+    path = _codex_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    args_repr = ", ".join(f'"{a}"' for a in args)
+    block = f'\n[mcp_servers.markitdown]\ncommand = "{command}"\nargs = [{args_repr}]\n'
+    existing = path.read_text() if path.is_file() else ""
+    if "[mcp_servers.markitdown]" in existing:
+        return str(path)  # already wired; leave user edits intact
+    path.write_text(existing + block)
+    return str(path)
+
+
+def _wire_hermes(env: MarkItDownEnv | None) -> str:
+    command, args = _launch(env)
+    path = _hermes_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data: dict[str, Any] = {}
+    if path.is_file():
+        try:
+            data = json.loads(path.read_text() or "{}")
+        except json.JSONDecodeError:
+            data = {}
+    servers = data.setdefault("mcpServers", {})
+    servers["markitdown"] = {"command": command, "args": args}
+    path.write_text(json.dumps(data, indent=2) + "\n")
+    return str(path)
+
+
+_WIRERS = {
+    "opencode": _wire_opencode,
+    "codex": _wire_codex,
+    "hermes": _wire_hermes,
+}
+
+
+def setup_agents(
+    agents: list[str] | None = None,
+    env: MarkItDownEnv | None = None,
+) -> dict[str, str]:
+    """Wire the MarkItDown MCP server into each agent's config.
+
+    Returns a mapping of agent -> config file written.
+    """
+    chosen = agents or list(AGENTS)
+    written: dict[str, str] = {}
+    for agent in chosen:
+        wirer = _WIRERS.get(agent)
+        if not wirer:
+            raise MarkItDownError(f"Unknown agent: {agent!r}. Known: {', '.join(AGENTS)}")
+        written[agent] = wirer(env)
+    return written