witan-code 0.2.0__py3-none-any.whl

witan_code/maintenance.py

@@ -0,0 +1,130 @@
+"""Store compaction (omnigraph optimize/cleanup) with throttling.
+
+Mirrors witan's own ``witan.maintenance`` (deliberately duplicated — no
+cross-package import, see ``graph.py``'s docstring), adapted for witan-code's
+per-repo, multi-store layout: there is no single configured store to
+throttle. The current repo's per-repo store and the shared cross-repo bridge
+store are each compacted and throttled independently, keyed by the store's
+own path, so a bloated bridge store doesn't gate (or get gated by) a repo
+store's compaction.
+
+Every witan-code write — the ``PostToolUse`` single-file reindex, the
+``SessionStart`` full index — appends a new tiny Lance fragment + manifest
+version to a store, and left uncompacted it bloats until *opening* the store
+dominates query latency, the same failure mode witan's own store hit (#98).
+``omnigraph optimize`` collapses the fragments (non-destructive); ``cleanup``
+GCs old versions to reclaim disk (destructive).
+
+The ``Stop`` hook (``witan-code checkpoint``) calls
+:func:`spawn_background_optimize` for whichever stores exist in the current
+repo — at most once per interval each, detached so the hook returns
+immediately. There is also a ``witan-code optimize`` / ``witan-code cleanup``
+CLI for cron/systemd-timer driven maintenance.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+from ._detach import popen_detached
+
+# Opportunistic optimize runs at most once per this window per store. Optimize
+# takes the store's write lock and is ~tens of seconds on a bloated store, so
+# daily is a safe default; override with WITAN_CODE_OPTIMIZE_INTERVAL (seconds;
+# 0 disables).
+_OPTIMIZE_INTERVAL = 24 * 3600.0
+
+_REMOTE_PREFIXES = ("http://", "https://", "s3://")
+
+
+def optimize_interval() -> float:
+    """Throttle window in seconds; ``0`` (or negative) disables auto-optimize."""
+    raw = os.environ.get("WITAN_CODE_OPTIMIZE_INTERVAL")
+    if raw is None:
+        return _OPTIMIZE_INTERVAL
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        return _OPTIMIZE_INTERVAL
+
+
+def _stamp_file(store: str | Path) -> Path:
+    digest = hashlib.sha256(str(store).encode()).hexdigest()[:16]
+    return Path(tempfile.gettempdir()) / f"witan-code-optimize-{digest}.json"
+
+
+def _last_run(store: str | Path) -> float:
+    try:
+        return float(json.loads(_stamp_file(store).read_text()).get("stamp", 0.0))
+    except Exception:  # noqa: BLE001 — missing/corrupt stamp → treat as never run
+        return 0.0
+
+
+def _mark_run(store: str | Path, when: float) -> None:
+    """Record the last-optimize time atomically.
+
+    Concurrent Stop hooks (or an interrupted write) could otherwise leave a
+    half-written stamp that ``_last_run`` reads as "never run", defeating the
+    throttle and letting optimize spawn repeatedly. Write a process-unique temp
+    file and ``os.replace`` it in, so a reader always sees a complete file.
+    """
+    path = _stamp_file(store)
+    tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+    try:
+        tmp.write_text(json.dumps({"stamp": when}))
+        os.replace(tmp, path)
+    except OSError:
+        try:
+            tmp.unlink(missing_ok=True)
+        except OSError:
+            pass
+
+
+def due(store: str | Path, now: float | None = None) -> bool:
+    """Whether an opportunistic optimize is due for ``store``.
+
+    False when auto-optimize is disabled, the store is remote (maintained
+    server-side, not by a client hook), the store doesn't exist yet, or the
+    throttle window hasn't elapsed.
+    """
+    interval = optimize_interval()
+    if interval <= 0:
+        return False
+    if str(store).startswith(_REMOTE_PREFIXES):
+        return False
+    if not Path(store).exists():
+        return False
+    now = time.time() if now is None else now
+    return now - _last_run(store) >= interval
+
+
+def spawn_background_optimize(store: str | Path, now: float | None = None) -> bool:
+    """If an optimize is due for ``store``, detach one and return ``True``.
+
+    Best-effort and non-blocking: the throttle stamp is written *before*
+    spawning (so a failing optimize can't hot-loop every session), and the
+    child is fully detached (its own session, no inherited stdio) so it
+    outlives the Stop hook. Never raises — a maintenance failure must not fail
+    the hook.
+    """
+    if not due(store, now):
+        return False
+    _mark_run(store, time.time() if now is None else now)
+    try:
+        popen_detached(
+            [sys.executable, "-m", "witan_code", "optimize", "--store", str(store)],
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            env=dict(os.environ),
+        )
+        return True
+    except OSError:
+        return False

witan_code/package_map.py

@@ -0,0 +1,74 @@
+"""Package map: a repo's declared canonical package identity.
+
+Read from ``witan-code.toml`` at the repo root (see docs/PACKAGE_MAP.md).
+The identity qualifies provider symbol strings and feeds the
+``known_provider_package`` confidence heuristic. Repos without the file get a
+fallback identity derived from the repo URI so provider symbols always carry a
+package qualifier.
+"""
+
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+PACKAGE_MAP_FILENAME = "witan-code.toml"
+
+# SCIP empty-field convention (see docs/SYMBOL_FORMAT.md).
+EMPTY = "."
+
+
+@dataclass(frozen=True)
+class PackageIdentity:
+    name: str
+    manager: str = EMPTY
+    version: str = "main"
+    provides: tuple[str, ...] = field(default_factory=tuple)
+    """Extra published identities as ``"manager:name"`` strings."""
+
+    declared: bool = False
+    """True when read from witan-code.toml, False for a URI-derived fallback."""
+
+    def provided_names(self) -> frozenset[str]:
+        """Bare package names this repo provides (primary name + provides)."""
+        names = {self.name}
+        for entry in self.provides:
+            _, _, name = entry.partition(":")
+            names.add(name or entry)
+        return frozenset(names)
+
+
+def fallback_identity(repo: str) -> PackageIdentity:
+    """Identity derived from the canonical repo URI when no TOML exists."""
+    name = repo.rstrip("/").rsplit("/", 1)[-1] or repo
+    return PackageIdentity(name=name)
+
+
+def load(repo_root: Path, repo: str) -> PackageIdentity:
+    """Load ``witan-code.toml`` from ``repo_root``, else the fallback identity.
+
+    Malformed TOML or a missing ``[package].name`` degrades to the fallback —
+    the package map must never make indexing fail.
+    """
+    path = repo_root / PACKAGE_MAP_FILENAME
+    if not path.is_file():
+        return fallback_identity(repo)
+    try:
+        data = tomllib.loads(path.read_text())
+    except (tomllib.TOMLDecodeError, OSError, UnicodeDecodeError):
+        return fallback_identity(repo)
+
+    pkg = data.get("package")
+    if not isinstance(pkg, dict) or not isinstance(pkg.get("name"), str):
+        return fallback_identity(repo)
+
+    provides_raw = pkg.get("provides", ())
+    if not isinstance(provides_raw, (list, tuple)):
+        provides_raw = ()
+    provides = tuple(entry for entry in provides_raw if isinstance(entry, str))
+    return PackageIdentity(
+        name=pkg["name"],
+        manager=str(pkg.get("manager", EMPTY)),
+        version=str(pkg.get("version", "main")),
+        provides=provides,
+        declared=True,
+    )

witan_code/queries/bridge.gq

@@ -0,0 +1,193 @@
+// Cross-Repo Context Bridge — read / delete / search queries (_bridge.omni).
+//
+// Linkages are NOT edges: they are computed by grouping InterfaceBinding rows on
+// (kind, key_norm) in Python. These queries just filter + search the flat node.
+// Read filters use the match-shorthand `{ field: $param }` only (standalone
+// `where`/quoted-key parse-errors in reads); deletes use `where`.
+
+// ── Deletes (reindex cleanup) ─────────────────────────────────────
+// Per-file purge: drop one source file's bindings so a reindex only replaces
+// what it re-extracted (an incremental full-repo index skips unchanged files,
+// so a repo-wide purge would orphan their bindings). Keyed on the composite
+// repo_file ("repo|file") because delete `where` takes a single condition
+// (no compound `and`).
+
+query delete_bindings_in_file($repo_file: String) {
+    delete InterfaceBinding where repo_file = $repo_file
+}
+
+// All bindings — read once to build the cross-repo dependency visualization.
+
+query all_bindings() {
+    match {
+        $b: InterfaceBinding
+    }
+    return {
+        $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.framework, $b.generic, $b.confidence,
+        $b.symbol, $b.line
+    }
+    limit 1000000
+}
+
+// ── Per-repo symbol table (Stage 1 — docs/SYMBOL_TABLE.md) ────────
+// The table is rebuilt exactly on every bridge write: delete the repo's rows,
+// re-aggregate from surviving + fresh bindings. Delete is keyed on the single
+// repo field (delete `where` takes one condition).
+
+query delete_repo_symbols($repo: String) {
+    delete RepoSymbol where repo = $repo
+}
+
+query repo_symbols($repo: String) {
+    match {
+        $s: RepoSymbol { repo: $repo }
+    }
+    return {
+        $s.slug, $s.repo, $s.role, $s.symbol, $s.scheme, $s.descriptor,
+        $s.key_norm, $s.manager, $s.package, $s.version, $s.kind,
+        $s.n_refs, $s.confidence, $s.file, $s.line
+    }
+    order { $s.symbol asc }
+    limit 100000
+}
+
+// Every RepoSymbol row across all repos — Stage 2 (witan_code.stitch) reads
+// this once and joins external/exported rows in Python, the same "bulk read,
+// group in Python" pattern all_bindings/build_graph already use for the
+// heuristic tier.
+
+query all_repo_symbols() {
+    match {
+        $s: RepoSymbol
+    }
+    return {
+        $s.repo, $s.role, $s.symbol, $s.scheme, $s.descriptor,
+        $s.key_norm, $s.manager, $s.package, $s.version, $s.kind,
+        $s.n_refs, $s.confidence, $s.file, $s.line
+    }
+    limit 1000000
+}
+
+// Stage-2 join primitives: exact descriptor match (env/svc), and the key_norm
+// match (http/pkg). pkg canonical descriptors are always "." (SYMBOL_FORMAT.md
+// — package identity lives in the manager/package fields, not the
+// descriptor), so symbols_by_descriptor cannot distinguish packages;
+// key_norm carries the package name for both exported and external rows
+// instead. http also joins on key_norm rather than descriptor because the
+// descriptor embeds the method, which consumers usually can't determine
+// statically ("*").
+
+query symbols_by_descriptor($scheme: String, $descriptor: String) {
+    match {
+        $s: RepoSymbol { scheme: $scheme, descriptor: $descriptor }
+    }
+    return {
+        $s.slug, $s.repo, $s.role, $s.symbol, $s.scheme, $s.descriptor,
+        $s.key_norm, $s.manager, $s.package, $s.version, $s.kind,
+        $s.n_refs, $s.confidence, $s.file, $s.line
+    }
+    order { $s.repo asc }
+    limit 1000
+}
+
+query symbols_by_key($scheme: String, $key_norm: String) {
+    match {
+        $s: RepoSymbol { scheme: $scheme, key_norm: $key_norm }
+    }
+    return {
+        $s.slug, $s.repo, $s.role, $s.symbol, $s.scheme, $s.descriptor,
+        $s.key_norm, $s.manager, $s.package, $s.version, $s.kind,
+        $s.n_refs, $s.confidence, $s.file, $s.line
+    }
+    order { $s.repo asc }
+    limit 1000
+}
+
+// ── Package maps ──────────────────────────────────────────────────
+// Every repo's declared/fallback package identity (docs/PACKAGE_MAP.md).
+// Read at bridge-write time to qualify symbols and feed the
+// known_provider_package heuristic.
+
+query all_package_maps() {
+    match {
+        $p: PackageMap
+    }
+    return {
+        $p.repo, $p.name, $p.manager, $p.version, $p.provides, $p.declared
+    }
+    limit 10000
+}
+
+// ── Grouping reads ────────────────────────────────────────────────
+// All bindings for a contract key (the workhorse behind providers / consumers /
+// impact grouping).
+
+query bindings_by_key($kind: String, $key_norm: String) {
+    match {
+        $b: InterfaceBinding { kind: $kind, key_norm: $key_norm }
+    }
+    return {
+        $b.slug, $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.line, $b.language, $b.framework, $b.generic,
+        $b.symbol
+    }
+    order { $b.repo asc }
+    limit 500
+}
+
+// Same filtered to one role — backs code_interface_providers / _consumers.
+
+query bindings_by_key_role($kind: String, $key_norm: String, $role: String) {
+    match {
+        $b: InterfaceBinding { kind: $kind, key_norm: $key_norm, role: $role }
+    }
+    return {
+        $b.slug, $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.line, $b.language, $b.framework, $b.generic,
+        $b.symbol
+    }
+    order { $b.repo asc }
+    limit 500
+}
+
+// Bindings attributed to a symbol — first half of code_cross_repo_impact.
+
+query bindings_for_symbol($symbol_id: String) {
+    match {
+        $b: InterfaceBinding { symbol_id: $symbol_id }
+    }
+    return {
+        $b.slug, $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.generic
+    }
+    limit 500
+}
+
+// ── Full-text search (BM25 on key_norm) ───────────────────────────
+
+query search_bindings($query: String) {
+    match {
+        $b: InterfaceBinding
+        search($b.key_norm, $query)
+    }
+    return {
+        $b.slug, $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.framework, $b.generic
+    }
+    order { bm25($b.key_norm, $query) desc }
+    limit 100
+}
+
+query search_bindings_by_kind($query: String, $kind: String) {
+    match {
+        $b: InterfaceBinding { kind: $kind }
+        search($b.key_norm, $query)
+    }
+    return {
+        $b.slug, $b.kind, $b.key, $b.key_norm, $b.role, $b.repo,
+        $b.file, $b.symbol_id, $b.framework, $b.generic
+    }
+    order { bm25($b.key_norm, $query) desc }
+    limit 100
+}

witan_code/queries/code_mutations.gq

@@ -0,0 +1,81 @@
+// Code Graph — insert/update mutations.
+//
+// NOTE: Omnigraph cannot mix deletes with inserts/updates in one query.
+// Deletes for reindexing live in delete.gq and are run as separate change()
+// calls BEFORE these inserts.
+
+// ── CodeFile ──────────────────────────────────────────────────────
+
+query insert_file(
+    $id: String,
+    $repo: String,
+    $path: String,
+    $language: String,
+    $content_hash: String,
+    $indexed_at: DateTime
+) {
+    insert CodeFile {
+        slug: $id,
+        repo: $repo,
+        path: $path,
+        language: $language,
+        content_hash: $content_hash,
+        indexed_at: $indexed_at
+    }
+}
+
+// ── Symbol ────────────────────────────────────────────────────────
+
+query insert_symbol(
+    $id: String,
+    $repo: String,
+    $file_id: String,
+    $name: String,
+    $qualified_name: String,
+    $kind: String,
+    $start_line: I32,
+    $end_line: I32,
+    $signature: String?,
+    $docstring: String?,
+    $indexed_at: DateTime
+) {
+    insert Symbol {
+        slug: $id,
+        repo: $repo,
+        file_id: $file_id,
+        name: $name,
+        qualified_name: $qualified_name,
+        kind: $kind,
+        start_line: $start_line,
+        end_line: $end_line,
+        signature: $signature,
+        docstring: $docstring,
+        indexed_at: $indexed_at
+    }
+}
+
+// ── Edges ─────────────────────────────────────────────────────────
+
+query link_defines($from: String, $to: String) {
+    insert Defines { from: $from, to: $to }
+}
+
+query link_contains($from: String, $to: String) {
+    insert Contains { from: $from, to: $to }
+}
+
+query link_calls($from: String, $to: String) {
+    insert Calls { from: $from, to: $to }
+}
+
+query link_references($from: String, $to: String) {
+    insert References { from: $from, to: $to }
+}
+
+query link_imports($from: String, $to: String) {
+    insert Imports { from: $from, to: $to }
+}
+
+query link_inherits($from: String, $to: String) {
+    insert Inherits { from: $from, to: $to }
+}

witan_code/queries/code_read.gq

@@ -0,0 +1,154 @@
+// Code Graph — read queries.
+//
+// Dispatch strategy mirrors witan: the MCP server picks the named
+// query based on the call. Symbol ids are `<repo>#<path/to/file.py>::<Name>`.
+// Queries project the id as `slug`; the server renames it to `symbol_id`
+// (see server._as_symbol_id) so it round-trips under the tools' param name.
+
+// ── Definition lookup ─────────────────────────────────────────────
+
+query find_by_name($name: String) {
+    match {
+        $s: Symbol { name: $name }
+    }
+    return {
+        $s.slug, $s.repo, $s.file_id, $s.name, $s.qualified_name,
+        $s.kind, $s.start_line, $s.end_line, $s.signature, $s.docstring,
+        $s.decorators
+    }
+    order { $s.qualified_name asc }
+    limit 50
+}
+
+query find_by_qualified_name($qualified_name: String) {
+    match {
+        $s: Symbol { qualified_name: $qualified_name }
+    }
+    return {
+        $s.slug, $s.repo, $s.file_id, $s.name, $s.qualified_name,
+        $s.kind, $s.start_line, $s.end_line, $s.signature, $s.docstring,
+        $s.decorators
+    }
+    order { $s.qualified_name asc }
+    limit 50
+}
+
+query get_symbol($id: String) {
+    match {
+        $s: Symbol { slug: $id }
+    }
+    return {
+        $s.slug, $s.repo, $s.file_id, $s.name, $s.qualified_name,
+        $s.kind, $s.start_line, $s.end_line, $s.signature, $s.docstring,
+        $s.decorators
+    }
+}
+
+// ── Full-text symbol search (BM25 on name / qualified_name) ────────
+
+query search_symbols($query: String) {
+    match {
+        $s: Symbol
+        search($s.qualified_name, $query)
+    }
+    return {
+        $s.slug, $s.repo, $s.file_id, $s.name, $s.qualified_name,
+        $s.kind, $s.start_line, $s.end_line, $s.signature
+    }
+    order { bm25($s.qualified_name, $query) desc }
+    limit 50
+}
+
+query search_symbols_by_kind($query: String, $kind: String) {
+    match {
+        $s: Symbol { kind: $kind }
+        search($s.qualified_name, $query)
+    }
+    return {
+        $s.slug, $s.repo, $s.file_id, $s.name, $s.qualified_name,
+        $s.kind, $s.start_line, $s.end_line, $s.signature
+    }
+    order { bm25($s.qualified_name, $query) desc }
+    limit 50
+}
+
+// ── File symbols ──────────────────────────────────────────────────
+
+query symbols_in_file($file_id: String) {
+    match {
+        $s: Symbol { file_id: $file_id }
+    }
+    return {
+        $s.slug, $s.name, $s.qualified_name, $s.kind,
+        $s.start_line, $s.end_line, $s.signature, $s.docstring, $s.decorators
+    }
+    order { $s.start_line asc }
+}
+
+// ── Edge traversal ────────────────────────────────────────────────
+// Incoming Calls: symbols that call $id.
+
+query callers($id: String) {
+    match {
+        $target: Symbol { slug: $id }
+        $caller: Symbol
+        $caller calls $target
+    }
+    return {
+        $caller.slug, $caller.repo, $caller.file_id, $caller.name,
+        $caller.qualified_name, $caller.kind,
+        $caller.start_line, $caller.end_line
+    }
+    order { $caller.qualified_name asc }
+    limit 200
+}
+
+// Incoming Calls + References: anything that uses $id.
+
+query referencers($id: String) {
+    match {
+        $target: Symbol { slug: $id }
+        $src: Symbol
+        $src references $target
+    }
+    return {
+        $src.slug, $src.repo, $src.file_id, $src.name,
+        $src.qualified_name, $src.kind,
+        $src.start_line, $src.end_line
+    }
+    order { $src.qualified_name asc }
+    limit 200
+}
+
+// ── CodeFile fetch (for incremental reindex) ──────────────────────
+
+query get_file($id: String) {
+    match {
+        $f: CodeFile { slug: $id }
+    }
+    return {
+        $f.slug, $f.repo, $f.path, $f.language,
+        $f.content_hash, $f.indexed_at
+    }
+}
+
+// All file hashes — read once per index run so the incremental skip check is
+// in-memory rather than a query per file.
+
+query all_file_hashes() {
+    match {
+        $f: CodeFile
+    }
+    return { $f.slug, $f.content_hash }
+    limit 1000000
+}
+
+// Symbol ids defined by a file — needed to delete their outgoing edges
+// before reindexing the file.
+
+query symbol_ids_in_file($file_id: String) {
+    match {
+        $s: Symbol { file_id: $file_id }
+    }
+    return { $s.slug }
+}

witan_code/queries/delete.gq

@@ -0,0 +1,16 @@
+// Code Graph — delete queries (reindex path).
+//
+// Omnigraph cannot mix deletes with inserts/updates in one query, so these
+// run as their own change() calls BEFORE re-inserting a file's data.
+//
+// Edges are deleted by deleting their endpoint nodes: removing a Symbol or
+// CodeFile removes its incident edges. We delete all Symbols belonging to a
+// file (by file_id) and then the CodeFile itself.
+
+query delete_symbols_in_file($file_id: String) {
+    delete Symbol where file_id = $file_id
+}
+
+query delete_file($id: String) {
+    delete CodeFile where slug = $id
+}

witan_code/queries_ts/bash.scm

@@ -0,0 +1,9 @@
+; Bash / Zsh symbol extraction. The tree-sitter bash grammar covers most zsh.
+
+; function foo() { ... }  and  foo() { ... }
+(function_definition
+  name: (word) @symbol.function)
+
+; calls — best-effort Calls/References edges to other functions.
+(command
+  name: (command_name (word) @call.name))

witan_code/queries_ts/hcl.scm

@@ -0,0 +1,16 @@
+; HCL symbol extraction (Terraform / Packer / Vault-policy style configs).
+;
+; Each labeled block (resource/variable/output/module/data/source/path, …)
+; becomes a Symbol keyed on its LAST label — the part that's actually unique
+; (`resource "aws_instance" "foo"` -> foo, not the shared "aws_instance" type;
+; `variable "app_name"` -> app_name; Vault's `path "secret/data/*"` -> the
+; path itself). The `.` anchor requires the label be immediately followed by
+; the opening brace, which is only true of the last label when there are two.
+;
+; Unlabeled blocks (locals, bare provider blocks) aren't captured — there's
+; no unique text to key them on.
+
+(block
+  (string_lit) @symbol.block
+  .
+  (block_start))