graphify-mesh 0.0.22__py3-none-any.whl

graphify_mesh/server/context_pack.py

@@ -0,0 +1,212 @@
+"""`context_pack(goal, scope, token_budget)` tool (WS5 tool 5): evidence
+cards built from `retrieval.rank`'s ranked hits, each carrying an explicit
+`[repo:path:line]` citation (same shape as everywhere else in this
+package), a snippet, and a confidence flag. Truncates strictly by whole-card
+boundary — a card is either fully included or fully excluded, it is never
+sliced mid-way (plan WS5 tool contract: "budgets").
+
+Confidence flag is always `ranking.CONFIDENCE_EXTRACTED` here:
+`context_pack` does not expose an `include_inferred` toggle the way
+`rank()` itself does, so every hit this function can ever surface is, by
+construction, drawn from the EXTRACTED-only default structural traversal
+plus lexical/vector matches — there is no code path by which a
+non-EXTRACTED (INFERRED/AMBIGUOUS) sourced hit reaches this function today. The field is still
+carried explicitly (not hardcoded as a bare string) so a future
+`include_inferred` passthrough only needs to plumb the real value through,
+not invent the field.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from graphify_mesh.server import ranking
+from graphify_mesh.server.retrieval import EmbedQueryFn, Hit, rank
+from graphify_mesh.server.scope import RegistryEntry
+from graphify_mesh.server.store import Generation
+from graphify_mesh.sync.embedding import build_snippet, node_line
+
+# Rough, dependency-free token estimate — no tokenizer dependency added just
+# for a budget heuristic. ~4 chars/token is the commonly-cited English-text
+# average; deliberately errs on the side of UNDER-counting cards (budget
+# runs out slightly early rather than ever overshooting the caller's stated
+# token_budget).
+CHARS_PER_TOKEN_ESTIMATE = 4
+
+# How many ranked candidates to consider building cards from before token
+# budget truncation. Independent of the caller's k (context_pack has no k
+# parameter) — kept generous since truncation is budget-driven, not count-driven.
+CONTEXT_PACK_CANDIDATE_K = 30
+
+# Snippets are read from the live working tree at query time (see
+# `_card_from_hit`), not from a snapshot taken at sync time — the citation
+# line number comes from the generation's graph data, so edits made after
+# the last sync can silently shift the cited line. Every card therefore
+# declares its snippet provenance explicitly, and is flagged stale when the
+# source file's mtime is newer than the generation manifest's `created_at`.
+SNIPPET_SOURCE_LIVE_TREE = "live_tree"
+
+# The generation manifest's `created_at` format (written by
+# `sync/pipeline.py` via `time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())`).
+_MANIFEST_CREATED_AT_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+@dataclass
+class EvidenceCard:
+    citation: str  # "[repo:path:line]"
+    repo: str
+    label: str
+    source_file: str
+    line: int | None
+    confidence: str
+    snippet: str
+    score: float
+    # Snippet provenance: snippets always come from the live working tree,
+    # never from generation data — declared explicitly so clients can tell
+    # the snippet's freshness class apart from the citation's (which IS
+    # generation data). `snippet_stale=True` marks the degraded case where
+    # the source file changed after the generation was built, so the cited
+    # line may no longer match the snippet content.
+    snippet_source: str = SNIPPET_SOURCE_LIVE_TREE
+    snippet_stale: bool = False
+
+    def estimated_tokens(self) -> int:
+        # citation + label are always present and cheap; snippet dominates
+        # the estimate. +1 keeps a zero-length snippet from costing 0 tokens.
+        return max(
+            1,
+            (len(self.snippet) + len(self.label) + len(self.citation)) // CHARS_PER_TOKEN_ESTIMATE,
+        )
+
+
+@dataclass
+class ContextPackResult:
+    goal: str
+    cards: list = field(default_factory=list)
+    truncated: bool = False
+    degraded: list = field(default_factory=list)
+
+
+def _root_for_repo(repo_id: str, registry_entries: list[RegistryEntry]) -> Path | None:
+    for entry in registry_entries:
+        if entry.repo_id == repo_id:
+            return entry.root
+    return None
+
+
+def _parse_manifest_timestamp(raw: object) -> float | None:
+    if not isinstance(raw, str):
+        return None
+    try:
+        parsed = datetime.strptime(raw, _MANIFEST_CREATED_AT_FORMAT)
+    except ValueError:
+        return None
+    return parsed.replace(tzinfo=UTC).timestamp()
+
+
+def _generation_created_at_epoch(generation: Generation) -> float | None:
+    """Staleness baseline (UTC epoch) for mtime comparison. Prefers the
+    manifest's `sync_started_at` (files edited DURING a long sync are newer
+    than sync start and correctly flagged), falling back to `created_at`
+    (publish stamp) for older generations. Returns `None` when absent or
+    malformed — staleness then simply cannot be determined and no card is
+    flagged."""
+    started = _parse_manifest_timestamp(generation.manifest.get("sync_started_at"))
+    if started is not None:
+        return started
+    return _parse_manifest_timestamp(generation.manifest.get("created_at"))
+
+
+def _snippet_file_mtime(root: Path | None, source_file: str | None) -> float | None:
+    """mtime of the live-tree file a snippet was read from, mirroring
+    `build_snippet`'s own path-safety gates (relative only, must resolve
+    inside `root`) so a hostile `source_file` can't be stat-probed outside
+    the repo root. Returns `None` whenever no snippet could have been read."""
+    if root is None or not source_file:
+        return None
+    if Path(source_file).is_absolute():
+        return None
+    path = (root / source_file).resolve()
+    try:
+        path.relative_to(root.resolve())
+    except ValueError:
+        return None
+    if not path.is_file():
+        return None
+    try:
+        return path.stat().st_mtime
+    except OSError:
+        return None
+
+
+def _card_from_hit(
+    hit: Hit,
+    generation: Generation,
+    registry_entries: list[RegistryEntry],
+    generation_created_at: float | None,
+) -> EvidenceCard:
+    node = generation.node_by_id.get(hit.node_id, {})
+    line = node_line(node)
+    root = _root_for_repo(hit.repo, registry_entries)
+    snippet = build_snippet(root, hit.source_file, line) if hit.source_file else ""
+
+    snippet_stale = False
+    # Staleness is checked whenever a source_file is cited, NOT gated on the
+    # snippet being non-empty: an empty snippet (cited line window beyond
+    # EOF, e.g. file truncated after the generation was built) is exactly
+    # the case where the citation is most likely invalid.
+    mtime = _snippet_file_mtime(root, hit.source_file) if hit.source_file else None
+    if generation_created_at is not None and mtime is not None and mtime > generation_created_at:
+        snippet_stale = True
+
+    line_part = line if line is not None else "?"
+    citation = f"[{hit.repo}:{hit.source_file}:{line_part}]"
+    return EvidenceCard(
+        citation=citation,
+        repo=hit.repo,
+        label=hit.label,
+        source_file=hit.source_file,
+        line=line,
+        confidence=ranking.CONFIDENCE_EXTRACTED,
+        snippet=snippet,
+        score=hit.score,
+        snippet_source=SNIPPET_SOURCE_LIVE_TREE,
+        snippet_stale=snippet_stale,
+    )
+
+
+def build_context_pack(
+    goal: str,
+    generation: Generation,
+    repo_filter: frozenset[str] | None,
+    registry_entries: list[RegistryEntry],
+    token_budget: int,
+    embed_query_fn: EmbedQueryFn,
+) -> ContextPackResult:
+    ranked = rank(goal, generation, repo_filter, CONTEXT_PACK_CANDIDATE_K, embed_query_fn)
+    generation_created_at = _generation_created_at_epoch(generation)
+
+    # Cards are built LAZILY inside the budget loop: each build costs a
+    # file open + read (`build_snippet`), so building all candidates
+    # eagerly wastes I/O whenever the budget keeps only the first few.
+    # Truncation stays whole-card and order-preserving: iterate hits in
+    # ranked order, stop at the first card that doesn't fit — a card is
+    # never split mid-way, and no card after the first non-fitting one is
+    # ever built.
+    selected: list[EvidenceCard] = []
+    remaining = token_budget
+    truncated = False
+    for hit in ranked.hits:
+        card = _card_from_hit(hit, generation, registry_entries, generation_created_at)
+        cost = card.estimated_tokens()
+        if cost > remaining:
+            truncated = True
+            break
+        selected.append(card)
+        remaining -= cost
+
+    return ContextPackResult(
+        goal=goal, cards=selected, truncated=truncated, degraded=ranked.degraded
+    )

graphify_mesh/server/embed_query.py

@@ -0,0 +1,206 @@
+"""Query-time embedding wrapper (WS5): turns a raw query string into a
+vector for `retrieval.vector_candidates`, reusing the exact `/api/embed`
+contract and base_url/model config `graphify_mesh.sync.embedding` /
+`graphify_mesh.sync.config` already established for the sync pipeline (C9) — the
+query MUST be embedded with the same model/endpoint that produced the
+published vectors, or cosine similarity is meaningless.
+
+Never raises: any transport/shape failure degrades to `None` (the caller,
+`retrieval.vector_candidates`, surfaces an `"embeddings_unavailable:<reason>"`
+marker in the response's `degraded` field), mirroring the pipeline's own
+"never crash on a down local service" convention (C9) — just applied at query
+time instead of build time.
+
+Cost of degrading (2026-09-23): a backend can accept TCP and answer its
+control-plane routes while every inference request hangs. Measured against
+the then-configured host: `/api/tags` 200 in 0.07 s, `/api/embed` timing out
+on 7 of 7 attempts. A per-query blocking call with a long timeout therefore
+costs the full timeout on EVERY query for as long as the backend stays in
+that state. Two guards below bound that cost:
+
+  * a short default timeout — a warm single-query embed measured 0.19-0.28 s
+    against a local backend, so the budget here is headroom, not a limit on
+    healthy operation;
+  * a negative cache — one failure suppresses the next `FAILURE_COOLDOWN`
+    seconds of calls in this process, so N degraded queries cost one timeout
+    per cooldown window instead of N timeouts.
+
+Neither guard is a health probe. A startup probe against `/api/tags` would
+have reported this backend HEALTHY (it answered 200 in 0.07 s while every
+embed hung), so the only honest signal is a real embed call's outcome, which
+is what the negative cache records.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+
+from graphify_mesh.sync.config import EMBED_DEFAULT_BASE_URL, EMBED_DEFAULT_MODEL
+from graphify_mesh.sync.embedding import embed_batch
+
+log = logging.getLogger("graphify_mesh.server.embed_query")
+
+# Kept short: a query-time embed call blocks one MCP `search`/`context_pack`
+# invocation, unlike the sync pipeline's bulk embed calls which can afford a
+# longer per-batch timeout. A healthy backend answers a single-input embed in
+# well under a second (0.19-0.28 s measured warm), so this is ~10x headroom;
+# what it actually bounds is the cost of a wedged backend.
+DEFAULT_QUERY_EMBED_TIMEOUT = 2.5
+# After a failure, skip the backend entirely for this long. Sized to be far
+# longer than the timeout (so the amortized cost of a dead backend is near
+# zero) but short enough that a backend coming back is picked up within a
+# minute without restarting the daemon.
+DEFAULT_FAILURE_COOLDOWN = 60.0
+
+# Kept as a module constant for callers/tests that imported it before the
+# env override existed.
+QUERY_EMBED_TIMEOUT = DEFAULT_QUERY_EMBED_TIMEOUT
+
+# Why the vector retriever contributed nothing. `retrieval` prefixes these
+# with `embeddings_unavailable:` so a client can tell a wedged/unreachable
+# backend apart from a misconfiguration without reading the server log.
+REASON_TIMEOUT = "timeout"
+REASON_UNREACHABLE = "unreachable"
+REASON_BAD_RESPONSE = "bad_response"
+
+
+def _positive_float_env(name: str, default: float) -> float:
+    """Read a positive float from the environment, falling back to `default`
+    on anything unparseable or non-positive. A bad value must not be able to
+    turn the timeout into 0 (every query degrades) or a negative cooldown."""
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        log.warning("%s=%r is not a number, using %s", name, raw, default)
+        return default
+    if value <= 0:
+        log.warning("%s=%r is not positive, using %s", name, raw, default)
+        return default
+    return value
+
+
+def _classify(exc: Exception) -> str:
+    """Map an `embed_batch` RuntimeError onto one of the REASON_* constants.
+
+    `embed_batch` flattens every transport failure into one RuntimeError whose
+    message carries the original exception's text, so the message is the only
+    signal available without changing that function's contract."""
+    text = str(exc).lower()
+    if "timed out" in text or "timeout" in text:
+        return REASON_TIMEOUT
+    if "shape mismatch" in text or "not a json object" in text or "too large" in text:
+        return REASON_BAD_RESPONSE
+    return REASON_UNREACHABLE
+
+
+class QueryEmbedder:
+    """Callable matching `retrieval.EmbedQueryFn` (`str -> list[float] | None`),
+    plus a `last_failure_reason` attribute the caller reads via `getattr` so
+    plain-callable stand-ins in tests keep satisfying the same contract."""
+
+    def __init__(self, base_url: str, model: str, timeout: float, cooldown: float):
+        self.base_url = base_url
+        self.model = model
+        self.timeout = timeout
+        self.cooldown = cooldown
+        self.last_failure_reason: str | None = None
+        # Guards the cooldown deadline and the reason: the MCP server can
+        # serve concurrent requests, and without this two queries racing on a
+        # dead backend would each pay a full timeout.
+        self._lock = threading.Lock()
+        self._cooldown_until = 0.0
+
+    def _cooling_down(self) -> bool:
+        with self._lock:
+            return time.monotonic() < self._cooldown_until
+
+    def _record_failure(self, reason: str) -> None:
+        with self._lock:
+            self._cooldown_until = time.monotonic() + self.cooldown
+            self.last_failure_reason = reason
+
+    def _record_success(self) -> None:
+        with self._lock:
+            self._cooldown_until = 0.0
+            self.last_failure_reason = None
+
+    def __call__(self, query: str) -> list[float] | None:
+        if not query or not query.strip():
+            return None
+        if self._cooling_down():
+            # Deliberately silent: the failure that opened the cooldown was
+            # already logged once. Logging here would restore exactly the
+            # per-query noise the cooldown exists to remove.
+            return None
+        try:
+            vectors = embed_batch(self.base_url, self.model, [query], timeout=self.timeout)
+        except RuntimeError as exc:
+            reason = _classify(exc)
+            self._record_failure(reason)
+            log.warning(
+                "graphify-mesh: query embedding unavailable (%s), degrading to "
+                "lexical+structural only for the next %.0fs: %s",
+                reason,
+                self.cooldown,
+                exc,
+            )
+            return None
+        if not vectors:
+            self._record_failure(REASON_BAD_RESPONSE)
+            log.warning(
+                "graphify-mesh: query embedding returned no vector (%s), degrading to "
+                "lexical+structural only for the next %.0fs",
+                REASON_BAD_RESPONSE,
+                self.cooldown,
+            )
+            return None
+        self._record_success()
+        return vectors[0]
+
+    def failure_reason(self) -> str:
+        """The reason to report when this embedder just declined to produce a
+        vector. A suppressed call reports the failure that opened the cooldown
+        rather than `cooling_down`: the cause is what the client needs, and
+        the log line already distinguishes the attempt from the suppression."""
+        with self._lock:
+            return self.last_failure_reason or REASON_UNREACHABLE
+
+
+def make_embed_query_fn(
+    base_url: str | None = None,
+    model: str | None = None,
+    timeout: float | None = None,
+    cooldown: float | None = None,
+) -> QueryEmbedder:
+    """Returns a `str -> list[float] | None` callable matching
+    `retrieval.EmbedQueryFn`, bound to the resolved base_url/model (env
+    override via the SAME `GRAPHIFY_MESH_OLLAMA_EMBED_*` variables the sync
+    pipeline reads, so a query never drifts onto a different embedding
+    space than the one the published vectors were built with).
+
+    Timeout and cooldown are read from `GRAPHIFY_MESH_QUERY_EMBED_TIMEOUT` /
+    `GRAPHIFY_MESH_QUERY_EMBED_COOLDOWN` so a wedged backend can be ridden out
+    by editing the unit, without promoting a new build."""
+    resolved_base_url = base_url or os.environ.get(
+        "GRAPHIFY_MESH_OLLAMA_EMBED_BASE_URL", EMBED_DEFAULT_BASE_URL
+    )
+    resolved_model = model or os.environ.get(
+        "GRAPHIFY_MESH_OLLAMA_EMBED_MODEL", EMBED_DEFAULT_MODEL
+    )
+    resolved_timeout = (
+        timeout
+        if timeout is not None
+        else _positive_float_env("GRAPHIFY_MESH_QUERY_EMBED_TIMEOUT", DEFAULT_QUERY_EMBED_TIMEOUT)
+    )
+    resolved_cooldown = (
+        cooldown
+        if cooldown is not None
+        else _positive_float_env("GRAPHIFY_MESH_QUERY_EMBED_COOLDOWN", DEFAULT_FAILURE_COOLDOWN)
+    )
+    return QueryEmbedder(resolved_base_url, resolved_model, resolved_timeout, resolved_cooldown)

graphify_mesh/server/frames.py

@@ -0,0 +1,93 @@
+"""JSON-RPC frame limits and envelope validation shared by both transports.
+
+Two things must read the same way whichever transport a client speaks:
+
+1. The incoming-message size bound. `MAX_MESSAGE_BYTES` is the single number
+   both transports state — `stdio_guard.MAX_LINE_BYTES` per line, and the
+   HTTP guard plus the SDK session manager per request body.
+2. The error a malformed frame gets back. The SDK validates envelopes with
+   pydantic and puts `str(ValidationError)` — which quotes the offending
+   input — in the client-visible message, so both transports check the
+   envelope themselves first and answer with a fixed generic message,
+   logging the detail instead.
+
+The checks here are deliberately narrower than the SDK's model validation:
+they reject only what no legal JSON-RPC 2.0 frame can be, so a frame that
+passes still reaches the SDK for full validation. A client-to-server
+*response* (a `result`/`error` frame answering a server-initiated request,
+which MCP uses for sampling and roots) carries no `method` and must pass.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Final
+
+# Hard per-message size cap for both transports. The shared HTTP daemon is one
+# process serving every local agent, so its memory is a machine-wide resource,
+# and no legitimate JSON-RPC frame for these seven tools comes close to 4 MiB —
+# a lower ceiling is worth more here than matching stdio's earlier 10 MB.
+# Over-limit input is rejected without ever being buffered whole.
+MAX_MESSAGE_BYTES: Final = 4 * 1024 * 1024
+
+PARSE_ERROR: Final = (-32700, "parse error")
+NOT_A_SINGLE_OBJECT: Final = (
+    -32600,
+    "invalid request: expected a single JSON-RPC object (batch requests are not supported)",
+)
+INVALID_REQUEST: Final = (-32600, "invalid request")
+INVALID_PARAMS: Final = (-32602, "invalid params")
+
+
+def frame_id(parsed: object) -> str | int | None:
+    """The request id to correlate an error response with, or `None` when the
+    frame carries no usable id. Only the two types JSON-RPC 2.0 allows count:
+    an id of any other shape is itself the defect being reported."""
+    if not isinstance(parsed, dict):
+        return None
+    candidate = parsed.get("id")
+    if isinstance(candidate, bool):  # `True` is an int in Python, not a JSON-RPC id
+        return None
+    if isinstance(candidate, str | int):
+        return candidate
+    return None
+
+
+def envelope_error(parsed: object) -> tuple[int, str] | None:
+    """`(code, message)` for a frame the SDK would reject with a detail-
+    carrying validation error, or `None` when the frame is shaped like a
+    legal JSON-RPC 2.0 message."""
+    if not isinstance(parsed, dict):
+        return NOT_A_SINGLE_OBJECT
+    if parsed.get("jsonrpc") != "2.0":
+        return INVALID_REQUEST
+
+    raw_id = parsed.get("id")
+    if "id" in parsed and (isinstance(raw_id, bool) or not isinstance(raw_id, str | int | None)):
+        return INVALID_REQUEST
+
+    if "method" not in parsed:
+        # A response frame (result/error) answering a server-initiated
+        # request is the one legal shape with no method.
+        if "result" in parsed or "error" in parsed:
+            return None
+        return INVALID_REQUEST
+    if not isinstance(parsed["method"], str) or not parsed["method"]:
+        return INVALID_REQUEST
+
+    params = parsed.get("params")
+    if "params" in parsed and params is not None and not isinstance(params, dict):
+        # JSON-RPC 2.0 also allows positional array params; MCP does not use
+        # them, and the SDK models type `params` as `dict | None`, so an array
+        # here is a params-shape error rather than an invalid envelope.
+        # Explicit `null` is exempt: it is what that `| None` accepts, and a
+        # client whose serializer emits it has always worked.
+        return INVALID_PARAMS
+
+    return None
+
+
+def error_frame_text(code: int, message: str, request_id: str | int | None = None) -> str:
+    return json.dumps(
+        {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
+    )