keywright-server 0.0.1__py3-none-any.whl

keywright_server/ssh.py

@@ -0,0 +1,290 @@
+"""Runs one command on one host over SSH. Keys live on the Keywright host only.
+
+Resource config for kind=ssh:
+  {"host": "10.0.0.5", "port": 22, "user": "root", "key_path": "/var/lib/keywright/keys/web1",
+   "host_key": "<pinned 'ssh-ed25519 AAAA...' line, learned on first connect>"}
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import re
+import shlex
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import asyncssh
+
+MAX_OUTPUT = 1024 * 1024
+MAX_PUT = 1024 * 1024
+CONNECT_TIMEOUT = 30
+MAX_TIMEOUT = 600
+
+
+@dataclass
+class RunResult:
+    exit_code: int | None
+    stdout: str
+    stderr: str
+    truncated: bool
+    duration_ms: int
+    learned_host_key: str | None = None  # set when we pinned the host key on this call
+
+
+class HostKeyMismatch(Exception):
+    pass
+
+
+def _server_key_line(conn) -> str | None:
+    """'ssh-ed25519 AAAA...' for the host key the server proved it holds on this connection."""
+    key = conn.get_server_host_key()
+    return key.export_public_key().decode().strip() if key is not None else None
+
+
+def _known_hosts_for(pinned: str):
+    """asyncssh known_hosts argument that trusts exactly the pinned key (for any hostname)."""
+    try:
+        return ([asyncssh.import_public_key(pinned)], [], [])
+    except asyncssh.KeyImportError as e:
+        raise HostKeyMismatch("stored host key is not a valid public key; forget it in the UI and re-pin") from e
+
+
+async def run(config: dict[str, Any], command: str, *, timeout: int = 300, user: str | None = None,
+              stdin: bytes | None = None) -> RunResult:
+    timeout = max(1, min(int(timeout or 300), MAX_TIMEOUT))
+    host = config["host"]
+    port = int(config.get("port") or 22)
+    username = user or config.get("user") or "root"
+    key_path = config.get("key_path")
+    pinned = config.get("host_key")
+    started = time.monotonic()
+
+    options: dict[str, Any] = {
+        "host": host,
+        "port": port,
+        "username": username,
+        "client_keys": [key_path] if key_path else [],
+        "agent_path": None,
+        "config": None,
+        "connect_timeout": CONNECT_TIMEOUT,
+    }
+    if pinned:
+        options["known_hosts"] = _known_hosts_for(pinned)
+    else:
+        options["known_hosts"] = None  # first contact: learn and pin below
+
+    learned = None
+    async with asyncssh.connect(**options) as conn:
+        if not pinned:
+            learned = _server_key_line(conn)
+        # Drain both streams concurrently, retaining at most MAX_OUTPUT bytes in total.
+        # conn.run() buffers all output before returning, so slicing afterwards is not a memory limit.
+        buffers = [bytearray(), bytearray()]
+        remaining = MAX_OUTPUT
+        truncated = False
+        async with conn.create_process(command, encoding=None) as process:
+            if stdin:
+                process.stdin.write(stdin)
+            process.stdin.write_eof()
+
+            async def drain(reader, target):
+                nonlocal remaining, truncated
+                while chunk := await reader.read(32768):
+                    keep = min(len(chunk), remaining)
+                    target.extend(chunk[:keep])
+                    remaining -= keep
+                    truncated |= keep < len(chunk)
+
+            try:
+                await asyncio.wait_for(asyncio.gather(
+                    drain(process.stdout, buffers[0]), drain(process.stderr, buffers[1]), process.wait_closed(),
+                ), timeout=timeout)
+                code = process.exit_status
+            except TimeoutError:
+                process.close()
+                code = 124
+                buffers[1] = bytearray(f"keywright: command timed out after {timeout}s".encode())
+                if len(buffers[0]) + len(buffers[1]) > MAX_OUTPUT:
+                    del buffers[0][MAX_OUTPUT - len(buffers[1]):]
+                    truncated = True
+        out, err = (bytes(b).decode(errors="replace") for b in buffers)
+    return RunResult(
+        exit_code=code,
+        stdout=out if isinstance(out, str) else out.decode(errors="replace"),
+        stderr=err if isinstance(err, str) else err.decode(errors="replace"),
+        truncated=truncated,
+        duration_ms=int((time.monotonic() - started) * 1000),
+        learned_host_key=learned,
+    )
+
+
+def remote_path(path: str) -> str:
+    """An absolute path that can also be spelled as permission-path segments, so a rule can name it
+    (ssh/<host>/put/etc/komodo/stacks/**). Anything else is refused rather than escaped."""
+    from keywright.patterns import validate
+
+    if not isinstance(path, str) or not path.startswith("/") or path.endswith("/") or "//" in path:
+        raise ValueError("path must be absolute, without a trailing slash or empty segments")
+    parts = path[1:].split("/")
+    if any(seg in (".", "..") or seg.endswith("*") for seg in parts):
+        raise ValueError("path must not contain . or .. segments or end a segment with *")
+    try:
+        validate("ssh/x/put/" + "/".join(parts))
+    except ValueError as e:
+        raise ValueError(f"path segments may only use letters, digits and _ . : @ + = , - ({e})") from e
+    return path
+
+
+def put_command(path: str, mode: str) -> str:
+    """Write stdin to `path` atomically: temp file beside it, mode set, then renamed into place, so a
+    dropped connection never leaves a half-written file. Parent directories are created."""
+    if not re.fullmatch(r"0?[0-7]{3}", mode):
+        raise ValueError("mode must be three octal digits, e.g. 0644")
+    script = ('set -e; f=$1; d=$(dirname -- "$f"); mkdir -p -- "$d"; t=$(mktemp -- "$d/.keywright.XXXXXX"); '
+              'cat > "$t"; chmod "$2" "$t"; mv -f -- "$t" "$f"')
+    return f"sh -c {shlex.quote(script)} keywright {shlex.quote(path)} {mode}"
+
+
+def append_command(path: str) -> str:
+    """Append stdin to `path`, creating it (and its directory) if needed, and never widening its mode."""
+    script = 'set -e; f=$1; mkdir -p -- "$(dirname -- "$f")"; [ -e "$f" ] || (umask 077; : > "$f"); cat >> "$f"'
+    return f"sh -c {shlex.quote(script)} keywright {shlex.quote(path)}"
+
+
+def get_command(path: str) -> str:
+    return f"cat -- {shlex.quote(path)}"
+
+
+def failure_stage(exc: BaseException) -> tuple[str, str]:
+    """Where a run broke, coarse enough to hold no secrets: (stage, what the agent should do)."""
+    if isinstance(exc, asyncssh.PermissionDenied):
+        return "auth", "The host rejected Keywright's key; the admin has to enrol it on the host."
+    if isinstance(exc, (asyncssh.HostKeyNotVerifiable, HostKeyMismatch)):
+        return "hostkey", "The host key does not match the pinned one; the admin has to re-probe the resource."
+    if isinstance(exc, (OSError, TimeoutError, asyncssh.ConnectionLost, asyncssh.DisconnectError)):
+        return "unreachable", "No SSH connection to the host; check that it is up and the port is open."
+    return "exec", "The connection worked but the command could not be run."
+
+
+async def probe(config: dict[str, Any]) -> tuple[bool, str, str | None]:
+    """Connection test used by the admin UI. Returns (ok, message, learned_host_key)."""
+    try:
+        r = await run(config, "echo keywright-ok", timeout=15)
+    except asyncssh.PermissionDenied:
+        return False, "Authentication failed: is the key authorized on the host?", None
+    except asyncssh.HostKeyNotVerifiable as e:
+        return False, f"Host key mismatch: {e}", None
+    except HostKeyMismatch as e:
+        return False, str(e), None
+    except (OSError, asyncssh.Error, TimeoutError) as e:
+        return False, f"{type(e).__name__}: {e}", None
+    if r.exit_code == 0 and "keywright-ok" in r.stdout:
+        return True, f"Connected as {config.get('user') or 'root'}@{config['host']} in {r.duration_ms} ms", r.learned_host_key
+    return False, f"Unexpected reply (exit {r.exit_code}): {r.stderr[:200]}", r.learned_host_key
+
+
+# ---- key generation & enrolment -----------------------------------------------------------------
+def generate_key(name: str, keys_dir, comment: str | None = None) -> tuple[str, str]:
+    """Create an ed25519 keypair under keys_dir/<name>. Returns (private_path, public_line)."""
+    from pathlib import Path
+
+    keys_dir = Path(keys_dir)
+    keys_dir.mkdir(parents=True, exist_ok=True)
+    priv = keys_dir / name
+    if priv.exists():
+        raise FileExistsError(f"{priv} already exists")
+    key = asyncssh.generate_private_key("ssh-ed25519", comment=comment or f"keywright->{name}")
+    import os
+
+    fd = os.open(priv, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(fd, "wb") as out:
+        out.write(key.export_private_key())
+    pub_line = key.export_public_key().decode().strip()
+    (keys_dir / f"{name}.pub").write_text(pub_line + "\n")
+    return str(priv), pub_line
+
+
+def authorized_keys_line(pub_line: str, restrict_from: str | None) -> str:
+    if restrict_from:
+        # This setting represents one outbound address, not arbitrary authorized_keys options.
+        restrict_from = str(ipaddress.ip_address(restrict_from))
+    if "\n" in pub_line or "\r" in pub_line:
+        raise ValueError("Public key must be a single line")
+    return (f'from="{restrict_from}" ' if restrict_from else "") + pub_line
+
+
+def enroll_script(pub_line: str, restrict_from: str | None) -> str:
+    """POSIX shell that installs the key for the invoking user, idempotently."""
+    line = authorized_keys_line(pub_line, restrict_from)
+    key_part = pub_line.split()[1]
+    return (
+        "set -e\n"
+        'D="$HOME/.ssh"; F="$D/authorized_keys"\n'
+        'mkdir -p "$D" && chmod 700 "$D"\n'
+        f"if grep -qF {shlex.quote(key_part)} \"$F\" 2>/dev/null; then echo 'keywright: key already present'; "
+        f"else printf '%s\\n' {shlex.quote(line)} >> \"$F\" && echo 'keywright: key installed for' \"$(id -un)@$(hostname)\"; fi\n"
+        'chmod 600 "$F"\n'
+    )
+
+
+async def enroll_with_password(config: dict[str, Any], password: str, pub_line: str, restrict: bool = True,
+                               *, login_user: str | None = None, escalate: str | None = None,
+                               root_password: str | None = None) -> tuple[bool, str, str | None]:
+    """Log in once with a password, install the public key for the resource user, learn the host key.
+
+    login_user: account to log in as when password SSH is disabled for the target user (e.g. root).
+    escalate:   how to become the target user from login_user: "sudo" (uses root_password or the
+                login password) or "su" (uses root_password). None when login_user is the target.
+    Passwords are used for this connection only and never stored. Returns (ok, message, host_key)."""
+    import base64
+    import shlex
+
+    host = config["host"]
+    port = int(config.get("port") or 22)
+    target = config.get("user") or "root"
+    login = (login_user or "").strip() or target
+
+    def safe_output(value: str) -> str:
+        for secret in (password, root_password):
+            if secret:
+                value = value.replace(secret, "[redacted]")
+        return value
+
+    try:
+        async with asyncssh.connect(host, port=port, username=login, password=password,
+                                    known_hosts=_known_hosts_for(config["host_key"]) if config.get("host_key") else None,
+                                    client_keys=[], agent_path=None, config=None, connect_timeout=CONNECT_TIMEOUT) as conn:
+            learned = _server_key_line(conn)
+            src = conn.get_extra_info("sockname")[0] if restrict else None
+            script = enroll_script(pub_line, src)
+            if login == target:
+                res = await asyncio.wait_for(conn.run(script, check=False), timeout=30)
+            else:
+                b64 = base64.b64encode(script.encode()).decode()
+                inner = f"echo {b64} | base64 -d | sh"
+                if escalate == "su":
+                    cmd = f"su - {shlex.quote(target)} -c {shlex.quote(inner)}"
+                    res = await asyncio.wait_for(
+                        conn.run(cmd, input=(root_password or "") + "\n", term_type="xterm", check=False), timeout=45)
+                elif escalate == "sudo":
+                    cmd = f"sudo -H -S -p '' -u {shlex.quote(target)} sh -c {shlex.quote(inner)}"
+                    res = await asyncio.wait_for(
+                        conn.run(cmd, input=(root_password or password) + "\n", check=False), timeout=45)
+                else:
+                    return False, f"Logged in as {login}, but no escalation method chosen to become {target}.", learned
+            out = safe_output(res.stdout or "").replace("\r", "")
+            if res.exit_status != 0:
+                err = safe_output(res.stderr or "")[:300] or out[-300:]
+                return False, f"install script failed (exit {res.exit_status}): {err.strip()}", learned
+            note = next((ln.strip() for ln in out.splitlines() if "keywright:" in ln), out.strip()[-200:])
+    except asyncssh.PermissionDenied:
+        return False, f"Password rejected for {login}@{host} (or password logins are disabled for that user).", None
+    except (OSError, asyncssh.Error, TimeoutError, HostKeyMismatch) as e:
+        return False, f"{type(e).__name__}: {e}", None
+    ok, msg, _ = await probe({**config, "host_key": learned} if learned else config)
+    if not ok:
+        return False, f"{note}; but key login as {target} failed afterwards: {msg}", learned
+    tail = f" Restricted to source {src}." if restrict else ""
+    return True, f"{note}; key login verified ({msg}).{tail}", learned

keywright_server/tool_access.py

@@ -0,0 +1,260 @@
+"""The Access page and the tool watch: what an MCP server offers, what each agent may do with it,
+and what changed upstream since the owner last looked.
+
+Rows are built from the live tool list (or a gateway's models). Each row has one rule path, the
+tool's path narrowed by an optional scope such as `account_name=agents` where the tool takes that
+argument, and a state per agent: allow, ask (no rule; the resource default decides) or deny.
+Saving writes only the difference, as ordinary standing rules, and touches no rule it did not
+make the exact shape of: qualified or constrained rules made elsewhere are shown, not edited.
+
+The watch keeps a fingerprint per tool (its input schema and annotations) in the resource
+config. A new tool is harmless on its own (no rule covers it, so it asks); a changed one can
+matter: an argument added to an allowed tool is not qualified by any argument key the owner set,
+so it passes on the tool's rule until the owner looks. Drift is recorded and shown, never acted on.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+from keywright.patterns import matches, qualifier
+
+from . import bundles
+
+GROUPS = ("read-only", "write", "destructive", "gone")
+STATES = ("allow", "ask", "deny")
+
+
+# ---- tools -------------------------------------------------------------------------------------
+def group_of(tool: Any) -> str:
+    """The server's own annotations first (MCP readOnlyHint / destructiveHint), the verb in the
+    name when it says nothing."""
+    ann = getattr(tool, "annotations", None)
+    if ann is not None:
+        if getattr(ann, "read_only_hint", None) is True:
+            return "read-only"
+        if getattr(ann, "destructive_hint", None) is True:
+            return "destructive"
+        if getattr(ann, "read_only_hint", None) is False and getattr(ann, "destructive_hint", None) is False:
+            return "write"
+    bucket = next(iter(bundles.suggest([tool.name])))
+    return {"read-only": "read-only", "destructive": "destructive"}.get(bucket, "write")
+
+
+def _schema(tool: Any) -> dict[str, Any]:
+    return dict(getattr(tool, "input_schema", None) or getattr(tool, "inputSchema", None) or {})
+
+
+def arguments(tool: Any) -> list[str]:
+    return sorted((_schema(tool).get("properties") or {}).keys())
+
+
+def takes(tool: Any, key: str) -> bool:
+    """Whether a dotted argument key exists in the tool's input schema."""
+    node: Any = _schema(tool)
+    for part in key.split("."):
+        props = node.get("properties") if isinstance(node, dict) else None
+        if not isinstance(props, dict) or part not in props:
+            return False
+        node = props[part]
+    return True
+
+
+def fingerprint(tool: Any) -> dict[str, Any]:
+    ann = getattr(tool, "annotations", None)
+    body = json.dumps({"schema": _schema(tool), "annotations": ann.model_dump(exclude_none=True) if ann else None},
+                      sort_keys=True, default=str)
+    return {"hash": hashlib.sha256(body.encode()).hexdigest()[:16], "args": arguments(tool)}
+
+
+@dataclass
+class Drift:
+    added: list[str] = field(default_factory=list)
+    removed: list[str] = field(default_factory=list)
+    changed: list[dict[str, Any]] = field(default_factory=list)  # {tool, new_args, dropped_args}
+
+    def __bool__(self) -> bool:
+        return bool(self.added or self.removed or self.changed)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {"added": self.added, "removed": self.removed, "changed": self.changed}
+
+
+def diff(seen: dict[str, dict[str, Any]], tools: list[Any]) -> Drift:
+    now_fp = {t.name: fingerprint(t) for t in tools}
+    d = Drift(added=sorted(set(now_fp) - set(seen)), removed=sorted(set(seen) - set(now_fp)))
+    for name in sorted(set(now_fp) & set(seen)):
+        old, new = seen[name], now_fp[name]
+        if old.get("hash") != new["hash"]:
+            d.changed.append({"tool": name,
+                              "new_args": sorted(set(new["args"]) - set(old.get("args") or [])),
+                              "dropped_args": sorted(set(old.get("args") or []) - set(new["args"]))})
+    return d
+
+
+def baseline(tools: list[Any]) -> dict[str, Any]:
+    return {"tools_seen": {t.name: fingerprint(t) for t in tools},
+            "tools_seen_at": datetime.now(UTC).isoformat(timespec="seconds"), "tools_drift": None}
+
+
+def record(config: dict[str, Any], tools: list[Any]) -> tuple[dict[str, Any], Drift | None]:
+    """The resource config after a watch pass: the first pass takes a baseline silently; later ones
+    store the drift against it (or clear it) and leave the baseline for the owner to accept."""
+    seen = config.get("tools_seen")
+    if seen is None:
+        return {**config, **baseline(tools)}, None
+    d = diff(seen, tools)
+    stored = {**d.as_dict(), "at": datetime.now(UTC).isoformat(timespec="seconds")} if d else None
+    return {**config, "tools_drift": stored}, d
+
+
+# ---- rows and rules ----------------------------------------------------------------------------
+@dataclass
+class Row:
+    name: str
+    group: str
+    pattern: str
+    description: str = ""
+    state: str = "ask"
+    everyone: str | None = None           # an every-agent rule on the same path, when viewing one agent
+    other_rules: int = 0                  # rules on this tool of another shape (qualified, constrained)
+    covered_by: list[str] = field(default_factory=list)  # broader rules that already decide this row
+    flag: str | None = None               # new | changed | gone
+    new_args: list[str] = field(default_factory=list)
+
+
+def scoped(base: str, scope: tuple[str, str] | None, tool: Any | None) -> str:
+    if scope and tool is not None and takes(tool, scope[0]):
+        return f"{base}/{qualifier(scope[0], scope[1])}"
+    return base
+
+
+def parse_scope(text: str, arg_keys: list[str]) -> tuple[str, str] | None:
+    text = (text or "").strip()
+    if not text:
+        return None
+    if "=" not in text:
+        raise ValueError("Scope is key=value, e.g. account_name=agents")
+    key, value = (s.strip() for s in text.split("=", 1))
+    if key not in arg_keys:
+        raise ValueError(f"{key!r} is not one of this server's argument keys ({', '.join(arg_keys) or 'none set'}); "
+                         "add it on the resource first, or rules on it would never match")
+    if not value or "*" in value:
+        raise ValueError("Scope needs a literal value")
+    return key, value
+
+
+def state_of(policies: list[Any], pattern: str, agent_id: str | None) -> str:
+    exact = [p for p in policies if p.pattern == pattern and not p.constraints and p.agent_id == agent_id]
+    if any(p.effect == "deny" for p in exact):
+        return "deny"
+    return "allow" if exact else "ask"
+
+
+def touches(policy_pattern: str, tool_path: str) -> bool:
+    return policy_pattern == tool_path or policy_pattern.startswith(tool_path + "/")
+
+
+def mcp_rows(res: Any, tools: list[Any], policies: list[Any], agent_id: str | None,
+             scope: tuple[str, str] | None) -> list[Row]:
+    seen = res.config.get("tools_seen") or {}
+    drift = res.config.get("tools_drift") or {}
+    changed = {c["tool"]: c for c in drift.get("changed") or []}
+    rows = []
+    for t in sorted(tools, key=lambda t: t.name):
+        base = f"mcp/{res.name}/tools/{t.name}"
+        rows.append(_row(t.name, group_of(t), scoped(base, scope, t), base, policies, agent_id,
+                         (t.description or "").strip().splitlines()[0][:160] if t.description else "",
+                         flag="new" if seen and t.name not in seen else "changed" if t.name in changed else None,
+                         new_args=(changed.get(t.name) or {}).get("new_args") or []))
+    # Rules on tools the server no longer offers: one row per rule, on its exact path, so Ask
+    # removes it whatever its shape.
+    live = {t.name for t in tools}
+    prefix = f"mcp/{res.name}/tools/"
+    stale = sorted({p.pattern for p in policies if p.agent_id == agent_id and not p.constraints
+                    and p.pattern.startswith(prefix)
+                    and p.pattern[len(prefix):].split("/", 1)[0] not in live | {"*", "**"}})
+    for pattern in stale:
+        rows.append(_row(pattern[len(prefix):], "gone", pattern, pattern, policies, agent_id,
+                         "No longer offered by the server; this rule can go.", flag="gone"))
+    return rows
+
+
+def llm_rows(res: Any, model_paths: dict[str, str], policies: list[Any], agent_id: str | None) -> list[Row]:
+    return [_row(m, "models", path, path, policies, agent_id, "") for m, path in model_paths.items()]
+
+
+def _row(name, group, pattern, base, policies, agent_id, description, flag=None, new_args=None) -> Row:
+    mine = [p for p in policies if p.agent_id == agent_id]
+    # Rules that match this row's path without being its own: a wildcard or a shorter prefix. They
+    # decide the call whatever this row says (a deny anywhere wins), so the page says so.
+    covered = [f"{p.effect} {p.pattern}{'' if p.agent_id else ' (every agent)'}" for p in policies
+               if (p.agent_id in (agent_id, None)) and not p.constraints and p.pattern != pattern
+               and matches(p.pattern, pattern)]
+    return Row(covered_by=covered, name=name, group=group, pattern=pattern, description=description,
+               state=state_of(policies, pattern, agent_id),
+               everyone=(state_of(policies, pattern, None) if agent_id else None),
+               other_rules=sum(1 for p in mine if touches(p.pattern, base) and (p.pattern != pattern or p.constraints)),
+               flag=flag, new_args=new_args or [])
+
+
+@dataclass
+class Plan:
+    add: list[tuple[str, str]] = field(default_factory=list)      # (pattern, effect)
+    delete: list[Any] = field(default_factory=list)               # Policy rows
+
+    def __bool__(self) -> bool:
+        return bool(self.add or self.delete)
+
+
+def plan(rows: list[Row], wanted: dict[str, str], policies: list[Any], agent_id: str | None) -> Plan:
+    """The rule changes that take each row from its state to the wanted one. Rows not in `wanted`
+    (not on the submitted form) are left alone."""
+    out = Plan()
+    for row in rows:
+        want = wanted.get(row.name)
+        if want not in STATES or want == row.state:
+            continue
+        out.delete += [p for p in policies if p.pattern == row.pattern and not p.constraints
+                       and p.agent_id == agent_id]
+        if want != "ask":
+            out.add.append((row.pattern, want))
+    return out
+
+
+# ---- the watch -----------------------------------------------------------------------------------
+async def watch_once(db) -> list[tuple[str, Drift]]:
+    """One pass over every MCP server: baseline the new ones, record drift on the rest. A server
+    that cannot be reached is skipped (its last drift stands). Returns the servers whose drift is
+    new since the last pass, which are also written to the audit log."""
+    import logging
+
+    from sqlalchemy import select
+
+    from . import audit, mcp_proxy
+    from .models import Resource
+
+    log = logging.getLogger("keywright.tools")
+    fresh: list[tuple[str, Drift]] = []
+    for res in (await db.execute(select(Resource).where(Resource.kind == "mcp"))).scalars().all():
+        try:
+            tools = await mcp_proxy.list_tools(res.name, res.config, force=True)
+        except Exception as e:  # noqa: BLE001
+            log.info("tool watch: %s unreachable: %s", res.name, mcp_proxy.describe(e))
+            continue
+        before = (res.config.get("tools_drift") or {})
+        config, d = record(res.config, tools)
+        if config == res.config:
+            continue
+        res.config = config
+        await db.commit()
+        if d and {k: before.get(k) for k in ("added", "removed", "changed")} != d.as_dict():
+            fresh.append((res.name, d))
+            log.warning("tool watch: %s changed: %s", res.name, d.as_dict())
+            await audit.record(db, tool="tools.drift", decision="changed",
+                               detail={"resource": res.name, **d.as_dict()})
+    return fresh

keywright_server/usage.py

@@ -0,0 +1,121 @@
+"""Usage totals from local proxy audit events, with independent provider identities."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime
+
+from sqlalchemy import select
+
+from .models import AuditEvent, LlmKey, Resource
+
+
+def count(value):
+    return value if type(value) is int and 0 <= value else None
+
+
+def token_counts(usage):
+    usage = usage if isinstance(usage, dict) else {}
+    incoming = count(usage.get("prompt", usage.get("input")))
+    outgoing = count(usage.get("completion", usage.get("output")))
+    # Anthropic cache counts are additional input; OpenAI cached tokens are a subset.
+    if incoming is not None and "prompt" not in usage:
+        incoming += (count(usage.get("cache_creation_input")) or 0)
+        incoming += (count(usage.get("cache_read_input")) or 0)
+    total = count(usage.get("total"))
+    if total is None and (incoming is not None or outgoing is not None):
+        total = (incoming or 0) + (outgoing or 0)
+    return incoming, outgoing, total
+
+
+@dataclass
+class Totals:
+    requests: int = 0
+    errors: int = 0
+    reported: int = 0
+    unreported: int = 0
+    partial: int = 0
+    input: int = 0
+    output: int = 0
+    total: int = 0
+
+    def add(self, event):
+        incoming, outgoing, total = token_counts(event.detail.get("usage"))
+        self.requests += 1
+        self.errors += event.decision == "error"
+        self.reported += total is not None
+        self.unreported += total is None
+        self.partial += total is not None and (
+            incoming is None or outgoing is None or event.detail.get("response_complete") is False)
+        self.input += incoming or 0
+        self.output += outgoing or 0
+        self.total += total or 0
+
+
+async def report(db, start: datetime, end: datetime, *, provider="", model="", agent="", key=""):
+    # Only select safe identity fields, never load upstream secrets into the report.
+    resources = (await db.execute(select(Resource.id, Resource.name).where(Resource.kind == "llm"))).all()
+    resource_names = dict(resources)
+    resource_ids = {name: rid for rid, name in resources}
+    aliases = {}
+    for kid, rid, aid, alias in (await db.execute(select(
+            LlmKey.id, LlmKey.resource_id, LlmKey.agent_id, LlmKey.key_alias))).all():
+        identity = (rid, aid, alias)
+        # Ambiguous historical aliases must stay separate from known key IDs.
+        aliases[identity] = kid if identity not in aliases else None
+
+    options = {name: {} for name in ("provider", "model", "agent", "key")}
+    providers, models, keys, days = {}, {}, {}, {}
+    summary = Totals()
+    recent = []
+    stmt = select(AuditEvent).where(
+        AuditEvent.tool == "llm.call", AuditEvent.ts >= start, AuditEvent.ts < end,
+        AuditEvent.decision.in_(["allow", "error"])
+    ).order_by(AuditEvent.ts.desc(), AuditEvent.id.desc()).execution_options(yield_per=500)
+    async for event in (await db.stream_scalars(stmt)):
+        detail = event.detail
+        # Exclude discovery, blocked calls, and key-mint failures before an upstream attempt.
+        if not detail.get("model") or not (detail.get("gateway_key_id") or detail.get("key_alias")):
+            continue
+        gateway = detail.get("gateway") or "Unknown provider"
+        pid = detail.get("resource_id") or resource_ids.get(gateway) or "legacy:" + gateway
+        provider_name = resource_names.get(pid, gateway)
+        model_name = detail["model"]
+        aid = event.agent_id or "legacy:" + event.agent_name
+        agent_name = event.agent_name or "Unknown agent"
+        alias = detail.get("key_alias") or "Unidentified key"
+        kid = (detail.get("gateway_key_id") or aliases.get((pid, event.agent_id, alias))
+               or "legacy:" + alias)
+        # A key selector includes its provider and agent, including for legacy aliases.
+        key_identity = json.dumps([pid, aid, kid], separators=(",", ":"))
+        labels = {"provider": (pid, provider_name), "model": (model_name, model_name),
+                  "agent": (aid, agent_name),
+                  "key": (key_identity, f"{alias} · {provider_name} · {agent_name} · {kid}")}
+        for dimension, (value, label) in labels.items():
+            options[dimension][value] = label
+        if any(selected and labels[dimension][0] != selected for dimension, selected in (
+                ("provider", provider), ("model", model), ("agent", agent), ("key", key))):
+            continue
+        summary.add(event)
+        p = providers.setdefault(pid, {"id": pid, "name": provider_name, "totals": Totals(), "models": {}})
+        p["totals"].add(event)
+        m = p["models"].setdefault(model_name, {"name": model_name, "totals": Totals()})
+        m["totals"].add(event)
+        models.setdefault(model_name, Totals()).add(event)
+        k = keys.setdefault(key_identity, {"id": key_identity, "alias": alias, "key_id": kid,
+                                          "provider": provider_name, "agent": agent_name,
+                                          "totals": Totals(), "models": set()})
+        k["totals"].add(event)
+        k["models"].add(model_name)
+        day = event.ts.date().isoformat()
+        days.setdefault(day, Totals()).add(event)
+        if len(recent) < 50:
+            recent.append({"id": event.id, "ts": event.ts, "provider": provider_name, "model": model_name,
+                           "agent": agent_name, "alias": alias, "decision": event.decision,
+                           "total": token_counts(detail.get("usage"))[2]})
+    ranked = lambda row: (-row["totals"].total, -row["totals"].requests)
+    return {"totals": summary, "providers": sorted(providers.values(), key=ranked),
+            "models": sorted(models.items(), key=lambda row: (-row[1].total, -row[1].requests)),
+            "keys": sorted(keys.values(), key=ranked), "days": sorted(days.items()), "recent": recent,
+            "options": {name: sorted(values.items(), key=lambda pair: pair[1].lower())
+                        for name, values in options.items()}}