puremethod 0.1.0__py3-none-any.whl

app/ssh_machines.py

@@ -0,0 +1,101 @@
+"""SSH machines (08-app.md §4, KB-036): hosts the engine reaches over SSH, used by `ssh`
+executors and, later, by `via_ssh` model endpoints.
+
+A machine is the source of truth for its connection. Each `ssh` executor keeps a resolved
+copy (address, user, port, key, host key and the jump chain) next to the machine's id, so a
+run's executor snapshot stays self-contained. Every change to a machine re-resolves the
+executors that use it, in the same transaction (`TenantRepo.save_machine`)."""
+
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from app.settings import Mode
+from app.ssh_hosts import parse_host_key
+
+MAX_JUMPS = 4
+
+
+class MachineConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    host: Annotated[str, Field(min_length=1, max_length=253)]
+    user: Annotated[str, Field(min_length=1, max_length=64)]
+    key_credential: Annotated[str, Field(min_length=1)]
+    port: Annotated[int, Field(ge=1, le=65535)] = 22
+    host_key: str | None = None  # the confirmed "<type> <base64>" (INV-10)
+    known_hosts: str | None = None  # self-hosted only; machines migrated from before KB-035
+    jumps: Annotated[list[str], Field(max_length=MAX_JUMPS)] = []  # machine ids, in order
+
+    @field_validator("host_key")
+    @classmethod
+    def _valid_host_key(cls, v: str | None) -> str | None:
+        if v is not None:
+            parse_host_key(v)  # raises SSHSetupError, a ValueError
+        return v
+
+
+CONNECTION = ("host", "user", "key_credential", "port", "host_key", "known_hosts")
+
+
+def validate(config: dict[str, Any], mode: Mode) -> MachineConfig:
+    cfg = MachineConfig.model_validate(config)
+    if mode == "hosted" and cfg.known_hosts is not None:
+        raise ValueError("hosted mode does not accept known_hosts paths; confirm the host key")
+    return cfg
+
+
+def from_executor(config: dict[str, Any]) -> dict[str, Any]:
+    """A machine config from an `ssh` executor's inline fields (08 §4: inline fields create
+    a machine; existing executors are migrated this way)."""
+    out = {k: config[k] for k in CONNECTION if config.get(k) is not None}
+    out.setdefault("port", 22)
+    return out
+
+
+def chain(machines: dict[str, dict[str, Any]], machine_id: str) -> list[dict[str, Any]]:
+    """The hops before `machine_id`, flattened in connection order, each a connection dict
+    with a confirmed host key. A hop's own jumps come first. Cycles and long chains are
+    refused."""
+    hops: list[dict[str, Any]] = []
+
+    def walk(mid: str, seen: tuple[str, ...]) -> None:
+        if mid in seen:
+            raise ValueError("the jump hosts form a loop")
+        cfg = machines.get(mid)
+        if cfg is None:
+            raise ValueError(f"no SSH machine {mid!r}")
+        for jid in cfg.get("jumps", []):
+            walk(jid, (*seen, mid))
+            jump = machines[jid]
+            if not jump.get("host_key"):
+                raise ValueError(f"jump host {jump['host']} has no confirmed host key")
+            hops.append({k: jump[k] for k in ("host", "user", "key_credential", "port",
+                                               "host_key")})
+
+    walk(machine_id, ())
+    if len(hops) > MAX_JUMPS:
+        raise ValueError(f"at most {MAX_JUMPS} jump hosts")
+    return hops
+
+
+def resolve(machines: dict[str, dict[str, Any]], machine_id: str) -> dict[str, Any]:
+    """The connection fields an `ssh` executor stores for `machine_id`."""
+    cfg = machines[machine_id]
+    out: dict[str, Any] = {k: cfg.get(k) for k in CONNECTION}
+    out["port"] = cfg.get("port", 22)
+    out["jumps"] = chain(machines, machine_id)
+    out["machine"] = machine_id
+    return out
+
+
+def users(machines: dict[str, tuple[str, dict[str, Any]]],
+          executors: list[tuple[str, dict[str, Any]]], machine_id: str) -> list[str]:
+    """What uses a machine, by name: executors on it, and machines that jump through it.
+    `machines` maps id to (name, config); `executors` lists (name, config)."""
+    names = [f"executor {name}" for name, cfg in executors if cfg.get("machine") == machine_id]
+    names += [f"SSH machine {name}" for name, cfg in machines.values()
+              if machine_id in cfg.get("jumps", [])]
+    return names

app/statuses.py

@@ -0,0 +1,4 @@
+"""Run statuses as `loop.status` reports them (03-loop.md §3)."""
+
+DRIVABLE = frozenset({"preflight", "running", "stopping", "writing"})  # `drive` advances these
+TERMINAL = frozenset({"delivered", "failed"})

app/supervisor.py

@@ -0,0 +1,87 @@
+"""Keeps a worker process for every run or check that needs one (08-app.md §5 step 2).
+
+Self-hosted mode runs this as a thread inside the server. A worker that exits with an error
+is retried with backoff, so a broken run cannot spin. Leases, not this class, guarantee one
+driver per run: a second worker for the same run exits at once."""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Callable
+from typing import Protocol
+
+from app import scan
+from app.db import Database
+
+log = logging.getLogger("app.supervisor")
+
+BACKOFF_S = (5.0, 30.0, 120.0, 600.0)
+
+
+class Proc(Protocol):
+    def poll(self) -> int | None: ...
+
+
+def spawn_worker(work: scan.Work) -> Proc:
+    # Arguments are ids this app generated, passed as a list: no shell.
+    return subprocess.Popen([sys.executable, "-m", "app.worker", work.kind,  # noqa: S603
+                             work.tenant_id, work.id])
+
+
+class Supervisor:
+    def __init__(self, db: Database, status_of: Callable[[str], str | None], *,
+                 spawn: Callable[[scan.Work], Proc] = spawn_worker,
+                 clock: Callable[[], float] = time.time, interval_s: float = 2.0) -> None:
+        self._db, self._status_of, self._spawn = db, status_of, spawn
+        self._clock, self._interval = clock, interval_s
+        self._procs: dict[str, Proc] = {}
+        self._failures: dict[str, int] = {}
+        self._not_before: dict[str, float] = {}
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def tick(self) -> list[scan.Work]:
+        now = self._clock()
+        for key, proc in list(self._procs.items()):
+            code = proc.poll()
+            if code is None:
+                continue
+            del self._procs[key]
+            if code in (0, 2):  # finished, or another worker holds the lease
+                self._failures.pop(key, None)
+                self._not_before.pop(key, None)
+            else:
+                n = self._failures.get(key, 0)
+                self._failures[key] = n + 1
+                self._not_before[key] = now + BACKOFF_S[min(n, len(BACKOFF_S) - 1)]
+                log.warning("worker %s exited with %s; retry in %.0f s", key, code,
+                            self._not_before[key] - now)
+        started = []
+        for w in scan.work(self._db, self._status_of, now):
+            key = w.lease_key
+            if key in self._procs or self._not_before.get(key, 0.0) > now:
+                continue
+            self._procs[key] = self._spawn(w)
+            started.append(w)
+        return started
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                self.tick()
+            except Exception as exc:  # keep supervising; the next tick retries
+                log.error("supervisor tick failed: %s", type(exc).__name__)
+            self._stop.wait(self._interval)
+
+    def start(self) -> None:
+        self._thread = threading.Thread(target=self._run, name="supervisor", daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=10)

app/tunnels.py

@@ -0,0 +1,222 @@
+"""SSH tunnels to model servers on SSH machines (07-llm.md §4 `via_ssh`, §5 step 9; 08-app.md
+INV-8, INV-10; KB-036).
+
+The worker process that makes the calls opens the tunnel: a listening socket on 127.0.0.1
+(or a bundled network's gateway address) at a free port, whose connections are carried over
+SSH, through the machine's jump chain, to `127.0.0.1:<remote port>` on the machine. The model
+server can stay bound to the machine's loopback. A dropped SSH connection is reopened on the
+next request; `llm` sees a transient error meanwhile and retries (§5 step 5)."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import select
+import socket
+import threading
+from collections.abc import Callable
+from typing import Any
+
+import paramiko
+
+from app import ssh_chain
+
+
+class Tunnel:
+    def __init__(self, hops: list[dict[str, Any]], remote_port: int,
+                 private_key: Callable[[str], str], bind: str = "127.0.0.1") -> None:
+        if not hops:
+            raise ValueError("a tunnel needs the machine to reach")
+        self._hops = hops
+        self._remote_port = remote_port
+        self._private_key = private_key
+        self._bind = bind
+        self._chain: ssh_chain.Chain | None = None
+        self._lock = threading.Lock()
+        self._sock: socket.socket | None = None
+        self._closed = threading.Event()
+        self.port = 0
+
+    @property
+    def machine(self) -> str:
+        return str(self._hops[-1]["host"])
+
+    def open(self) -> int:
+        """Listens; connects over SSH on the first request. Returns the local port."""
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind((self._bind, 0))
+        sock.listen(16)
+        self._sock, self.port = sock, sock.getsockname()[1]
+        threading.Thread(target=self._accept, name=f"tunnel-{self.port}", daemon=True).start()
+        return self.port
+
+    def url(self) -> str:
+        return f"http://{self._bind}:{self.port}/v1"
+
+    def close(self) -> None:
+        self._closed.set()
+        if self._sock is not None:
+            with contextlib.suppress(OSError):
+                self._sock.close()
+        with self._lock:
+            if self._chain is not None:
+                self._chain.close()
+                self._chain = None
+
+    def _connected(self) -> paramiko.Transport:
+        with self._lock:
+            transport = self._chain.client.get_transport() if self._chain else None
+            if transport is None or not transport.is_active():
+                if self._chain is not None:
+                    self._chain.close()
+                self._chain = ssh_chain.connect(self._hops, self._private_key)
+                transport = self._chain.client.get_transport()
+            if transport is None:
+                raise ssh_chain.HopFailed(f"{self.machine}: no SSH transport")
+            transport.set_keepalive(30)
+            return transport
+
+    def _accept(self) -> None:
+        assert self._sock is not None  # noqa: S101 (set by open before this thread starts)
+        while not self._closed.is_set():
+            try:
+                client, _ = self._sock.accept()
+            except OSError:
+                return
+            threading.Thread(target=self._serve, args=(client,), daemon=True).start()
+
+    def _serve(self, client: socket.socket) -> None:
+        try:
+            channel = self._connected().open_channel(
+                "direct-tcpip", ("127.0.0.1", self._remote_port), client.getpeername())
+        except (paramiko.SSHException, OSError, EOFError) as exc:
+            # Answered as a 503 naming the machine or hop: `llm` retries it as transient, and
+            # a persistent failure pauses the run with this reason (07 §5 step 9).
+            reason = str(exc) if isinstance(exc, ssh_chain.HopFailed) else (
+                f"{self.machine}: no model server answers on port {self._remote_port} "
+                f"({type(exc).__name__})")
+            _refuse(client, f"SSH tunnel to {self.machine} failed: {reason}")
+            return
+        try:
+            _pump(client, channel)
+        finally:
+            channel.close()
+            client.close()
+
+
+class Relay:
+    """A plain TCP relay from `bind` to 127.0.0.1:`port` on this machine: lets the bundled
+    PureScholar container reach a model that listens only on the host's loopback where
+    Docker cannot (Docker Engine on Linux). It listens only on the bundled network's gateway
+    address (08-app.md INV-8, KB-036)."""
+
+    def __init__(self, port: int, bind: str) -> None:
+        self._target, self._bind = port, bind
+        self._sock: socket.socket | None = None
+        self._closed = threading.Event()
+        self.port = 0
+
+    def open(self) -> int:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind((self._bind, 0))
+        sock.listen(16)
+        self._sock, self.port = sock, sock.getsockname()[1]
+        threading.Thread(target=self._accept, name=f"relay-{self.port}", daemon=True).start()
+        return self.port
+
+    def close(self) -> None:
+        self._closed.set()
+        if self._sock is not None:
+            with contextlib.suppress(OSError):
+                self._sock.close()
+
+    def _accept(self) -> None:
+        assert self._sock is not None  # noqa: S101 (set by open before this thread starts)
+        while not self._closed.is_set():
+            try:
+                client, _ = self._sock.accept()
+            except OSError:
+                return
+            threading.Thread(target=self._serve, args=(client,), daemon=True).start()
+
+    def _serve(self, client: socket.socket) -> None:
+        try:
+            upstream = socket.create_connection(("127.0.0.1", self._target), timeout=10)
+        except OSError as exc:
+            _refuse(client, f"no model server answers on this computer's port {self._target} "
+                            f"({type(exc).__name__})")
+            return
+        try:
+            upstream.settimeout(None)
+            _pump(client, upstream)
+        finally:
+            upstream.close()
+            client.close()
+
+
+def _refuse(client: socket.socket, message: str) -> None:
+    body = json.dumps({"error": {"message": message, "type": "tunnel_unavailable"}}).encode()
+    try:
+        client.settimeout(5)
+        with contextlib.suppress(OSError):
+            client.recv(65536)  # the request line and headers; their content does not matter
+        client.sendall(b"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
+                       b"Connection: close\r\nContent-Length: " + str(len(body)).encode()
+                       + b"\r\n\r\n" + body)
+    except OSError:
+        pass
+    finally:
+        client.close()
+
+
+def _pump(a: socket.socket, b: Any) -> None:
+    while True:
+        ready, _, _ = select.select([a, b], [], [], 60)
+        if not ready:
+            continue
+        for src, dst in ((a, b), (b, a)):
+            if src in ready:
+                try:
+                    data = src.recv(65536)
+                except OSError:
+                    return
+                if not data:
+                    return
+                dst.sendall(data)
+
+
+def open_tunnels(profile: dict[str, dict[str, Any]], private_key: Callable[[str], str],
+                 bind: str = "127.0.0.1") -> tuple[dict[str, dict[str, Any]], list[Tunnel]]:
+    """For each `via_ssh` role (and fallback), a tunnel and the role pointed at it. Roles on
+    the same machine and port share one tunnel. Tunnels connect on first use, so this only
+    fails if a local port cannot be bound; every tunnel opened so far is closed then."""
+    opened: dict[tuple[str, int, int], Tunnel] = {}
+    tunnels: list[Tunnel] = []
+
+    def point(spec: dict[str, Any]) -> dict[str, Any]:
+        ssh = spec.get("ssh")
+        if not isinstance(ssh, dict):
+            return spec
+        hops, port = list(ssh["hops"]), int(ssh["remote_port"])
+        key = (str(hops[-1]["host"]), int(hops[-1].get("port", 22)), port)
+        if key not in opened:
+            t = Tunnel(hops, port, private_key, bind)
+            t.open()
+            opened[key] = t
+            tunnels.append(t)
+        out = {k: v for k, v in spec.items() if k != "ssh"}
+        out["base_url"] = opened[key].url()
+        return out
+
+    try:
+        result: dict[str, dict[str, Any]] = {}
+        for role, spec in profile.items():
+            spec = point(spec)
+            if isinstance(spec.get("fallback"), dict):
+                spec = {**spec, "fallback": point(spec["fallback"])}
+            result[role] = spec
+    except BaseException:
+        for t in tunnels:
+            t.close()
+        raise
+    return result, tunnels

app/unseal.py

@@ -0,0 +1,27 @@
+"""The only credential decryption in `app`. Worker processes only (INV-2, KB-021): import-linter
+forbids `app.api` from importing this module, and `calls` lets tests check the API process
+never decrypts."""
+
+from __future__ import annotations
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from app.keys import aad
+
+calls = 0  # decryptions in this process
+
+
+class CredentialUnreadable(Exception):
+    """Wrong master key, or a ciphertext that is not this row's. Never carries the value."""
+
+
+def unseal(master_key: bytes, tenant_id: str, credential_id: str, nonce: bytes,
+           ciphertext: bytes) -> str:
+    global calls
+    calls += 1
+    try:
+        plain = AESGCM(master_key).decrypt(nonce, ciphertext, aad(tenant_id, credential_id))
+    except InvalidTag:
+        raise CredentialUnreadable(credential_id) from None
+    return plain.decode()