puremethod 0.1.0__py3-none-any.whl

app/worker.py

@@ -0,0 +1,657 @@
+"""Worker processes (08-app.md §5 steps 2 and 3, KB-021).
+
+`python -m app.worker run <tenant> <run>` drives one run while it holds the run's lease;
+`python -m app.worker preflight <tenant> <check>` runs one executor preflight;
+`python -m app.worker credcheck <tenant> <check>` checks one credential (KB-029);
+`python -m app.worker llmsearch <tenant> <search>` looks for model servers (KB-036);
+`python -m app.worker profilecheck <tenant> <check>` checks an LLM profile (KB-037);
+`python -m app.worker literature <tenant> <tenant>` starts the tenant's bundled PureScholar
+(KB-025, KB-031). These are the only processes that decrypt credentials (INV-2)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import sys
+import threading
+import time
+import uuid
+from collections.abc import Callable
+from typing import Any
+
+from pydantic import ValidationError
+
+from app import (
+    bundled,
+    leases,
+    literature_sources,
+    model_endpoints,
+    model_search,
+    ssh_chain,
+    ssh_hosts,
+    tunnels,
+)
+from app.compose import (
+    CredentialMissing,
+    Credentials,
+    ServiceUrls,
+    allowed_for_run,
+    build_executor,
+    make_loop,
+    prices,
+    run_config,
+)
+from app.db import Database
+from app.executor_config import credentials_named
+from app.repo import CommandRow, ExecutorRow, RunRow, TenantRepo
+from app.settings import Settings
+from app.statuses import DRIVABLE, TERMINAL
+from app.unseal import CredentialUnreadable
+from executor import Executor
+from llm import LLM
+from loop import Loop
+from purescholar import PureScholar
+from puretrace import PureTrace
+
+log = logging.getLogger("app.worker")
+
+IDLE_S = 600.0  # 10 idle minutes (KB-021)
+POLL_S = 1.0
+INTERRUPTED = "start interrupted; create the run again"
+EXIT_OK, EXIT_ERROR, EXIT_BUSY, EXIT_LEASE_LOST = 0, 1, 2, 3
+
+LoopFactory = Callable[[RunRow, Callable[[str], str], Callable[[], None]], Loop]
+ExecutorFactory = Callable[[ExecutorRow, Callable[[str], str]], Executor]
+
+
+class LeaseLost(Exception):
+    pass
+
+
+class _Lease:
+    """Holds a lease and renews it from a thread; a failed renewal ends the process at once.
+    `check` runs before every ledger write (`compose.GuardedLedger`): a worker that was
+    suspended past its lease never writes, even before its next renewal runs (KB-021)."""
+
+    def __init__(self, db: Database, key: str, *, clock: Callable[[], float], ttl: float,
+                 renew_s: float, on_lost: Callable[[int], Any],
+                 monotonic: Callable[[], float] = time.monotonic) -> None:
+        self._db, self.key, self._clock, self._mono = db, key, clock, monotonic
+        self._ttl, self._renew_s, self._on_lost = ttl, renew_s, on_lost
+        self.holder = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._loop, name="lease", daemon=True)
+        self._ok_wall = self._ok_mono = 0.0
+
+    def acquire(self) -> bool:
+        if not leases.acquire(self._db, self.key, self.holder, self._clock(), self._ttl):
+            return False
+        self._ok_wall, self._ok_mono = self._clock(), self._mono()
+        self._thread.start()
+        return True
+
+    def check(self) -> None:
+        """Both clocks must show a renewal within half the TTL: wall time catches a suspend
+        that a monotonic clock may not count, and the other way round."""
+        limit = self._ttl / 2
+        if self._clock() - self._ok_wall > limit or self._mono() - self._ok_mono > limit:
+            self._lost()
+            raise LeaseLost(self.key)
+
+    def _lost(self) -> None:
+        log.error("lease %s lost; stopping", self.key)
+        self._on_lost(EXIT_LEASE_LOST)
+
+    def _loop(self) -> None:
+        while not self._stop.wait(self._renew_s):
+            try:
+                ok = leases.renew(self._db, self.key, self.holder, self._clock(), self._ttl)
+            except Exception:  # a locked database: retry while the lease still has time
+                ok = None
+            if ok:
+                self._ok_wall, self._ok_mono = self._clock(), self._mono()
+            elif ok is False or self._clock() - self._ok_wall >= self._ttl * 0.75:
+                self._lost()
+                return
+
+    def release(self) -> None:
+        self._stop.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=5)
+        leases.release(self._db, self.key, self.holder)
+
+
+class RunWorker:
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, run_id: str, *,
+                 loop_factory: LoopFactory | None = None,
+                 clock: Callable[[], float] = time.time,
+                 sleep: Callable[[float], None] = time.sleep,
+                 exit: Callable[[int], Any] = os._exit,
+                 idle_s: float = IDLE_S, poll_s: float = POLL_S,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S) -> None:
+        self._settings, self._db, self._run_id = settings, db, run_id
+        self._repo = TenantRepo(db, tenant_id)
+        self._urls = ServiceUrls.from_settings(settings)  # replaced by the run's own in _work
+        self._factory = loop_factory or (
+            lambda run, lookup, guard: make_loop(settings, run, lookup, guard, self._urls))
+        self._clock, self._sleep = clock, sleep
+        self._idle_s, self._poll_s = idle_s, poll_s
+        self._lease = _Lease(db, f"run:{run_id}", clock=clock, ttl=lease_ttl, renew_s=renew_s,
+                             on_lost=exit)
+        self._creds: Credentials | None = None
+
+    def scrub(self, text: str) -> str:
+        return self._creds.scrub(text) if self._creds is not None else text
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            return self._work()
+        except Exception as exc:
+            log.error("worker for run %s failed: %s", self._run_id,
+                      self.scrub(f"{type(exc).__name__}: {exc}"))
+            return EXIT_ERROR
+        finally:
+            self._lease.release()
+
+    # ---- the run -------------------------------------------------------------------------
+
+    def _work(self) -> int:
+        run = self._repo.get_run(self._run_id)
+        if run is None:
+            return EXIT_BUSY
+        try:
+            config = run_config(run)
+        except ValidationError:
+            # A stored config this version cannot read: retrying cannot help (review finding 6).
+            self._repo.set_worker_error(self._run_id, "the run's stored config is not valid")
+            return EXIT_OK
+        self._urls = ServiceUrls.for_run(run, self._repo, self._settings)
+        self._creds = Credentials(self._repo, self._settings.master_key,
+                                  allowed_for_run(run, config, self._settings, self._urls))
+        if run.state == "starting" and run.ledger_run_id is None:
+            self._fail_start(INTERRUPTED)
+            return EXIT_OK
+        if run.state == "start_failed":
+            self._refuse_all("the run did not start")
+            return EXIT_OK
+        try:
+            loop = self._factory(run, self._creds, self._lease.check)
+            if run.ledger_run_id is None:
+                lid = self._start(loop, config)
+                if lid is None:
+                    return EXIT_OK
+            else:
+                lid = run.ledger_run_id
+            self._drive(run, loop, lid)
+        except (CredentialMissing, CredentialUnreadable) as exc:
+            # No ledger write is possible without the run's secrets (04 INV-7): record it for
+            # the user in the run index; a new command tries again.
+            self._repo.set_worker_error(self._run_id, self.scrub(_credential_reason(exc)))
+        return EXIT_OK
+
+    def _start(self, loop: Loop, config: Any) -> str | None:
+        self._repo.set_run_state(self._run_id, "starting")
+        try:
+            lid = loop.start(config)
+        except (CredentialMissing, CredentialUnreadable) as exc:
+            self._fail_start(self.scrub(_credential_reason(exc)))
+            return None
+        except ValueError as exc:
+            self._fail_start(self.scrub(str(exc)))
+            return None
+        self._repo.set_run_state(self._run_id, "started", ledger_run_id=lid)
+        for cmd in self._repo.pending_commands(self._run_id):
+            if cmd.kind == "start":
+                self._repo.finish_command(cmd.id, "done")
+        return lid
+
+    def _fail_start(self, reason: str) -> None:
+        self._repo.set_run_state(self._run_id, "start_failed", start_error=reason)
+        self._refuse_all(f"the run did not start: {reason}")
+
+    def _refuse_all(self, reason: str) -> None:
+        for cmd in self._repo.pending_commands(self._run_id):
+            self._repo.finish_command(cmd.id, "refused", reason)
+
+    def _drive(self, run: RunRow, loop: Loop, lid: str) -> None:
+        stop = threading.Event()
+        errors: list[BaseException] = []
+        thread = threading.Thread(target=self._commands, args=(run, lid, stop, errors),
+                                  name="commands", daemon=True)
+        thread.start()
+        try:
+            idle_since = self._clock()
+            while not errors:
+                status = loop.status(lid).status
+                if status in TERMINAL:
+                    break
+                if status in DRIVABLE:
+                    loop.drive(lid)
+                    idle_since = self._clock()
+                    continue
+                if self._repo.pending_commands(self._run_id):
+                    idle_since = self._clock()
+                elif self._clock() - idle_since >= self._idle_s:
+                    break
+                self._sleep(self._poll_s)
+        finally:
+            stop.set()
+            thread.join(timeout=30)
+        if errors:
+            raise errors[0]
+        if thread.is_alive():
+            # Still inside a command: applying it again here could record it twice. The
+            # pending rows stay for the next worker (review finding 4).
+            log.warning("command thread of run %s did not stop; leaving commands", self._run_id)
+            return
+        self._apply_pending(loop, lid)  # commands that arrived as the run ended
+
+    def _commands(self, run: RunRow, lid: str, stop: threading.Event,
+                  errors: list[BaseException]) -> None:
+        """Applies user actions while `drive` runs, through a second `Loop` on its own ledger
+        connection; `loop` re-reads the ledger every step, so a pause lands mid-drive."""
+        try:
+            if self._creds is None:
+                raise RuntimeError("credentials are set up before driving")
+            loop = self._factory(run, self._creds, self._lease.check)
+            while not stop.is_set():
+                self._apply_pending(loop, lid)
+                stop.wait(self._poll_s)
+        except BaseException as exc:
+            errors.append(exc)
+
+    def _apply_pending(self, loop: Loop, lid: str) -> None:
+        for cmd in self._repo.pending_commands(self._run_id):
+            self._apply(loop, lid, cmd)
+
+    def _apply(self, loop: Loop, lid: str, cmd: CommandRow) -> None:
+        p = cmd.payload
+        try:
+            if cmd.kind == "pause":
+                loop.pause(lid, str(p.get("reason") or "paused by the user"))
+            elif cmd.kind == "resume":
+                loop.resume(lid)
+            elif cmd.kind == "stop":
+                loop.stop(lid)
+            elif cmd.kind == "checkpoint":
+                loop.decide_checkpoint(lid, str(p["checkpoint_id"]), p["decision"])
+            elif cmd.kind == "start":
+                raise ValueError("the run has already started")
+            else:
+                raise ValueError(f"unknown command {cmd.kind}")
+        except ValueError as exc:
+            self._repo.finish_command(cmd.id, "refused", self.scrub(str(exc)))
+            return
+        self._repo.finish_command(cmd.id, "done")
+
+
+def _credential_reason(exc: BaseException) -> str:
+    if isinstance(exc, CredentialUnreadable):
+        return "a stored credential cannot be decrypted; store it again"
+    return f"credential {exc.args[0] if exc.args else ''} is not stored"
+
+
+def _docker_unused_ssh(kind: str, config: dict[str, Any]) -> bool:
+    return kind == "ssh" and not config.get("use_docker") and bool(config.get("host_key"))
+
+
+class PreflightWorker:
+    """Runs `executor.preflight()` with only the executor's own credential (KB-021)."""
+
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, check_id: str, *,
+                 executor_factory: ExecutorFactory | None = None,
+                 clock: Callable[[], float] = time.time,
+                 exit: Callable[[int], Any] = os._exit,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S) -> None:
+        self._settings, self._check_id = settings, check_id
+        self._repo = TenantRepo(db, tenant_id)
+        self._factory = executor_factory or (
+            lambda row, lookup: build_executor(row, lookup, settings))
+        self._lease = _Lease(db, f"preflight:{check_id}", clock=clock, ttl=lease_ttl,
+                             renew_s=renew_s, on_lost=exit)
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            check = self._repo.check(self._check_id)
+            if check is None or check.state != "pending":
+                return EXIT_BUSY
+            row = self._repo.get_executor(check.executor_id)
+            if row is None:
+                self._repo.finish_check(check.id, {"ok": False, "problems": ["no executor"]})
+                return EXIT_OK
+            creds = Credentials(self._repo, self._settings.master_key,
+                                set(credentials_named(row.kind, row.config)))
+            try:
+                report = self._factory(row, creds).preflight()
+                result = {"ok": report.ok,
+                          "problems": [creds.scrub(str(p)) for p in report.problems]}
+                if report.ok and _docker_unused_ssh(row.kind, row.config):
+                    # 08 §3 Onboarding step 3 (KB-035): offer containers when Docker answers.
+                    result["docker"] = ssh_hosts.docker_answers_via(
+                        ssh_chain.hops_of(row.config), creds)
+            except (CredentialMissing, CredentialUnreadable) as exc:
+                result = {"ok": False, "problems": [_credential_reason(exc)]}
+            except Exception as exc:
+                result = {"ok": False,
+                          "problems": [creds.scrub(f"{type(exc).__name__}: {exc}")]}
+            self._repo.finish_check(check.id, result)
+            return EXIT_OK
+        finally:
+            self._lease.release()
+
+
+CheckRunner = Callable[..., list[str]]
+
+
+def run_check(kind: str, request: dict[str, Any], value: str,
+              lookup: Callable[[str], str] | None = None) -> list[str]:
+    """One credential check (08 §3, KB-029). Returns the problems; empty means passed. A check
+    of a model on an SSH machine opens its tunnel with `lookup`'s SSH keys (KB-036)."""
+    if kind == "llm":
+        target: dict[str, Any] = {"provider": request["provider"], "model": request["model"],
+                                  "credential": "key"}
+        if request.get("base_url"):
+            target["base_url"] = request["base_url"]
+        if request.get("public_only"):
+            target["public_only"] = True
+        if request.get("ssh"):
+            target["ssh"] = request["ssh"]
+        table = prices({f"{request['provider']}/{request['model']}": request["price"]}
+                       if request.get("price") else {})
+        profile = {r: dict(target) for r in ("planner", "coder", "analyst")}
+        opened: list[Any] = []
+        if "ssh" in target:
+            if lookup is None:
+                return ["this check needs the SSH machine's keys"]
+            profile, opened = tunnels.open_tunnels(profile, lookup)
+        try:
+            report = LLM(profile, table, {"key": value}).preflight()
+        finally:
+            for t in opened:
+                t.close()
+        return list(report.problems)
+    if kind == "purescholar":
+        async def ps() -> None:
+            client = PureScholar(request["url"], value)
+            try:
+                await client.preflight()
+            finally:
+                await client.aclose()
+        asyncio.run(ps())
+        return []
+    if kind == "puretrace":
+        async def pt() -> None:
+            client = PureTrace(request["url"], value, "preflight", [])
+            try:
+                await client.preflight()
+            finally:
+                await client.aclose()
+        asyncio.run(pt())
+        return []
+    if kind == "source":  # a literature-source key: one search request (KB-039)
+        return literature_sources.check(str(request["group"]), value)
+    raise ValueError(f"unknown check kind {kind!r}")
+
+
+class CredentialCheckWorker:
+    """Checks one credential with only that credential decrypted (INV-2, KB-029)."""
+
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, check_id: str, *,
+                 runner: CheckRunner = run_check, clock: Callable[[], float] = time.time,
+                 exit: Callable[[int], Any] = os._exit,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S) -> None:
+        self._settings, self._check_id, self._runner = settings, check_id, runner
+        self._repo = TenantRepo(db, tenant_id)
+        self._lease = _Lease(db, f"credcheck:{check_id}", clock=clock, ttl=lease_ttl,
+                             renew_s=renew_s, on_lost=exit)
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            check = self._repo.credential_check(self._check_id)
+            if check is None or check.state != "pending":
+                return EXIT_BUSY
+            cred = self._repo.credential(check.credential_id)
+            if cred is None:
+                self._repo.finish_credential_check(check.id, {"ok": False,
+                                                              "problems": ["no credential"]})
+                return EXIT_OK
+            ssh = check.request.get("ssh") or {}
+            names = {cred.name, *(str(h["key_credential"]) for h in ssh.get("hops", []))}
+            creds = Credentials(self._repo, self._settings.master_key, names)
+            try:
+                extra = [creds] if ssh else []  # the tunnel's SSH keys (KB-036)
+                problems = [creds.scrub(str(p)) for p in self._runner(
+                    check.kind, check.request, creds(cred.name), *extra)]
+            except (CredentialMissing, CredentialUnreadable) as exc:
+                problems = [_credential_reason(exc)]
+            except Exception as exc:
+                problems = [creds.scrub(f"{type(exc).__name__}: {exc}")]
+            self._repo.finish_credential_check(check.id, {"ok": not problems,
+                                                          "problems": problems})
+            return EXIT_OK
+        finally:
+            self._lease.release()
+
+
+SearchRunner = Callable[[dict[str, Any], TenantRepo, Callable[[str], str], Settings],
+                        dict[str, Any]]
+
+
+def run_search(target: dict[str, Any], repo: TenantRepo, lookup: Callable[[str], str],
+               settings: Settings) -> dict[str, Any]:
+    """One model search (08 §3, KB-036). `target` was validated by the API."""
+    ports = [int(p) for p in target.get("ports", [])]
+    if target["kind"] == "this-computer":
+        return model_search.search_this_computer(ports).to_dict()
+    if target["kind"] == "machine":
+        mid = str(target["machine"])
+        conn = repo.resolve_machine(mid)
+        hops = [*conn.get("jumps", []),
+                {k: conn[k] for k in ("host", "user", "port", "key_credential", "host_key")}]
+        return model_search.search_machine(mid, hops, ports, lookup, settings.mode).to_dict()
+    key = lookup(str(target["credential"])) if target.get("credential") else ""
+    return model_search.search_url(str(target["url"]), key, settings.mode).to_dict()
+
+
+def search_credentials(repo: TenantRepo, target: dict[str, Any]) -> set[str]:
+    """Only what this search needs: the machine's SSH keys, or the URL's API key (INV-2)."""
+    if target.get("kind") == "machine":
+        conn = repo.resolve_machine(str(target["machine"]))
+        return {str(conn["key_credential"]),
+                *(str(h["key_credential"]) for h in conn.get("jumps", []))}
+    return {str(target["credential"])} if target.get("credential") else set()
+
+
+class SearchWorker:
+    """Looks for model servers and their models with only the keys the search needs."""
+
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, search_id: str, *,
+                 runner: SearchRunner = run_search, clock: Callable[[], float] = time.time,
+                 exit: Callable[[int], Any] = os._exit,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S) -> None:
+        self._settings, self._search_id, self._runner = settings, search_id, runner
+        self._repo = TenantRepo(db, tenant_id)
+        self._lease = _Lease(db, f"llmsearch:{search_id}", clock=clock, ttl=lease_ttl,
+                             renew_s=renew_s, on_lost=exit)
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            search = self._repo.get_search(self._search_id)
+            if search is None or search["state"] != "pending":
+                return EXIT_BUSY
+            target = search["target"]
+            try:
+                names = search_credentials(self._repo, target)
+            except (KeyError, ValueError) as exc:
+                self._repo.finish_search(search["id"], {"servers": [], "problems": [
+                    f"the SSH machine cannot be used: {exc}"]})
+                return EXIT_OK
+            creds = Credentials(self._repo, self._settings.master_key, names)
+            try:
+                report = self._runner(target, self._repo, creds, self._settings)
+                report["problems"] = [creds.scrub(str(p)) for p in report.get("problems", [])]
+            except (CredentialMissing, CredentialUnreadable) as exc:
+                report = {"servers": [], "problems": [_credential_reason(exc)]}
+            except Exception as exc:
+                report = {"servers": [], "problems": [creds.scrub(f"{type(exc).__name__}: {exc}")]}
+            self._repo.finish_search(search["id"], report)
+            return EXIT_OK
+        finally:
+            self._lease.release()
+
+
+ProfileRunner = Callable[[dict[str, dict[str, Any]], dict[str, dict[str, Any]],
+                          Callable[[str], str]], list[str]]
+
+
+def run_profile_check(roles: dict[str, dict[str, Any]], prices_: dict[str, dict[str, Any]],
+                      lookup: Callable[[str], str]) -> list[str]:
+    """`llm.preflight()` for every role of an expanded profile, through its tunnels (KB-037).
+    Returns the problems; empty means passed."""
+    names = {str(s["credential"]) for spec in roles.values()
+             for s in (spec, spec.get("fallback") or {}) if isinstance(s, dict)
+             and s.get("credential")}
+    profile, opened = (tunnels.open_tunnels(roles, lookup)
+                       if model_endpoints.ssh_credentials(roles) else (roles, []))
+    try:
+        report = LLM(profile, prices(prices_), {n: lookup(n) for n in names}).preflight()
+        return list(report.problems)
+    finally:
+        for t in opened:
+            t.close()
+
+
+class ProfileCheckWorker:
+    """Checks one LLM profile with only its credentials and SSH keys decrypted (INV-2,
+    KB-037)."""
+
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, check_id: str, *,
+                 runner: ProfileRunner = run_profile_check,
+                 clock: Callable[[], float] = time.time, exit: Callable[[int], Any] = os._exit,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S) -> None:
+        self._settings, self._check_id, self._runner = settings, check_id, runner
+        self._repo = TenantRepo(db, tenant_id)
+        self._lease = _Lease(db, f"profilecheck:{check_id}", clock=clock, ttl=lease_ttl,
+                             renew_s=renew_s, on_lost=exit)
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            check = self._repo.profile_check(self._check_id)
+            if check is None or check["state"] != "pending":
+                return EXIT_BUSY
+            profile = self._repo.get_profile(check["profile_id"])
+            if profile is None:
+                self._repo.finish_profile_check(check["id"], {"ok": False,
+                                                              "problems": ["no profile"]})
+                return EXIT_OK
+            # A proposed edit is checked as proposed; it is applied only if this passes
+            # (08 §3 Profile edits, KB-041).
+            proposal = check.get("proposal") or {}
+            roles_in = proposal.get("roles", profile.roles)
+            prices_in = proposal.get("prices", profile.prices)
+            try:
+                roles = {r: model_endpoints.expand(s, self._settings.mode,
+                                                   self._repo.resolve_machine)
+                         for r, s in roles_in.items()}
+            except (KeyError, ValueError) as exc:
+                self._repo.finish_profile_check(check["id"], {"ok": False, "problems": [
+                    f"the profile's SSH machine cannot be used: {exc}"]})
+                return EXIT_OK
+            names = {str(s["credential"]) for spec in roles.values()
+                     for s in (spec, spec.get("fallback") or {}) if isinstance(s, dict)
+                     and s.get("credential")} | set(model_endpoints.ssh_credentials(roles))
+            creds = Credentials(self._repo, self._settings.master_key, names)
+            try:
+                problems = [creds.scrub(str(p))
+                            for p in self._runner(roles, prices_in, creds)]
+            except (CredentialMissing, CredentialUnreadable) as exc:
+                problems = [_credential_reason(exc)]
+            except Exception as exc:
+                problems = [creds.scrub(f"{type(exc).__name__}: {exc}")]
+            self._repo.finish_profile_check(check["id"], {"ok": not problems,
+                                                          "problems": problems})
+            return EXIT_OK
+        finally:
+            self._lease.release()
+
+
+class LiteratureWorker:
+    """The literature-service task (08 §5 steps 3 and 12): decrypts only the LLM credentials
+    the bundled PureScholar uses, then starts it."""
+
+    def __init__(self, settings: Settings, db: Database, tenant_id: str, *,
+                 docker: bundled.Docker | None = None,
+                 clock: Callable[[], float] = time.time,
+                 exit: Callable[[int], Any] = os._exit, timeout_s: float = bundled.READY_TIMEOUT_S,
+                 lease_ttl: float = leases.LEASE_TTL_S, renew_s: float = leases.RENEW_S,
+                 sleep: Callable[[float], None] = time.sleep) -> None:
+        self._settings = settings
+        self._repo = TenantRepo(db, tenant_id, settings.master_key)  # seals the new key
+        self._starter = bundled.Starter(settings, self._repo, docker=docker, timeout_s=timeout_s)
+        self._sleep = sleep
+        self._lease = _Lease(db, f"literature:{tenant_id}", clock=clock, ttl=lease_ttl,
+                             renew_s=renew_s, on_lost=exit)
+
+    def run(self) -> int:
+        if not self._lease.acquire():
+            return EXIT_BUSY
+        try:
+            row = self._repo.bundled()
+            if row is None or row["state"] not in ("pending", "starting"):
+                return EXIT_BUSY
+            creds = Credentials(self._repo, self._settings.master_key,
+                                self._starter.credentials_needed())
+            try:
+                self._starter.run(creds, creds.scrub)
+            except (CredentialMissing, CredentialUnreadable) as exc:
+                self._repo.set_bundled("failed", _credential_reason(exc), expect=("starting",))
+            except Exception as exc:
+                self._repo.set_bundled("failed", creds.scrub(f"{type(exc).__name__}: {exc}"),
+                                       expect=("starting",))
+            # A model on this computer or an SSH machine: the bridge stays open while the
+            # backend uses it (08 §5 step 12, KB-036). Holding the lease meanwhile is safe:
+            # a restart request ends the wait first.
+            self._starter.keep_open(self._sleep)
+            return EXIT_OK
+        finally:
+            self._lease.release()
+
+
+def main(argv: list[str]) -> int:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
+    if len(argv) != 3 or argv[0] not in ("run", "preflight", "credcheck", "llmsearch",
+                                          "profilecheck", "literature"):
+        print("usage: python -m app.worker run|preflight|credcheck|llmsearch|profilecheck|"
+              "literature "
+              "<tenant> <id>",
+              file=sys.stderr)
+        return EXIT_ERROR
+    settings = Settings.from_env()
+    db = Database(settings.app_db)
+    kind, tenant_id, item = argv
+    if kind == "run":
+        return RunWorker(settings, db, tenant_id, item).run()
+    if kind == "credcheck":
+        return CredentialCheckWorker(settings, db, tenant_id, item).run()
+    if kind == "llmsearch":
+        return SearchWorker(settings, db, tenant_id, item).run()
+    if kind == "profilecheck":
+        return ProfileCheckWorker(settings, db, tenant_id, item).run()
+    if kind == "literature":
+        return LiteratureWorker(settings, db, tenant_id).run()
+    return PreflightWorker(settings, db, tenant_id, item).run()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))