minima-cli 0.8.0__tar.gz → 0.10.0__tar.gz

{minima_cli-0.8.0 → minima_cli-0.10.0}/.gitignore

@@ -29,3 +29,12 @@ docs/PLAN/
 # Meta files
 .codaph 
 .claude
+
+# macOS junk
+.DS_Store
+
+# accidental session dumps
+context_window.json
+
+# generated by /plan finalize during dev runs — never commit
+GROUND_TRUTH.md

{minima_cli-0.8.0 → minima_cli-0.10.0}/CHANGELOG.md

@@ -4,7 +4,87 @@ All notable changes to Minima are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/), and the project aims to follow
 [Semantic Versioning](https://semver.org/).
 
-## [0.8.0] - 2026-07-06
+## [0.10.0] - 2026-07-12
+
+### Fixed
+- **Ground-truth spine hardened end-to-end** (still opt-in via `MINIMA_TUI_GROUND_TRUTH=1`):
+  - Gate verdicts now carry the `rec_id` of the turn that minted them, so grounded
+    feedback can never be poisoned by a stale gate from an earlier prompt, and a
+    blocked step's red verdict is superseded by its retry (content-first flip
+    identity). Plans close when their last flip is verified, instead of haunting
+    every later turn.
+  - Step identity survives rewording (token-set matching), a step's baseline resets
+    when its `verify` command changes (no fabricated red→green), and resuming a run
+    re-adopts its active plan.
+  - Bash writes and sub-agent writes are attributed to the run; unattributed
+    ("blind") writes cap confidence at 🟡 instead of silently passing.
+  - Check runner: verify commands run in their own process group and are killed as a
+    group on timeout/abort (no orphaned children), run under a minimal env allowlist,
+    and honor `MINIMA_TUI_CHECK_TIMEOUT`.
+  - `/plan` council: injected findings are fenced as untrusted data, the whole plan
+    turn aborts cleanly on Esc (partial research kept), and each council round is
+    budget-metered ($0.25/round soft cap).
+- **Default path** (no flag required): the SQLite migration runner is race-safe across
+  concurrent sessions and self-heals a previously wedged DB; a bash tool timeout/abort
+  kills the whole process group; a throwing after-hook no longer wedges the agent loop;
+  the `task` tool is blocked in plan mode.
+- Gate-focus modal: keyboard verdict overrides (approve / reject / steer with a note)
+  reach the correct gate, and the TUI footer/overlay no longer overflow on narrow
+  terminals.
+
+### Added
+- With `MINIMA_LLM_JUDGE=1`, judge grading spend is now booked to the session wallet —
+  visible in `/cost` (as `judge overhead`), the footer, and enforced by `--budget` —
+  while staying out of feedback's `actual_cost_usd` so the cost model Minima learns
+  from stays clean.
+
+## [0.9.0] - 2026-07-10
+
+### Added
+- **Ground-Truth verification spine (experimental, opt-in via `MINIMA_TUI_GROUND_TRUTH=1`)** —
+  the agent's plan steps can carry `verify` shell commands; the harness runs them
+  (pre-work baseline → done-gate red→green), records every verdict in the SQLite ledger,
+  and derives a confidence tier that drives the UI (🟢 glide / 🟡 flag / 🔴 stop) plus
+  grounded, deterministic-over-judge feedback to Minima. Includes `/why` (inspect why the
+  harness trusts/distrusts a step), a plan footer strip, and red-gate override capture.
+  Off by default — with the flag unset the CLI behaves exactly as 0.8.x.
+- **`/plan` planning workflow (part of the same experimental gate)** — a planner persona
+  plus a design council of sub-agents that researches, drafts, critiques, and synthesizes
+  a `GROUND_TRUTH.md` design document (`/plan start · status · finalize · cancel`).
+  Without the flag, `/plan` stays the read-only toggle it always was.
+- Permission hardening for verify commands: every LLM-authored `verify` is shown verbatim
+  in the approval overlay (with an explicit "… +N more lines" marker when truncated), and
+  a stored "always allow" on todowrite never covers a verify command the user hasn't seen.
+
+### Changed
+- Transcript rendering is memoized (windowing + `memo()` message rows, render-time ref
+  mutations moved to effects) — typing no longer re-renders the whole transcript per
+  keystroke.
+- Repo-root `CLAUDE.md` added (agent-facing repo guide).
+
+## [0.8.0] - 2026-07-07
+
+### Fixed
+- **Learning loop now takes effect** — the client sends a stable `user_id` on every
+  recommend, so Minima's memory recall actually surfaces your prior outcomes and routing
+  can move off the cold-start prior. Previously no `user_id` was sent, so server-side
+  recall was always empty and `decision_basis` never left `prior`.
+- **Esc reliably aborts** — pressing Esc now cancels during the routing/recommend phase
+  too (it was a no-op there, so the model still ran), and an aborted turn no longer leaves
+  a dangling message that made the next prompt re-answer the previous one.
+- **Web search/fetch work without an Exa key** — `web_search` and `web_fetch` fall back to
+  DuckDuckGo when `EXA_API_KEY` is unset (they previously failed at call time); with a key
+  set they still prefer Exa.
+- **Rendering** — long lines (routing warnings, reasoning, tool output) are clipped inside
+  their bordered cells instead of drawing past the border; the `escalation_suggested`
+  internal hint is no longer leaked into the info line; the question and `/tree` overlays
+  reserve their height so they can't corrupt terminal scrollback.
+
+### Changed
+- Learning-loop write failures are now surfaced — a muted `ℹ learning loop: …` line in the
+  TUI and a `feedback_error` event in `--mode json` — instead of being silently swallowed.
+
+## [0.7.2] - 2026-07-06
 
 ### Added (TUI)
 - **Fullscreen renderer (new default)** — alternate screen buffer with the prompt glued

minima_cli-0.10.0/CLAUDE.md

@@ -0,0 +1,104 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this repo is
+
+A Python service + client SDK, and a separate TypeScript harness:
+
+- **`src/minima/`** — Minima: a **recommend-only** LLM cost-optimization service (FastAPI, Python 3.11+). It never proxies, runs, or caches an LLM call. It answers "which model should I run for this task?" and learns from the outcome you report back. Memory is backed by [Mubit](https://docs.mubit.ai) and is the *only* external dependency of the core recommender. Hosted at `api.minima.sh`; deployed from GitHub releases (publishing a release triggers the prod deploy AND the CLI binary build).
+- **`client_sdk/minima_client/`** — bundled sync + async Python client. Ships in the same wheel as `src/minima` (`minima-cli` on PyPI).
+- **`packages/tui/`** — `minima-tui`: the **active harness** — a TypeScript/Ink agent runtime + TUI run on **Bun** (`bun:sqlite`, `>=1.2`). This is the `minima` CLI users install via Homebrew (`brew tap mubit-ai/minima`). Same `route → run → judge → feedback` loop against the `/v1/*` contract, with its own SQLite persistence spine (`src/db/minima_db.ts`, event-sourced, WAL, append-only migrations) and a `BudgetLedger`. Separate toolchain — not part of the Python wheel or `uv`. The version of record is `packages/tui/package.json` (kept in lockstep with `pyproject.toml`).
+
+### The Minima loop (respect this everywhere)
+
+```
+recommend  ->  run the model yourself  ->  judge quality  ->  feedback
+```
+
+Minima never runs the model. Feedback MUST carry realized `input_tokens` / `output_tokens` / `actual_cost_usd` / `latency_ms` so the cost basis can climb `estimate -> observed -> rescaled` — the single biggest accuracy lever. Do **not** echo Minima's own `est_cost_usd` back as the actual cost. Outcome thresholds: `success >= 0.8`, `partial >= 0.4`, else `failure`.
+
+## Commands
+
+`uv` is the package manager for the Python side. Common targets (see `Makefile`):
+
+```bash
+make install     # uv sync --extra dev
+make run         # local Minima on :8080 (uvicorn, --reload); interactive docs at /docs
+make test        # unit + integration, hermetic/offline: pytest -m "not live and not eval"
+make lint        # ruff check + mypy src/minima
+make fmt         # ruff check --fix + ruff format
+make live        # pytest -m live  — needs a running Mubit (make run-mubit in the Mubit repo)
+make eval        # pytest -m eval  — slow offline RouterBench savings evaluation
+make seed        # minima-seed (LIMIT=, LANE= overridable) — load cold-start memory
+uv run pytest tests/unit/test_foo.py::test_bar   # single test
+```
+
+Test markers (`pyproject.toml`): `live` = requires a running Mubit; `eval` = slow offline RouterBench. Default `make test`/`pytest` excludes both.
+
+The **TypeScript harness** (`packages/tui/`) has its own Bun toolchain (also exposed as `make tui-*` targets):
+
+```bash
+cd packages/tui && bun install
+bun test          # hermetic (mock fetch + faux provider + in-process mock service)
+bun run check     # tsc --noEmit
+bun run lint      # biome check src
+bun run format    # biome format --write src
+bun run build     # -> dist/minima native binary (bun build --compile)
+```
+
+## Testing constraints (important)
+
+- **Offline tests MUST stay hermetic.** `tests/conftest.py` has an autouse fixture that neutralizes `.env` — never rely on ambient credentials in a non-`live` test.
+- For service tests: `tests/factories.py:FakeMemory` + `create_app(...)` + `TestClient` gives a full in-process app with no Mubit.
+- For TUI tests: register a faux provider/model and mock `fetch` — see existing `packages/tui/tests/*.test.ts` for the pattern. No test may hit the network or spend money.
+
+## Conventions
+
+Python (`src/minima`):
+
+- `from __future__ import annotations` at the top of every module.
+- Pydantic v2 `BaseModel` for serializable schemas; `@dataclass(slots=True)` for internal types; `StrEnum` for enums; `Protocol` for seams (`Memory`, `Reasoner`, `QualityJudge`).
+- Async-first. Bridge sync SDKs (Mubit) off the event loop via `anyio`/threadpools.
+- Logging: `structlog` via `get_logger("minima.<sub>")`.
+- ruff `line-length=100`, target `py311`, lint set `E,F,I,UP,B,C4`. **Never break the hot path**: bookkeeping/feedback failures are logged-and-swallowed.
+
+TypeScript (`packages/tui`): biome for lint/format, strict tsc, Bun APIs (`bun:sqlite`, `Bun.write`) are fine. **Do NOT add the `openai` SDK** — the OpenAI-compatible provider speaks raw HTTP. Routing must stay bypassable (offline/no-Minima still works, with no feedback sent).
+
+- **No comments unless asked.** Match existing style.
+
+## Architecture — the service (`src/minima/`)
+
+Request flow (`api/routers/recommend.py` → `recommender/engine.py`):
+
+1. **classify** (`recommender/classify.py`) — derives task features/type from the prompt (regex + signal tables).
+2. **recall** (`memory/adapter.py`) — the *only* Mubit touchpoint; pulls similar past `task → model → outcome` records using Mubit's server-side embeddings (no local embedding model).
+3. **aggregate + score** (`recommender/aggregate.py`, `score.py`) — ranks candidates by *real* cost. The cost basis is one of three tiers chosen for the whole candidate set: `estimate` (catalog prices, cold start) → `observed` (median realized $/call) → `rescaled` (this request's input priced + observed output behavior).
+4. **escalation** (`recommender/escalation.py`) — when memory is thin/conflicting, optionally consult a cheap-LLM reasoner (`llm/`, off by default, `MINIMA_REASONER_PROVIDER`).
+5. **feedback** (`api/routers/feedback.py`) — writes the outcome back to Mubit, reinforcing memory.
+
+Other packages: `catalog/` (model cost + capability priors), `tenancy/` (multi-tenant runtime — many orgs, per-org Mubit, one deployment; auth is pass-through: the client's Mubit key IS the credential), `seeding/` (`minima-seed` CLI for cold-start history), `schemas/` (Pydantic request/response models — the wire source of truth; every field must also land in the TS mirror `packages/tui/src/minima/schemas.ts`).
+
+Everything external is behind a Protocol so it can be faked in tests. The server stack (fastapi/uvicorn/psycopg2/redis) lives in the `[server]` extra — the core recommender + client is dependency-light on purpose.
+
+## Architecture — the harness (`packages/tui/`)
+
+`MinimaAgent.promptRouted(task)` = **route** (`src/minima/router.ts` → `POST /v1/recommend`) → **run** (agent loop in `src/agent/`, streams the model, executes tools, before/afterToolCall hook stacks, abort via AbortSignal) → **judge** (`src/minima/judge.ts`, abstains unless `MINIMA_LLM_JUDGE=1`) → **feed back** (`POST /v1/feedback` with realized cost). Layers: `src/ai/` (multi-provider LLM API + `faux` test provider) · `src/agent/` (loop, tools, hooks) · `src/minima/` (router, runtime, meter, budget, spawn/sub-agents, ground truth) · `src/tools/` (bash/edit/read/write/grep/todowrite/websearch…) · `src/tui/` (Ink app) · `src/db/` (SQLite spine).
+
+**Ground-Truth verification spine** (shipped 2026-07, `docs/ground-truth-build-guide.md`): behind `MINIMA_TUI_GROUND_TRUTH=1` (**off by default — the default path must behave exactly as before**). When on: plan steps carry `verify` shell commands, the harness runs them (baseline → done-gate red→green), verdicts land in the `gates` ledger (migrations v3–v5), a confidence tier drives the UI (🟢 glide / 🟡 flag / 🔴 stop), and grounded outcomes stamp `routing_decisions` for deterministic-over-judge feedback. `/plan` (planner persona + design council + `GROUND_TRUTH.md` output) is part of this gate. Anything that changes default-path behavior does NOT belong behind guidance text — gate it on `config.groundTruth`.
+
+Design principles that constrain any work in this direction:
+
+- **State in the DB, projections in the context** — anything that must survive scroll/compaction/restart/a lying model lives in the ledger; the model sees only a compact re-injected projection.
+- **Enforcement in the dispatcher, guidance in the prompt** — plan/permission/gate guarantees are enforced by harness code at tool-dispatch or turn boundaries, never by prompt text (which is bypassable).
+- **Feedback truth** — realized usage only; no fabricated quality (gate verdict → outcome label only; judge abstention stays `null`); `verified_in_production` only from user/repo/ci-origin gates, never agent-authored ones.
+- **Recommend-only server stands** — plan state is never server-authoritative.
+- **Append-only migrations** (shipped batch strings never edited) · **`rec_id` discipline** as the sole local↔server↔Mubit join key · **propensity integrity** (no client-side re-ranking; pins are pre-request candidate assembly, never post-hoc overrides).
+
+## Entry points
+
+Python (`pyproject.toml [project.scripts]`): `minima-seed` · `minima-calibration-report`. Server app: `minima.main:app`. The `minima` CLI/TUI is NOT a Python entry point — it is the compiled Bun binary from `packages/tui`, distributed via Homebrew.
+
+## Docs
+
+Deep docs live in `docs/` (concepts, api-reference, configuration, multi-tenancy, operations, seeding, ground-truth-build-guide). All service config is via environment variables — see `.env.example` and `docs/configuration.md`; the only required value (single-tenant mode) is `MUBIT_API_KEY`. Never commit `.env`/`.env.harness`; never print secret values.

{minima_cli-0.8.0 → minima_cli-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: minima-cli
-Version: 0.8.0
+Version: 0.10.0
 Summary: Minima CLI: cost-aware LLM model routing — recommend cheaper models, backed by Mubit memory.
 Project-URL: Homepage, https://docs.minima.sh
 Project-URL: Documentation, https://docs.minima.sh

minima_cli-0.10.0/bench/.gitignore

@@ -0,0 +1 @@
+artifacts/

minima_cli-0.10.0/bench/README.md

@@ -0,0 +1,374 @@
+# bench/ — Minima harness scripted test flows
+
+Scripted end-to-end flows that drive the **installed** `minima` binary (Homebrew, currently
+0.7.1) the way a user would — headless one-shots and full interactive PTY sessions — and
+assert on the artifacts the harness actually produces: the SQLite DB, the transcript, git
+state, and server responses. Born from the 2026-07-06 demo/test-flow research plan
+(14-agent inventory + web research + 3 designs + 2 adversarial critiques).
+
+## Why this exists
+
+We want to demo every Minima harness feature and regression-test the full loop
+(recommend → run → judge → feedback → memory) with **long conversational sessions on
+disposable test dirs**, one session id per flow. No such infrastructure existed: the repo
+had only in-process bun tests (faux provider) and two live smoke scripts.
+
+## Layout
+
+```
+bench/
+  driver/
+    rig.ts          PTY rig — spawns the TUI in a real pseudo-terminal via Bun.spawn
+                    {terminal} (Bun >= 1.3.5), accumulates output, regex wait/assert,
+                    types + submits prompts and slash commands
+    headless.ts     one-shot runner for --print / --mode json invocations
+    mock_server.ts  scripted /v1 Minima server (recommend/feedback/models/capabilities)
+                    with full request capture — makes offline/reconnect + ladder flows
+                    deterministic and free
+  assert/
+    db.ts           readonly bun:sqlite helpers over the harness DB
+                    (runs / routing_decisions / budgets / budget_events / tool_calls)
+    check.ts        assertion collector → per-flow PASS/FAIL report
+  flows/
+    f1_headless.ts          F1: headless CLI battery (exit codes, routed/pinned/offline
+                            rows, JSON-event contract, budget deny)
+    f9_offline_reconnect.ts F9: offline fallback → mock server comes up → /reconnect →
+                            routed=server (deterministic, no config writes)
+    f4_cost_budget.ts       F4: PTY session — /budget set/mode ladder (warn → enforce
+                            refusal → warn → shadow), /clear display-only lock-in,
+                            /cost, context retention
+  gen/
+    materialize.ts    template + patches → agent-visible working copy (git HEAD = seeded
+                      state; hidden tests applied only at grade time)
+    validate_task.ts  the 5 execution gates every task must pass (SWE-bench-Verified
+                      style): template green → bug invisible to public suite → hidden
+                      tests fail pre-fix → oracle restores green → statement >= 40 words
+    calibrate.ts      k-run empirical difficulty calibration: pinned cheap/frontier arms
+                      through the installed binary, cheat-guarded hidden-test grading,
+                      appends bench/tasks/calibration.jsonl
+    build_index.ts    task dirs (+ calibration rates when present) → bench/tasks/tasks.jsonl
+  fixtures/templates/ clean fixture repos (py-cli, ts-api, js-lib, katas; _example is the
+                      canonical reference the validator smoke uses)
+  tasks/<repo>/<id>/  task instances: task.json, bug.patch?, hidden_tests.patch, oracle.patch?
+  run.ts            entry point: `bun bench/run.ts f1 f9 f4`
+  artifacts/        (gitignored) per-run scratch dirs, DBs, transcripts on failure
+```
+
+## The session recipe (validated live 2026-07-06)
+
+There is **no CLI flag** to pin or resume a session id in 0.7.1. The only supported
+recipe, validated against the installed binary:
+
+```sh
+cd <disposable-dir>                       # NEVER a real repo: headless mode has ZERO
+                                          # permission gating (hook only wired in the TUI)
+MINIMA_DB_PATH=<flow>.db \
+MINIMA_NAMESPACE=bench-<flow>-<id> \
+minima -nt --budget 0.05 --budget-enforce -p "..."
+sqlite3 <flow>.db "SELECT run_id,status FROM runs"   # ← captured session id
+```
+
+One DB path per flow (one per case for headless batteries) removes all "newest run"
+ambiguity. Multi-turn = PTY-drive the interactive TUI; `/resume <FULL run_id>` after
+restart (exact match — the picker's 12-char ids are display-only).
+
+## Task dataset (Phase B, 2026-07-06)
+
+**33 execution-validated tasks** across 4 purpose-built fixture repos (34 counting the
+`_example` reference), authored by a 4-agent fan-out and independently re-validated
+(`bun bench/gen/validate_task.ts --all` → 34/34):
+
+| repo | template | tasks | mix |
+|---|---|---|---|
+| `py-cli` ("taskman") | ~1.2k LOC argparse CLI, 60 pytest tests | 8 | 3 easy + 3 medium + 1 hard (cross-module schema-migration data loss) + 1 **trap** (168-word panic statement, 1-line fix) |
+| `ts-api` ("linkbox") | ~1.6k LOC bun HTTP handlers, 60 tests | 7 | 3 feature-adds (hidden-test spec) + 3 medium bugfixes + 1 **trap** (casual statement, diagnosis spans store/persist/stats) |
+| `js-lib` ("datakit") | ~0.8k LOC zero-dep utils, 50 tests, fastest suite | 6 | 2 easy + 2 medium + 1 feature + 1 hard (CSV round-trip state machine) |
+| `katas` | 12 single-file stubs (4 py / 4 ts / 4 js) | 12 | all trivial → the unambiguous cheap tier |
+
+**Difficulty labels are EMPIRICAL** (k=5 × cheap(haiku)/frontier(sonnet) arms, 330
+attempts, $17.90, 0 errors, 0 cheats — `calibration.jsonl`, report via
+`gen/calibration_report.ts`). The 2026-07-06 full pass rewrote the structural bins:
+
+- **haiku solves 30/33 tasks at ~100%** — well-specified, symptoms-only bugfixes in
+  1-2k-LOC repos are simply within a 2026 cheap model's reach, including the authored-
+  hard 72-line cross-module migration (pc-007: 5/5 cheap). 12 authored-medium/hard
+  tasks were NO-SIGNAL (both arms 5/5) → re-labeled easy/cheap with
+  `authored_difficulty` preserved.
+- **jl-006 is the perfect hard task**: cheap 0/5, frontier 5/5 (CSV state-machine
+  round-trip) — the escalation-value demo in one row.
+- **ta-004 is a cheap-beats-frontier anomaly**: cheap 4/5, frontier 0/5 — sonnet
+  consistently fails a task haiku solves (kept medium, route=cheap, flagged for
+  failure-mode investigation; excellent routing-demo material).
+- Final spread: 12 trivial / 19 easy / 1 medium / 1 hard; expected_route: 32 cheap,
+  1 frontier. **Consequence for the demo:** this dataset proves the savings story
+  (quality parity at cheap prices) overwhelmingly, but differentiated escalation
+  routing rests on jl-006/ta-004 alone — a "genuinely hard" authoring round (vaguer
+  statements, composite multi-bug patches, larger repos) is queued for the demo phase.
+
+Anti-leakage: agents under test see ONLY the materialized template (+ seeded bug) and
+the `problem_statement` — never `task.json` (whose `notes` describe the defect), never
+`hidden_tests.patch`. Statements are symptoms-only for bugfixes (audited: no defect
+file/function names, no fix quotes); feature statements specify routes/signatures.
+
+## Savings A/B (Phase D, 2026-07-06) — the honest headline
+
+Paired protocol per router-eval best practice: all 33 tasks fresh-executed twice
+(identical prompts, fresh checkouts, hidden-test graded) — arm A pinned
+`claude-opus-4-8` (all-premium baseline), arm B **routed** via api.minima.sh
+(namespace `bench-ab-v1`, feedback live). Runner: `gen/savings_ab.ts` · report:
+`gen/savings_report.ts` · raw log: `tasks/savings_ab.jsonl`. Total run cost $8.71.
+
+```
+1. COST   premium: $7.89 total, $0.2546/completed
+          routed:  $0.82 total, $0.0302/completed   → 89.7% saved
+2. PARITY premium 31/33 (93.9%) vs routed 27/33 (81.8%)
+          both=26 · premium-only=5 · routed-only=1 · neither=1
+          McNemar exact p=0.219 — no significant difference at n=33
+3. ROUTED DISTRIBUTION: gemini-2.5-flash × 33 (cold-prior argmin; no escalation)
+```
+
+Present all three together, never % saved alone; state the n=33 parity bound (±10-15pp).
+Noteworthy pairs: jl-003 is a cheap-beats-premium case (flash PASS $0.008, opus FAIL
+$0.40); the 5 routed-only failures are all tasks haiku solved 5/5 in calibration —
+i.e. flash-specific misses on a cold prior, exactly the signal the learning loop
+should absorb (F12 tests that transition).
+
+**Finding — the server's premium counterfactual understates reality ~10×**: summed
+`all_premium_cost_usd` estimates for the routed arm came to $0.77 vs the MEASURED
+premium arm's $7.89. Single-call token estimates don't capture multi-turn agentic
+tool loops. Real savings are LARGER than /v1/savings claims, but the estimate is not
+a credible counterfactual for agentic workloads — measured paired baselines are.
+(Feeds the GT-8 observed-best/IPS work.)
+
+## Learning loop (F12) — GREEN (took TWO fixes: the key AND user_id)
+
+`flows/f12_learning_loop.ts` (live-lane, ~$0.60/run): first a free WRITE-HEALTH probe
+(direct recommend→feedback, hard check on the feedback BODY's `accepted` field,
+early-exit on failure), then warms a fresh namespace with 8 judged kata runs (separate
+sessions) and re-runs 4 of the SAME tasks as probes in warm-vs-cold namespaces.
+
+**Verified PASS 2026-07-07 (5/5 hard + 4/4 soft)** against `api.minima.sh` with a binary
+carrying BOTH fixes below — cold probes route `prior`, all four warm probes flip to
+`memory`. It needed two independent fixes; #1 alone left the flip still red.
+
+- **FIX 1 — writes (the key).** Minima uses pass-through auth (`src/minima/api/auth.py`):
+  the `MUBIT_API_KEY` the harness/binary send as `Authorization: Bearer` IS the key
+  Minima uses against Mubit. Repo `.env` ships the LOCAL dev key `mbt_local_admin` (instance
+  `local`) that prod can't reach → `accepted=false`/`memory_write_failed`. `.env.harness`
+  has the DEPLOYED key that writes. Wired via `driver/env.ts` `loadBenchEnv()` (layers
+  `.env.harness` over Bun's auto-loaded `.env`, forcing `MUBIT_API_KEY`/`MINIMA_URL`/
+  `MINIMA_API_KEY`; called from `run.ts`, `gen/savings_ab.ts`, `f12()`). Escape hatches:
+  `BENCH_NO_HARNESS_ENV=1`, `BENCH_MUBIT_API_KEY=…`.
+- **FIX 2 — recall (user_id), PR #90.** Fixing #1 uncovered this: prod recall is scoped
+  by `user_id`, and `router.ts` recommend omitted it → the server surfaced nothing and
+  `decision_basis` never left `prior`, so the binary couldn't recall its own writes.
+  Direct A/B: single write+read in one namespace flips to `basis=memory,ev=1` WITH a
+  user_id, stays `prior,ev=0` WITHOUT. PR #90 sends the stable `memorySession`
+  (`namespace ?? repo identity`) as `user_id`. **The installed brew binary does NOT have
+  #90 yet** — until a release ships it, run F12 against a local build:
+  `cd packages/tui && bun run build` then
+  `BENCH_MINIMA_BIN="$PWD/dist/minima" bun bench/run.ts f12`.
+- Still worth keeping: the write-health probe (asserts on the BODY — HTTP 200 hid the
+  write failure) and PR #84 (the binary swallowed the rejection in a write-only
+  `lastFeedbackError`; #84 surfaces it as a TUI line + `--mode json` `feedback_error`).
+- Optional server hardening (defense-in-depth): the SDK treats a done ingest-job with
+  `writes[].success=false` as success; `recall.degraded` is parsed but never read.
+- Also noteworthy: routing drifted flash→pro for identical kata prompts within hours
+  (catalog/prior drift) — validates the suite-wide rule of never asserting model ids.
+
+## Running
+
+```sh
+bun bench/run.ts            # all flows
+bun bench/run.ts f1 f9      # a subset
+
+bun bench/gen/validate_task.ts --all        # re-validate every task (local, free)
+bun bench/gen/calibrate.ts --tasks all --arms cheap,frontier --k 5   # full calibration (costed)
+bun bench/gen/build_index.ts                # rebuild tasks.jsonl (+ measured rates)
+```
+
+Live-lane cost: the full Phase A suite spends well under $0.10/run (trivial prompts,
+mock server wherever determinism matters). Exit code is non-zero if any hard check fails;
+per-flow transcripts land in `bench/artifacts/scratch/<flow>-<ts>/transcript.txt`.
+
+Status (2026-07-06): **full suite 7/7 flows PASS — f1 30/30 · f9 11/11 · f4 16/16 ·
+f5 5/5 · f6 6/6 · f7 13/13 · f10 5/5 (86 hard checks)** against installed 0.7.1 +
+hosted api.minima.sh; ~3.3 min, ~$0.15/run.
+
+## Findings log
+
+Datestamped facts discovered while building/running the flows — kept current; newest first.
+
+### 2026-07-07 — memory loop GREEN: TWO bugs (key + user_id); F12 PASS 5/5
+- Two independent bugs; fixing the first uncovered the second. Neither is a server bug.
+- **BUG 1 (writes) — the key.** Minima is pass-through auth (`api/auth.py`): the client's
+  `Authorization: Bearer <mubit_key>` IS the key used against Mubit. A/B (same endpoint +
+  payload, only the key differs): repo `.env` `mbt_local_admin` →
+  `accepted=false`/`memory_write_failed`; `.env.harness` deployed key → `accepted=true`,
+  real record_id. Wired via `driver/env.ts` `loadBenchEnv()` (forces `.env.harness`'s
+  routing keys over Bun's auto-loaded `.env`; `run.ts`/`savings_ab.ts`/`f12()`;
+  `BENCH_NO_HARNESS_ENV=1` / `BENCH_MUBIT_API_KEY=…` escape hatches).
+- **BUG 2 (recall) — user_id, PR #90.** After BUG 1, F12 warm probes STILL all `prior`.
+  A/B: single write+read in one namespace flips to `basis=memory,ev=1` WITH a `user_id`,
+  stays `prior,ev=0` WITHOUT. `router.ts` recommend omitted `user_id` → prod recall
+  (user-scoped) surfaced nothing → basis never left `prior`; the binary couldn't recall
+  its own writes (affects every user, any key). PR #90 sends the stable `memorySession`
+  as `user_id`.
+- **F12 PASS 5/5 (+4/4 soft)** against api.minima.sh with a binary carrying #90: cold
+  probes `prior`, all 4 warm probes `memory`. Installed brew binary lacks #90 → run F12
+  against a local `packages/tui` build until a release ships it.
+- The earlier "blocked SERVER-side" framing was wrong on cause, right on symptom; the
+  write-health probe + PR #84 (surface the swallowed rejection) stay useful.
+
+### 2026-07-06 (later) — memory-loop root cause + write-health detector
+- **Every memory write is rejected by the server**: direct `POST /v1/feedback` returns
+  HTTP 200 `accepted=false, record_id=null, warnings=["memory_write_failed"]`. The
+  HTTP status is useless as a health signal — assert on the BODY.
+- The binary knew and stayed silent: the rejection landed in `lastFeedbackError`,
+  which no code read (write-only diagnostics). PR #84 surfaces it (TUI info line +
+  `--mode json` `feedback_error` event).
+- F12 now opens with a direct-API write-health probe and exits early on failure:
+  fails in <1s at the true cause instead of ~5min/$0.60 at the basis=prior symptom.
+- `memory_unavailable` on recommends is INTERMITTENT — its absence during one run must
+  not be read as "recall healthy" (it briefly sent this investigation toward evidence
+  thresholds).
+- 5-agent code-trace of the server path (journal in the session workflow dir):
+  `reinforced_entry_ids` echoes recommend-time recall neighbors, so `[]` during
+  warm-up is expected and diagnostic of nothing; `MEMORY_WEIGHT_MIN=0.0` means ONE
+  retrieved outcome flips basis to memory — no threshold story; lane/namespace
+  computed once and reused on both paths — no scoping mismatch. The break is at the
+  Minima↔Mubit ingest boundary (and the SDK treats a done-ingest-job with
+  `writes[].success=false` as success — silent-failure hole worth a server PR).
+
+### 2026-07-06 — v0.7.2 release compat + memory-loop status
+- **Suite green on both binaries**: installed 0.7.1 and the v0.7.2 release asset
+  (`BENCH_MINIMA_BIN=<path> bun bench/run.ts`). Version-tolerant assertions cover the
+  0.7.2 renames: `· working`→`· running` (isIdle accepts both), denial copy
+  ("Permission denied for X" → "The user declined the X call"), `/fork`+`/clone`
+  honest copy, and models satisfying an "edit" ask with the new `apply_patch` tool.
+- v0.7.2 plan-mode correctly blocks `apply_patch` (the #77 review fix) — verified live
+  by F7 against the release binary.
+- **Memory loop: partially restored, still not closing.** The v0.7.2 prod redeploy
+  cleared the `memory_unavailable` recommend warning, but F12 still shows
+  reinforced_entry_ids never echoing and same-task warm-namespace probes routing on
+  `prior`. Needs server-side investigation of the feedback→Mubit write path; F12 is
+  the standing detector. (Possible contributor: one outcome per task may be below the
+  evidence threshold — a heavier warm phase, k=3 per task, is worth trying.)
+
+### 2026-07-06 — Phase C (flow suite) findings
+- **Child routing decisions DO persist correctly** — each sub-agent writes its own
+  routing_decisions row with `agent_id = <childId>` demuxing it from the lead (verified
+  live: 3 children with per-child tool_calls read/glob/bash + worktree isolation). Two
+  false alarms on the way were both harness-side: the assert library's SELECT omitted
+  agent_id, and rows become visible to external readers slightly after the lead's
+  (write-queue flush lag) — poll with a grace window, never point-read.
+- **The packages/tui test suite leaks git worktrees**: stale `/tmp/minima-wt-dirty-step-*`
+  and `/tmp/minima-wt-wt-step-*` dirs from spawn tests predate bench runs. Flows must
+  baseline `/tmp/minima-wt-*` pre-run and assert on the set DIFFERENCE; worth a small
+  cleanup fix in the tui test suite.
+- **In-cwd reads DO prompt** despite the `app.tsx:565` comment claiming "read/ls
+  auto-allow within cwd" — `permissions.ts:41` starts `allowedDirs` empty and nothing
+  pre-approves the cwd. Locked in by F7; either the comment or the behavior should change.
+- **Overlay answering must be effect-keyed**: a single keypress can race the overlay
+  lifecycle (deny → model instantly retries → new overlay paints inside the same poll
+  window). F7's driver retries each answer key until its observable effect (deny text,
+  disk change, transcript content) lands.
+- **Idle detection that survives repaints**: compare `lastIndexOf("· ready")` vs
+  `lastIndexOf("· working")` over the accumulated output — the last footer paint always
+  reflects current state. Needed before typing after DB-predicate turn ends (the visible
+  turn end precedes busy-clear by seconds; typed chars during the tail may not echo).
+- **Prompt-dictated task-tool JSON is genuinely nondeterministic**: the model sometimes
+  omits `isolation:"workdir"` even when asked; quoting the exact field name + a one-shot
+  in-session retry makes F5 reliable. The ladder (F10), by contrast, is fully
+  deterministic via the mock's recommendQueue + a present-but-invalid provider key.
+
+### 2026-07-06 — Phase B (dataset) findings
+- **Never `await` child stdout/stderr streams to completion after killing a process**:
+  the agent's grandchildren (pytest, bun test) inherit the pipe and hold it open past
+  the parent's death, hanging the collector indefinitely — this stalled the first
+  calibration batch. `sh()` now awaits `proc.exited` and gives the streams a 2s grace
+  race instead.
+- **k=1 calibration is a smoke, not a label**: single-attempt solves by the cheap arm
+  demoted ta-007 (with the audit's structural evidence agreeing) but cannot
+  distinguish "cheap solves 100%" from "cheap solves 40% and got lucky" — medium bins
+  especially need the full k=5 pass before routing accuracy is judged against them.
+- **Pinned calibration runs never touch the Minima server** — `--model X` bypasses
+  routing entirely, so difficulty calibration is pure provider spend with zero effect
+  on the routing namespace. Exactly what we want: calibration cannot pollute the
+  learning loop.
+- **Cheat-guard needs an artifact filter**: running pytest inside the attempt drops
+  `tests/__pycache__/*.pyc` as untracked files under `tests/`, which false-flagged the
+  very first calibration attempt as test-tampering. The guard now ignores
+  `__pycache__/.pyc/.pytest_cache/node_modules` before applying the tests/ rule.
+- Headless agent runs on seeded tasks work exactly as designed: haiku found and fixed
+  the `_example` seeded bug in ~23s for ~$0.01, graded by hidden tests post-hoc.
+- Go is not installed on this machine → the plan's `go-tool` fixture became `js-lib`
+  (zero-dep bun:test library, same fastest-suite role for future worktree/DAG flows).
+
+### 2026-07-06 — findings from making the flows green
+
+- **PRODUCT FINDING — offline turns bypass the budget ledger entirely.** An
+  offline-fallback turn runs the default model (real provider spend, `actual_cost_usd`
+  recorded on the decision row) but books NOTHING to the ledger: no reserve/reconcile
+  events, `spent_usd` stays 0, and `--budget-enforce` cannot refuse it (the reserve path
+  needs a positive server cost estimate; offline/pinned synthetic rows have est 0 and the
+  reserve block is routing-gated in runtime.ts). Locked in as F1's `offline-budget
+  lock-in` case — flips loudly if a fix lands. Follow-on: enforce-deny tests MUST route
+  through a server (bench uses the mock) or a transient offline fallback silently
+  neuters them.
+- **TUI drops Enter while busy — including the invisible post-turn tail.** After the
+  reply finishes streaming, judge + feedback + memory-write keep `busy` true for up to
+  several seconds; during that window typed characters still echo into the input box but
+  Enter is silently ignored (same `if (busy) return` guard family as the known abort bug
+  #83). Naive scripted typing concatenates commands in the input box. Driver fix:
+  `rig.submitUntil(text, effect)` — type once, then retry Enter until the command's
+  unique effect appears (fresh response regex or DB predicate). A retried Enter is
+  harmless (input is empty if the previous one submitted).
+- **Accumulated-output matching must use UNIQUE expected strings.** Ink repaints the
+  whole screen constantly, so previously-rendered content reappears after any
+  `mark()` — a `since` offset does NOT protect against matching stale repaints.
+  Assert on strings that appear exactly once per flow (include arguments/amounts:
+  `Budget set: $0.02 (warn mode)`), and prefer DB predicates over screen text.
+- **Live routing can transiently fall back offline** (cold-start recommend latency) even
+  when api.minima.sh is healthy — never build a hard check on "this turn will be
+  server-routed" unless the flow retries or uses the mock. F4 asserts all-turns-routed
+  as a flow-level invariant and accepts the rerun cost.
+- **Turn-end signal**: the `routing_decisions` row is the reliable "turn substantively
+  done" oracle (WAL lets a readonly reader poll while the TUI writes), but it lands
+  BEFORE busy clears — combine row-polling with submitUntil for the next input.
+- Boot state: default model is `gpt-4o-mini` before any routing; readiness marker is
+  the status-bar `· ready`; typing `/xyz` opens a commands suggestion box that does not
+  interfere with Enter-submission of a fully-typed command; `/exit` exits 0; `/clear`
+  clears only the transcript (locked in by F4: planted token still recalled after).
+- `/budget set <tiny>` renders `$0.00` (`toFixed(2)`) — match `Budget set: \$0\.00` for
+  sub-cent limits; the exhausted-refusal message is `budget exhausted: $X spent of $Y`.
+
+### 2026-07-06 — Phase A bring-up
+- **Bun 1.3.14 native PTY works** for driving the Ink TUI: `Bun.spawn(cmd, {terminal:
+  {cols, rows, data(term, chunk){...}}})`; the data callback signature is
+  `(Terminal, Buffer)`; write with `proc.terminal.write("...\r")`. No node-pty needed.
+- **Keys resolve from the global config store** (`~/.minima-harness/config.env` +
+  keychain) even in disposable cwds — that's what makes scratch-dir flows work with zero
+  per-flow env setup. Repo `.env.harness` has the same key names if explicit env is wanted.
+- **`/reconnect` re-reads `process.env.MINIMA_URL`** (`config.ts:75` — env wins over cfg).
+  So the deterministic offline→online story is: launch with `MINIMA_URL=http://127.0.0.1:
+  <port>` while nothing listens (offline fallback), start the mock server on that port,
+  `/reconnect`. `--offline` itself is a one-way door (sets cfg.minimaUrl='' and no env var
+  exists to restore) — never use it for reconnect flows.
+- **`/budget set` preserves accumulated spend** (new ledger, same `session:<run_id>`
+  scope, ON CONFLICT keeps `spent_usd`) and **preserves the current mode**. Deterministic
+  enforce-refusal: spend under warn first, tighten the limit below observed spend, then
+  `/budget mode enforce` — the next prompt is refused at the exhausted gate
+  (`budget exhausted: $X spent of $Y`) *before any provider spend*, and no
+  routing_decisions row is written for the refused prompt.
+- **Smoke economics**: one routed trivial turn ≈ $0.004 (sonnet via prior basis, argmin).
+
+### 2026-07-06 — from the research inventory (file:line ground truth)
+- Do not script/demo: `/undo` (no-op `git checkout --`), `/fork`+`/clone` (stubs),
+  `--thinking` (parsed, never applied), Ctrl+R confirm (cosmetic), Esc/Ctrl+C abort
+  (dead code, task #83), `/tree` cost/status rows (stuck "running" — screenshot during
+  run only), `save$` column (baselineModelId has no setter).
+- `--mode json` events are lossy (no tool args / model / cost) → SQLite is the primary
+  oracle; the JSON stream is asserted only as a frozen vocabulary contract.
+- Headless `--print`/`--mode json`: single prompt per process, budget events on stderr,
+  exit 1 on provider hard-failure/offline-empty, exit 2 on usage errors.

minima_cli-0.10.0/bench/assert/check.ts

@@ -0,0 +1,41 @@
+/** Assertion collector: gathers named checks, prints a per-flow report, computes PASS/FAIL. */
+
+export interface CheckResult {
+  name: string;
+  ok: boolean;
+  soft: boolean;
+  detail?: string;
+}
+
+export class Checks {
+  results: CheckResult[] = [];
+  constructor(readonly flowId: string) {}
+
+  check(name: string, ok: boolean, detail?: string): boolean {
+    this.results.push({ name, ok, soft: false, detail });
+    console.log(`  ${ok ? "✓" : "✗"} ${name}${!ok && detail ? ` — ${detail}` : ""}`);
+    return ok;
+  }
+
+  /** Recorded but never gates the flow (nondeterministic/live-dependent signals). */
+  soft(name: string, ok: boolean, detail?: string): boolean {
+    this.results.push({ name, ok, soft: true, detail });
+    console.log(`  ${ok ? "✓" : "~"} [soft] ${name}${!ok && detail ? ` — ${detail}` : ""}`);
+    return ok;
+  }
+
+  get passed(): boolean {
+    return this.results.filter((r) => !r.soft).every((r) => r.ok);
+  }
+
+  summary(): string {
+    const hard = this.results.filter((r) => !r.soft);
+    const soft = this.results.filter((r) => r.soft);
+    const failedHard = hard.filter((r) => !r.ok);
+    return (
+      `${this.flowId}: ${this.passed ? "PASS" : "FAIL"} — ${hard.length - failedHard.length}/${hard.length} checks` +
+      (soft.length ? ` (+${soft.filter((r) => r.ok).length}/${soft.length} soft)` : "") +
+      (failedHard.length ? `\n  failed: ${failedHard.map((r) => r.name).join(", ")}` : "")
+    );
+  }
+}