rigorloop 0.1.0__tar.gz → 0.2.0__tar.gz

{rigorloop-0.1.0 → rigorloop-0.2.0}/.gitignore

@@ -221,3 +221,5 @@ __marimo__/
 src/rigorloop/_version.py
 runs/
 .uv-cache/
+stage_markdowns/
+stage_markdowns

rigorloop-0.2.0/AGENTS.md

@@ -0,0 +1,121 @@
+# AGENTS.md
+
+Orientation for coding agents working in this repository. Read this first, then
+read **[`CODING_STYLE.md`](CODING_STYLE.md)** before writing or modifying any
+code — its rules are hard constraints, not preferences.
+
+## What this package is
+
+**RigorLoop** is a statistically-sound agentic loop-engineering framework. You give it a
+task description, a pile of gold-standard input/output examples, and a set of
+checks; it runs agentic loops (a strategy agent directing concurrent executor
+agents) that iteratively build a solution and evaluate it on a strict
+**dev / validation / test** split, so the final score is trustworthy. The
+produced artifact is portable: an executable Python script, an agent skill
+(`SKILL.md`), or a guidance file (`AGENTS.md`/`CLAUDE.md`).
+
+- Pure Python, **stdlib-only** by design (no runtime dependencies), Python ≥ 3.12.
+- Ships a single CLI: `rigorloop` (`init` / `check` / `run` / `report`).
+- Invokes agents headless and tool-less via the `claude` CLI (`claude -p`).
+- User-facing docs: [`README.md`](README.md). Contributing: [`CONTRIBUTING.md`](CONTRIBUTING.md).
+
+## Architecture: functional core / imperative shell
+
+This is the single most important thing to understand, and it is enforced by
+[`CODING_STYLE.md`](CODING_STYLE.md):
+
+- **`src/rigorloop/core/`** — the functional core. 100% pure: no I/O, no
+  mutation, no time, no randomness, no network, no environment access. It
+  *decides*; it returns values and plans of effects. Testable with plain inputs
+  and zero mocks. Core coverage is held to a higher bar (≥95%).
+- **`src/rigorloop/shell/`** — the thin imperative shell. Performs all effects
+  (filesystem, subprocess, the `claude` CLI) and hands plain data to the core.
+  Keep it small.
+
+The dev/val/test split is encoded in the type system (`DevExample`,
+`ValExample`, `TestExample`) so leaking holdout data into an agent-context
+prompt is a *type error*, not a runtime bug. Don't defeat this.
+
+## Working in this repo
+
+- Dev commands live in the [`justfile`](justfile), each mirroring a CI job:
+  `just lint`, `just typecheck`, `just test`, `just check` (all three), `just fmt`.
+- Tooling: `uv` for env/build, `ruff` (lint + format), `mypy --strict`, `pytest`.
+  `T20` (print) is banned in the core and allowed in the shell.
+- Every source module has a sibling test in `tests/` (e.g. `scoring_calcs.py` →
+  `test_scoring_calcs.py`); `test_leakage.py` guards the split-type invariant and
+  `test_e2e.py` runs full loops against fake agents.
+
+## Folder tree
+
+```
+RigorLoop/
+├── AGENTS.md                     # this file
+├── CODING_STYLE.md               # MANDATORY coding rules (read before editing)
+├── CONTRIBUTING.md               # contributor workflow
+├── README.md                     # user-facing overview & docs
+├── CHANGELOG.md
+├── SECURITY.md                   # runs generated code locally — read this
+├── LICENSE                       # MIT
+├── justfile                      # dev entry points (mirror CI jobs)
+├── pyproject.toml                # package metadata + tool config
+├── uv.lock
+├── .pre-commit-config.yaml
+├── .gitignore
+│
+├── src/
+│   └── rigorloop/
+│       ├── __init__.py           # package version resolution
+│       ├── py.typed              # PEP 561 typing marker
+│       ├── core/                 # FUNCTIONAL CORE — pure, effect-free
+│       │   ├── __init__.py
+│       │   ├── types.py          # domain model: algebraic data types (products + sums)
+│       │   ├── config_calcs.py   # parse rigorloop.toml → typed RunConfig
+│       │   ├── dataset_calcs.py  # parse examples, dedup, split, manifests, power warnings
+│       │   ├── prompt_calcs.py   # prompt builders (agent-context vs. evaluation channels)
+│       │   ├── scoring_calcs.py  # checks, aggregation, Wilson/bootstrap CIs, McNemar
+│       │   ├── strategy_calcs.py # validation cohorts/cadence, stopping rules, champion selection
+│       │   └── report_calcs.py   # render report, check summary, budget estimate
+│       └── shell/                # IMPERATIVE SHELL — effects at the edges
+│           ├── __init__.py
+│           ├── cli.py            # argparse entry point + orchestration driver
+│           ├── agent_calls.py    # claude CLI subprocess wrapper, retries, concurrency
+│           └── io_actions.py     # run dir, artifact persist/reload, sandboxed exec
+│
+├── tests/                        # one test module per source module + leakage/e2e
+│   ├── __init__.py
+│   ├── conftest.py
+│   ├── test_config_calcs.py
+│   ├── test_dataset_calcs.py
+│   ├── test_prompt_calcs.py
+│   ├── test_scoring_calcs.py
+│   ├── test_strategy_calcs.py
+│   ├── test_report_calcs.py
+│   ├── test_agent_calls.py
+│   ├── test_io_actions.py
+│   ├── test_cli.py
+│   ├── test_leakage.py           # asserts split types prevent holdout leakage
+│   └── test_e2e.py               # full runs against fake agents
+│
+├── examples/
+│   └── contact-cards/            # toy project; exactly what `rigorloop init` scaffolds
+│       ├── rigorloop.toml
+│       ├── task.md
+│       └── examples.jsonl
+│
+├── scripts/
+│   └── live-smoke.sh             # live smoke test against the real claude CLI
+│
+├── .github/                      # CI/release workflows, issue/PR templates, dependabot
+│   ├── workflows/
+│   │   ├── ci.yml
+│   │   └── release.yml
+│   ├── ISSUE_TEMPLATE/
+│   │   ├── bug_report.md
+│   │   └── feature_request.md
+│   ├── PULL_REQUEST_TEMPLATE.md
+│   └── dependabot.yml
+│
+└── .claude/
+    └── settings.json             # Claude Code project settings & safety hooks
+```

rigorloop-0.2.0/CHANGELOG.md

@@ -0,0 +1,87 @@
+# Changelog
+
+All notable changes to this project are documented in this file, following
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
+[SemVer](https://semver.org/) (`0.x`: minor = features/breaking, patch = fixes).
+
+## [v0.2.0] - 2026-07-10
+
+### Changed
+
+- **Validation now steers the search, not just final selection.** Previously
+  the raw dev leaderboard chose both the artifact each loop refined and the
+  only candidate ever validated, so an overfit dev leader could monopolize the
+  run while a better generalizer was never measured on validation:
+  - The base artifact for refinement (`base_on_champion`) and the strategy
+    prompt's primary "champion" are now the **validation champion** once any
+    candidate has been validated (the dev leader before that). A diverging dev
+    leader is still shown, as a clearly labeled diagnostic line with an
+    overfit warning — aggregate scores only, never its content.
+  - Each validation checkpoint evaluates a **precommitted cohort** (new
+    `validation.cohort_size` knob, default 2): the top unvalidated candidates
+    by dev score, with the last slot reserved for the best unvalidated
+    candidate not built on the champion (approach diversity). `max_peeks` now
+    budgets individual candidate evaluations, and an already-validated dev
+    leader no longer stalls validation of newer candidates.
+  - Within the McNemar noise band, champion selection tie-breaks on the
+    validation pass rate instead of the dev pass rate (dev is the one metric
+    under direct selection pressure). Significant wins still gate the
+    plateau/patience rule, which now counts checkpoint loops rather than
+    individual cohort evaluations.
+  - `target_pass_rate` early stopping requires the validation score's Wilson
+    lower confidence bound to clear the target, not the raw point estimate.
+  - Per-candidate dev failure samples are persisted
+    (`failure_samples.json` in each candidate directory) and the champion's
+    are reloaded every loop, so the strategy agent keeps concrete
+    counterexamples across non-improving loops and `--resume`.
+  - The report's selection-bias caveat now states that validation both steered
+    the search and selected the winner (the untouched test set remains the
+    honest number).
+
+## [v0.1.1] - 2026-07-9
+
+Documentation and repository-hygiene only; no changes to the `rigorloop`
+package or its behavior.
+
+### Added
+
+- `AGENTS.md`: an orientation guide for coding agents, describing the package,
+  the functional-core / imperative-shell architecture (pointing at
+  `CODING_STYLE.md` as the binding rules), and a full annotated folder tree.
+
+### Changed
+
+- `CONTRIBUTING.md`: expanded the maintainer *Releases* section to document the
+  tag-driven flow — `hatch-vcs` derives the version from the `vX.Y.Z` tag (no
+  file to bump), the `[Unreleased]` → `[X.Y.Z]` changelog promotion, OIDC
+  Trusted Publishing, prerelease/TestPyPI rehearsal, and the "publish a tag
+  exactly once, fix forward" rule.
+
+### Removed
+
+- Internal planning docs `PLAN.md`, `PACKAGING_PLAN.md`, and `CLAUDE.md` removed
+  from the repository (superseded by the shipped implementation and the
+  README/CONTRIBUTING docs).
+
+## [0.1.0] - 2026-07-08
+
+### Added
+
+- Initial implementation of the full RigorLoop protocol:
+  - `rigorloop init | check | run | report` CLI.
+  - Deterministic dev/validation/test splitting with content-hash manifests,
+    exact-duplicate collapsing, and statistical power warnings.
+  - Strategy agent / executor agent loop over the `claude` CLI (headless,
+    tool-less), with strict output contracts, reformat retries, and a
+    fallback path for malformed strategy replies.
+  - Checks: exact/normalized/JSON/regex/numeric matching, custom Python
+    checkers, and n-sample majority-vote LLM judges.
+  - Statistics: Wilson intervals, seeded bootstrap CIs, exact McNemar paired
+    tests; CI-band-gated champion selection, budgeted validation peeks, and
+    a one-shot final test evaluation with a selection-bias caveat in the
+    report.
+  - Solution kinds: executable Python scripts, agent skills (SKILL.md), and
+    guidance markdown (AGENTS.md/CLAUDE.md style).
+  - Resumable run directories (`runs/<run_id>/`) with split-drift protection.
+  - Type-level and test-level leakage guards keeping holdout data out of all
+    agent-context prompts.

{rigorloop-0.1.0 → rigorloop-0.2.0}/CONTRIBUTING.md

@@ -72,7 +72,33 @@ Two invariants are non-negotiable and covered by dedicated tests in
 
 ## Releases (maintainer)
 
-1. Ensure `main` is green and `CHANGELOG.md` has the new section.
-2. `git tag vX.Y.Z && git push origin vX.Y.Z`.
-3. Approve the `pypi` environment deployment.
-4. Verify `pip install rigorloop==X.Y.Z` in a scratch venv.
+The git tag **is** the version: `hatch-vcs` derives it from the latest
+`vX.Y.Z` tag, so there is no version number to bump in any file. The only file
+you edit for a release is `CHANGELOG.md`; pushing the tag does the rest.
+
+1. Land the change the normal way — branch, PR, `just check` green, add an
+   entry under `## [Unreleased]` in `CHANGELOG.md`, merge to `main`.
+2. On `main` (`git checkout main && git pull`), promote the changelog: rename
+   `[Unreleased]` to `[X.Y.Z] - YYYY-MM-DD`, add a fresh empty `[Unreleased]`
+   above it, commit, and push. The tag will point at this commit, so `main`
+   must be green first (branch protection requires `ci-ok`).
+3. Tag and push — this triggers `release.yml`:
+   ```bash
+   git tag vX.Y.Z && git push origin vX.Y.Z
+   ```
+4. Approve the `pypi` environment deployment when prompted (the publish job is
+   gated on manual approval). The workflow then rebuilds from the tag,
+   publishes to PyPI via OIDC Trusted Publishing (no token), and cuts a GitHub
+   Release with generated notes and the built artifacts.
+5. Verify `pip install rigorloop==X.Y.Z` in a scratch venv, then
+   `rigorloop --version`.
+
+Version choice while `0.x`: **patch** (`v0.1.1`) = fixes only; **minor**
+(`v0.2.0`) = features or breaking changes (call breakage out in the changelog);
+`v1.0.0` when `rigorloop.toml`, the CLI, and the run-directory format are
+declared stable.
+
+A version publishes to PyPI exactly once — never re-push or edit a tag. If a
+release fails partway, fix forward with a new patch tag rather than reusing the
+old one. To rehearse the full pipeline first, push a prerelease tag
+(e.g. `v0.2.0rc1`) against TestPyPI.

{rigorloop-0.1.0 → rigorloop-0.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: rigorloop
-Version: 0.1.0
-Summary: Statistically-sound agentic build framework: dev/val/test-split loops that produce code artifacts without overfitting.
+Version: 0.2.0
+Summary: Statistically-sound agentic loop-engineering framework: dev/val/test-split loops that produce code artifacts without overfitting.
 Project-URL: Homepage, https://github.com/ronikobrosly/RigorLoop
 Project-URL: Issues, https://github.com/ronikobrosly/RigorLoop/issues
 Project-URL: Changelog, https://github.com/ronikobrosly/RigorLoop/blob/main/CHANGELOG.md
@@ -22,9 +22,13 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 
+<p align="center">
+  <img src="logo.webp" alt="RigorLoop logo" width="800">
+</p>
+
 # RigorLoop
 
-A statistically-sound agentic build framework. You give it a task description,
+A statistically-sound agentic loop-engineering framework. You give it a task description,
 a pile of gold-standard input/output examples, and a set of checks; it runs
 agentic loops (a strategy agent directing concurrent executor agents) that
 iteratively build and refine a solution — and it evaluates that solution the
@@ -150,9 +154,12 @@ dev_examples_in_prompt = 30        # examples each builder sees (resampled per l
 
 [validation]
 val_every        = 3               # scheduled validation checkpoint cadence
-max_peeks        = 10              # total validation evaluations allowed per run
-patience         = 2               # checkpoints without real improvement => stop
-target_pass_rate = 0.95            # optional: stop early on hitting this on validation
+max_peeks        = 10              # total candidate validation evaluations per run
+cohort_size      = 2               # candidates validated per checkpoint: top dev
+                                   # scorers + one diverse (non-champion-based) slot
+patience         = 2               # checkpoint loops without real improvement => stop
+target_pass_rate = 0.95            # optional: stop early once the validation score's
+                                   # lower confidence bound clears this
 
 [agents]
 model     = "claude-sonnet-5"      # model for all agent roles
@@ -168,9 +175,12 @@ The knobs that matter most in practice:
   executed locally (cheap); skills/guidance are evaluated by running a model
   per example (expensive — check the budget estimate).
 - **`max_loops` × `executors_per_loop`** — your primary cost lever.
-- **`max_peeks` / `patience`** — how often the loop is allowed to look at the
-  validation set, and how long it tolerates no genuine (beyond-noise)
-  improvement before stopping.
+- **`max_peeks` / `cohort_size` / `patience`** — how many candidate evaluations
+  the loop may spend on the validation set, how many candidates each checkpoint
+  compares, and how long the loop tolerates no genuine (beyond-noise)
+  improvement before stopping. For expensive artifact kinds (skill/guidance),
+  every validation evaluation is a model call per example — keep `cohort_size`
+  small and watch the budget estimate.
 - **`seed`** — makes the split reproducible; changing it reshuffles which
   examples land in the holdout.
 
@@ -180,11 +190,21 @@ The knobs that matter most in practice:
   resumed run can never quietly reshuffle it.
 - The building agents only ever see **dev** examples. Validation scores reach
   the strategy agent only as aggregates; test examples reach no agent, ever.
+- The artifact each loop refines is the **validation champion** — the
+  candidate with the best evidence of generalizing — not the raw dev leader.
+  Each checkpoint validates a precommitted cohort (the top unvalidated dev
+  candidates plus one diverse alternative not built on the champion), so a
+  candidate that is slightly worse on dev but generalizes better still gets
+  discovered. The dev leaderboard is a diagnostic, not a selection rule.
 - "Improved" always means *beyond the statistical noise band* (paired tests),
-  not just a higher number — so the loop doesn't chase luck.
-- The validation set (capped, counted peeks) picks the winner; the **test set
-  is evaluated exactly once**, at the very end, and that is the number to
-  report.
+  not just a higher number — so the loop doesn't chase luck. Early stopping on
+  `target_pass_rate` likewise requires the validation score's *lower
+  confidence bound* to clear the target, not a lucky point estimate.
+- The validation set (capped, counted peeks) steers the search and picks the
+  winner — which is ordinary model selection, but it means the winner's
+  validation score is optimistically biased. That is exactly what the **test
+  set** is for: it is evaluated **exactly once**, at the very end, and that is
+  the number to report.
 
 ## ⚠️ The final test set is only honest once
 
@@ -213,6 +233,87 @@ A complete toy project (contact-card text → JSON) lives in
 [`examples/contact-cards/`](examples/contact-cards/) — it is exactly what
 `rigorloop init` scaffolds.
 
+
+## FAQ
+
+### When should I use this?
+
+When all three of these are true:
+
+1. **The task is a transformation with a checkable right answer** — messy
+   text in, structured output out (extraction, normalization, tagging,
+   classification-with-a-format) — and "correct" can be expressed as checks.
+2. **You have, or can collect, a real pile of gold examples** — dozens at
+   minimum, ideally 100+ — that are representative of the inputs you'll see
+   in production.
+3. **What you care about is performance on *new* inputs**, and you need a
+   score you can quote without an asterisk.
+
+If any of these is false, don't reach for RigorLoop. No examples means there
+is nothing to split and nothing to measure honestly. And if the goal is a
+deterministic script that passes a fixed handful of unit tests, a single
+coding-agent session is cheaper and better — see
+[When to use it (and when not to)](#when-to-use-it-and-when-not-to).
+
+### Can't standalone frontier agents do this? Why should I use this?
+
+A frontier agent can absolutely write the solution — RigorLoop's executors
+*are* frontier agents doing exactly that. What a standalone session can't
+give you is a score you can trust. Hand an agent your examples and ask it to
+iterate until things pass, and it grades its own homework: "98% accurate"
+means 98% on the very examples it tuned against, which says little about the
+next thousand inputs. There is no held-out data, no accounting of how many
+times it peeked, and no way to tell a real improvement from a lucky one.
+
+RigorLoop is the harness around those agents that supplies the missing
+discipline:
+
+- **a dev / validation / test split**, enforced up front — building agents
+  only ever see dev examples, and the test set is scored exactly once, at
+  the end;
+- **statistics instead of vibes** — confidence intervals on every score,
+  paired tests so "improved" means beyond the noise band, and a counted,
+  capped budget of validation peeks;
+- **search instead of one shot** — a strategy agent directing concurrent
+  executors across many loops, keeping the candidate with the best evidence
+  of *generalizing* rather than the one that got luckiest on dev.
+
+You could rebuild all of that by hand around a chat session. RigorLoop is
+that machinery, prebuilt and honest by construction.
+
+### What do I do if the solution the loop converges to still isn't accurate enough?
+
+First, believe the number — that's the point of the tool. An honest 74% is
+worth more than the inflated score a self-graded loop would have reported,
+because it tells you the problem isn't solved yet. Then work through this
+list, roughly cheapest first:
+
+1. **Read `report.md`.** The per-check breakdown and loop history usually
+   show *how* it fails: one check dominating the failures, scores plateauing
+   after a couple of loops, or confidence intervals so wide the run couldn't
+   tell candidates apart.
+2. **Check your checks.** An `exact_match` where you meant
+   `normalized_match` or `json_equality`, or a vague `llm_judge` rubric, can
+   make a good solution look bad.
+3. **Sharpen `task.md`.** Failing examples often reveal rules and edge cases
+   the description never stated. Spell them out.
+4. **Add more — and more representative — examples.** Small sets cap what
+   the loop can even detect (heed the power warnings from
+   `rigorloop check`), and noisy labels cap the ceiling no solution can
+   exceed.
+5. **Raise the budget.** More `max_loops`, `executors_per_loop`, and
+   `max_peeks` buy a wider search; a stronger model in `[agents]` buys
+   better builders.
+6. **Reconsider `solution_kind`.** A deterministic script may be too rigid
+   for a fuzzy task; a `skill` or `guidance` artifact puts a model in the
+   loop at inference time (at a much higher evaluation cost).
+7. **Decompose the task.** Two simple transformations chained often beat one
+   complicated one.
+
+One caution: the moment you iterate *after* seeing a test score, that test
+set is spent — bring fresh, never-before-used examples for the next run's
+holdout (see "The final test set is only honest once" above).
+
 ## Contributing & development
 
 See [`CONTRIBUTING.md`](CONTRIBUTING.md). The design plan is in `PLAN.md`, the

{rigorloop-0.1.0 → rigorloop-0.2.0}/README.md

@@ -1,6 +1,10 @@
+<p align="center">
+  <img src="logo.webp" alt="RigorLoop logo" width="800">
+</p>
+
 # RigorLoop
 
-A statistically-sound agentic build framework. You give it a task description,
+A statistically-sound agentic loop-engineering framework. You give it a task description,
 a pile of gold-standard input/output examples, and a set of checks; it runs
 agentic loops (a strategy agent directing concurrent executor agents) that
 iteratively build and refine a solution — and it evaluates that solution the
@@ -126,9 +130,12 @@ dev_examples_in_prompt = 30        # examples each builder sees (resampled per l
 
 [validation]
 val_every        = 3               # scheduled validation checkpoint cadence
-max_peeks        = 10              # total validation evaluations allowed per run
-patience         = 2               # checkpoints without real improvement => stop
-target_pass_rate = 0.95            # optional: stop early on hitting this on validation
+max_peeks        = 10              # total candidate validation evaluations per run
+cohort_size      = 2               # candidates validated per checkpoint: top dev
+                                   # scorers + one diverse (non-champion-based) slot
+patience         = 2               # checkpoint loops without real improvement => stop
+target_pass_rate = 0.95            # optional: stop early once the validation score's
+                                   # lower confidence bound clears this
 
 [agents]
 model     = "claude-sonnet-5"      # model for all agent roles
@@ -144,9 +151,12 @@ The knobs that matter most in practice:
   executed locally (cheap); skills/guidance are evaluated by running a model
   per example (expensive — check the budget estimate).
 - **`max_loops` × `executors_per_loop`** — your primary cost lever.
-- **`max_peeks` / `patience`** — how often the loop is allowed to look at the
-  validation set, and how long it tolerates no genuine (beyond-noise)
-  improvement before stopping.
+- **`max_peeks` / `cohort_size` / `patience`** — how many candidate evaluations
+  the loop may spend on the validation set, how many candidates each checkpoint
+  compares, and how long the loop tolerates no genuine (beyond-noise)
+  improvement before stopping. For expensive artifact kinds (skill/guidance),
+  every validation evaluation is a model call per example — keep `cohort_size`
+  small and watch the budget estimate.
 - **`seed`** — makes the split reproducible; changing it reshuffles which
   examples land in the holdout.
 
@@ -156,11 +166,21 @@ The knobs that matter most in practice:
   resumed run can never quietly reshuffle it.
 - The building agents only ever see **dev** examples. Validation scores reach
   the strategy agent only as aggregates; test examples reach no agent, ever.
+- The artifact each loop refines is the **validation champion** — the
+  candidate with the best evidence of generalizing — not the raw dev leader.
+  Each checkpoint validates a precommitted cohort (the top unvalidated dev
+  candidates plus one diverse alternative not built on the champion), so a
+  candidate that is slightly worse on dev but generalizes better still gets
+  discovered. The dev leaderboard is a diagnostic, not a selection rule.
 - "Improved" always means *beyond the statistical noise band* (paired tests),
-  not just a higher number — so the loop doesn't chase luck.
-- The validation set (capped, counted peeks) picks the winner; the **test set
-  is evaluated exactly once**, at the very end, and that is the number to
-  report.
+  not just a higher number — so the loop doesn't chase luck. Early stopping on
+  `target_pass_rate` likewise requires the validation score's *lower
+  confidence bound* to clear the target, not a lucky point estimate.
+- The validation set (capped, counted peeks) steers the search and picks the
+  winner — which is ordinary model selection, but it means the winner's
+  validation score is optimistically biased. That is exactly what the **test
+  set** is for: it is evaluated **exactly once**, at the very end, and that is
+  the number to report.
 
 ## ⚠️ The final test set is only honest once
 
@@ -189,6 +209,87 @@ A complete toy project (contact-card text → JSON) lives in
 [`examples/contact-cards/`](examples/contact-cards/) — it is exactly what
 `rigorloop init` scaffolds.
 
+
+## FAQ
+
+### When should I use this?
+
+When all three of these are true:
+
+1. **The task is a transformation with a checkable right answer** — messy
+   text in, structured output out (extraction, normalization, tagging,
+   classification-with-a-format) — and "correct" can be expressed as checks.
+2. **You have, or can collect, a real pile of gold examples** — dozens at
+   minimum, ideally 100+ — that are representative of the inputs you'll see
+   in production.
+3. **What you care about is performance on *new* inputs**, and you need a
+   score you can quote without an asterisk.
+
+If any of these is false, don't reach for RigorLoop. No examples means there
+is nothing to split and nothing to measure honestly. And if the goal is a
+deterministic script that passes a fixed handful of unit tests, a single
+coding-agent session is cheaper and better — see
+[When to use it (and when not to)](#when-to-use-it-and-when-not-to).
+
+### Can't standalone frontier agents do this? Why should I use this?
+
+A frontier agent can absolutely write the solution — RigorLoop's executors
+*are* frontier agents doing exactly that. What a standalone session can't
+give you is a score you can trust. Hand an agent your examples and ask it to
+iterate until things pass, and it grades its own homework: "98% accurate"
+means 98% on the very examples it tuned against, which says little about the
+next thousand inputs. There is no held-out data, no accounting of how many
+times it peeked, and no way to tell a real improvement from a lucky one.
+
+RigorLoop is the harness around those agents that supplies the missing
+discipline:
+
+- **a dev / validation / test split**, enforced up front — building agents
+  only ever see dev examples, and the test set is scored exactly once, at
+  the end;
+- **statistics instead of vibes** — confidence intervals on every score,
+  paired tests so "improved" means beyond the noise band, and a counted,
+  capped budget of validation peeks;
+- **search instead of one shot** — a strategy agent directing concurrent
+  executors across many loops, keeping the candidate with the best evidence
+  of *generalizing* rather than the one that got luckiest on dev.
+
+You could rebuild all of that by hand around a chat session. RigorLoop is
+that machinery, prebuilt and honest by construction.
+
+### What do I do if the solution the loop converges to still isn't accurate enough?
+
+First, believe the number — that's the point of the tool. An honest 74% is
+worth more than the inflated score a self-graded loop would have reported,
+because it tells you the problem isn't solved yet. Then work through this
+list, roughly cheapest first:
+
+1. **Read `report.md`.** The per-check breakdown and loop history usually
+   show *how* it fails: one check dominating the failures, scores plateauing
+   after a couple of loops, or confidence intervals so wide the run couldn't
+   tell candidates apart.
+2. **Check your checks.** An `exact_match` where you meant
+   `normalized_match` or `json_equality`, or a vague `llm_judge` rubric, can
+   make a good solution look bad.
+3. **Sharpen `task.md`.** Failing examples often reveal rules and edge cases
+   the description never stated. Spell them out.
+4. **Add more — and more representative — examples.** Small sets cap what
+   the loop can even detect (heed the power warnings from
+   `rigorloop check`), and noisy labels cap the ceiling no solution can
+   exceed.
+5. **Raise the budget.** More `max_loops`, `executors_per_loop`, and
+   `max_peeks` buy a wider search; a stronger model in `[agents]` buys
+   better builders.
+6. **Reconsider `solution_kind`.** A deterministic script may be too rigid
+   for a fuzzy task; a `skill` or `guidance` artifact puts a model in the
+   loop at inference time (at a much higher evaluation cost).
+7. **Decompose the task.** Two simple transformations chained often beat one
+   complicated one.
+
+One caution: the moment you iterate *after* seeing a test score, that test
+set is spent — bring fresh, never-before-used examples for the next run's
+holdout (see "The final test set is only honest once" above).
+
 ## Contributing & development
 
 See [`CONTRIBUTING.md`](CONTRIBUTING.md). The design plan is in `PLAN.md`, the

{rigorloop-0.1.0 → rigorloop-0.2.0}/examples/contact-cards/rigorloop.toml

@@ -15,9 +15,10 @@ executors_per_loop     = 4
 dev_examples_in_prompt = 30
 
 [validation]
-val_every  = 3
-max_peeks  = 10
-patience   = 2
+val_every   = 3
+max_peeks   = 10
+cohort_size = 2
+patience    = 2
 target_pass_rate = 0.95
 
 [agents]

{rigorloop-0.1.0 → rigorloop-0.2.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "rigorloop"
 dynamic = ["version"]
-description = "Statistically-sound agentic build framework: dev/val/test-split loops that produce code artifacts without overfitting."
+description = "Statistically-sound agentic loop-engineering framework: dev/val/test-split loops that produce code artifacts without overfitting."
 readme = "README.md"
 license = "MIT"
 license-files = ["LICENSE"]

{rigorloop-0.1.0 → rigorloop-0.2.0}/src/rigorloop/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
 __commit_id__ = commit_id = None

{rigorloop-0.1.0 → rigorloop-0.2.0}/src/rigorloop/core/config_calcs.py

@@ -176,6 +176,9 @@ def _parse_validation(data: dict[str, object]) -> Result[ValidationConfig, Confi
     patience = _get_int(table, "patience", "validation.patience", 2)
     if isinstance(patience, Err):
         return patience
+    cohort_size = _get_int(table, "cohort_size", "validation.cohort_size", 2)
+    if isinstance(cohort_size, Err):
+        return cohort_size
     target: Option[float] = NOTHING
     if "target_pass_rate" in table:
         parsed = _get_float(table, "target_pass_rate", "validation.target_pass_rate", 0.0)
@@ -185,7 +188,14 @@ def _parse_validation(data: dict[str, object]) -> Result[ValidationConfig, Confi
             return Err(InvalidValue("validation.target_pass_rate", "must be in (0, 1]"))
         target = Some(parsed.value)
     return Ok(
-        ValidationConfig(val_every.value, max_peeks.value, min_gap.value, patience.value, target)
+        ValidationConfig(
+            val_every.value,
+            max_peeks.value,
+            min_gap.value,
+            patience.value,
+            target,
+            cohort_size.value,
+        )
     )