rulesmith 0.1.0__tar.gz

rulesmith-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+.pytest_cache/
+.hypothesis/
+.ruff_cache/
+*.egg-info/
+dist/
+runs/
+.env
+.env.*
+.DS_Store

rulesmith-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rulesmith contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

rulesmith-0.1.0/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.5
+Name: rulesmith
+Version: 0.1.0
+Summary: Search for rule programs that ask a small judge for typed decisions, with DSPy and GEPA.
+Project-URL: Homepage, https://github.com/bradAGI/rulesmith
+Project-URL: Repository, https://github.com/bradAGI/rulesmith
+Project-URL: Issues, https://github.com/bradAGI/rulesmith/issues
+Author: rulesmith contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: decision graphs,dspy,gepa,llm,program search,rules
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <3.15,>=3.12
+Requires-Dist: dspy==3.4.0
+Requires-Dist: gepa==0.1.4
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: pydantic<3,>=2.11
+Requires-Dist: typesafe-sdk<0.7,>=0.6
+Provides-Extra: bench
+Requires-Dist: huggingface-hub<2,>=1; extra == 'bench'
+Requires-Dist: pyarrow<24,>=17; extra == 'bench'
+Requires-Dist: scikit-learn<2,>=1.5; extra == 'bench'
+Provides-Extra: chess
+Requires-Dist: chess<2,>=1.11; extra == 'chess'
+Provides-Extra: clef
+Requires-Dist: huggingface-hub<2,>=1; extra == 'clef'
+Requires-Dist: pillow<13,>=11; extra == 'clef'
+Requires-Dist: torch<3,>=2.11; extra == 'clef'
+Requires-Dist: torchvision<1,>=0.26; extra == 'clef'
+Requires-Dist: transformers[torch]<6,>=5.10.2; extra == 'clef'
+Provides-Extra: doom
+Requires-Dist: imageio-ffmpeg<1,>=0.6; extra == 'doom'
+Requires-Dist: numpy<3,>=2; extra == 'doom'
+Requires-Dist: pillow<13,>=11; extra == 'doom'
+Requires-Dist: vizdoom<2,>=1.3; extra == 'doom'
+Description-Content-Type: text/markdown
+
+# rulesmith
+
+![rulesmith: rules that ask a local model for typed decisions](https://raw.githubusercontent.com/bradAGI/rulesmith/main/image.png)
+
+**rulesmith lets DSPy build decision graphs for you.** Give it a classification
+task and labeled examples. It searches for a small program of rules: cheap tests
+on the input's fields settle the cases they can, and a local or hosted judge is
+asked typed questions about the rest. [GEPA](https://github.com/gepa-ai/gepa)
+keeps the revisions that score better, and the result is a rules file you can
+read, edit, run, and serve.
+
+```
+read has_company_logo in [0, 1]
+read length of company_profile in [0, 10000]
+when has_company_logo == 0 and length of company_profile == 0: fraudulent
+always: ask "Is this job posting a scam?" [fraudulent: "fraudulent"] [genuine: "genuine"]
+```
+
+![Same 4B judge, same 100 examples: rulesmith beat DSPy's tuned classifier by 6.7 points on job-fraud detection with 29% of the model calls, and tied it on clothing reviews with 6%](https://raw.githubusercontent.com/bradAGI/rulesmith/main/docs/assets/claim.svg)
+
+The [benchmarks](https://github.com/bradAGI/rulesmith/blob/main/docs/benchmarks.md) have every dataset, including where rulesmith
+does not help: on one-sentence inputs it ties DSPy, and on job postings a TF-IDF
+classifier still beats it.
+
+An independent research prototype, not affiliated with TypeSafe.
+
+## Installation
+
+```sh
+pip install rulesmith
+```
+
+Python 3.12 to 3.14. Extras for the games and benchmarks are listed in the
+[quickstart](https://github.com/bradAGI/rulesmith/blob/main/docs/quickstart.md#installation).
+
+## Quickstart
+
+With a judge running ([Ruling](https://github.com/bradAGI/ruling) on a Mac,
+Cloudflare's Clef-flash on an NVIDIA GPU or a Mac, TypeSafe's hosted Jev, or any
+OpenAI-compatible chat API that returns log-probabilities), any OpenAI-compatible model to write the rules, and a
+labeled task such as [this one](https://github.com/bradAGI/rulesmith/blob/main/examples/support-routing/task.json):
+
+```sh
+export OPENAI_API_KEY=local-no-auth   # a local model server takes any key
+rulesmith optimize task.json --backend ruling --max-workers 1 \
+  --reflection-model "openai/<model>" --reflection-base-url http://127.0.0.1:8080/v1 \
+  --output runs/my-task
+rulesmith run runs/my-task/plan.rules input.json --backend ruling
+```
+
+The [quickstart](https://github.com/bradAGI/rulesmith/blob/main/docs/quickstart.md) walks through it with every command's output.
+
+## Beyond classification
+
+The same search plays games. Pitted against a league of other graphs, this one learned to
+hunt in Doom deathmatch and beat the champion 12-0, with no model calls at all
+([how](https://github.com/bradAGI/rulesmith/blob/main/docs/doom.md)):
+
+![A searched rules graph hunting down the Doom champion it dethroned](https://raw.githubusercontent.com/bradAGI/rulesmith/main/docs/assets/doom/duel-hunter.gif)
+
+## Learn more
+
+- [Quickstart](https://github.com/bradAGI/rulesmith/blob/main/docs/quickstart.md): installation, judges, and a task to a running graph.
+- [Notebook](https://github.com/bradAGI/rulesmith/blob/main/notebooks/quickstart.ipynb): rules for spotting fake job postings, end to end in Colab with an OpenAI key.
+- [Local notebook](https://github.com/bradAGI/rulesmith/blob/main/notebooks/local.ipynb): the same with Clef-flash judging on your Mac or NVIDIA GPU, free per call.
+- [Tasks](https://github.com/bradAGI/rulesmith/blob/main/docs/tasks.md): the task file, drafting labels, structured inputs, costs, and graders.
+- [Rules](https://github.com/bradAGI/rulesmith/blob/main/docs/rules.md): the language search writes, confidence, calibration, escalation, and extraction.
+- [Graph format](https://github.com/bradAGI/rulesmith/blob/main/docs/graphs.md): the JSON behind each node, execution, and drawing a graph.
+- [Maps](https://github.com/bradAGI/rulesmith/blob/main/docs/maps.md): many decisions as one map, searched a unit at a time.
+- [Optimization](https://github.com/bradAGI/rulesmith/blob/main/docs/optimization.md): how search runs, its settings, and its limits.
+- [Command line](https://github.com/bradAGI/rulesmith/blob/main/docs/cli.md): every command, including `serve` and `drift`, and the backends.
+- [Python](https://github.com/bradAGI/rulesmith/blob/main/docs/python.md): running and building graphs from code.
+- [Benchmarks](https://github.com/bradAGI/rulesmith/blob/main/docs/benchmarks.md): rulesmith against DSPy, TF-IDF and direct calls on five public datasets.
+- [Related work](https://github.com/bradAGI/rulesmith/blob/main/docs/related-work.md): when to use rulesmith and when DSPy, and what came before.
+- [Doom](https://github.com/bradAGI/rulesmith/blob/main/docs/doom.md) and [Chess](https://github.com/bradAGI/rulesmith/blob/main/docs/chess.md): graphs that play.
+- [Claude Code skill](https://github.com/bradAGI/rulesmith/blob/main/skills/rulesmith/SKILL.md): copy `skills/rulesmith` into `~/.claude/skills/`.
+
+## Development
+
+```sh
+uv sync --locked
+uv run pytest -q
+```
+
+## License
+
+[MIT](https://github.com/bradAGI/rulesmith/blob/main/LICENSE).

rulesmith-0.1.0/README.md

@@ -0,0 +1,87 @@
+# rulesmith
+
+![rulesmith: rules that ask a local model for typed decisions](image.png)
+
+**rulesmith lets DSPy build decision graphs for you.** Give it a classification
+task and labeled examples. It searches for a small program of rules: cheap tests
+on the input's fields settle the cases they can, and a local or hosted judge is
+asked typed questions about the rest. [GEPA](https://github.com/gepa-ai/gepa)
+keeps the revisions that score better, and the result is a rules file you can
+read, edit, run, and serve.
+
+```
+read has_company_logo in [0, 1]
+read length of company_profile in [0, 10000]
+when has_company_logo == 0 and length of company_profile == 0: fraudulent
+always: ask "Is this job posting a scam?" [fraudulent: "fraudulent"] [genuine: "genuine"]
+```
+
+![Same 4B judge, same 100 examples: rulesmith beat DSPy's tuned classifier by 6.7 points on job-fraud detection with 29% of the model calls, and tied it on clothing reviews with 6%](docs/assets/claim.svg)
+
+The [benchmarks](docs/benchmarks.md) have every dataset, including where rulesmith
+does not help: on one-sentence inputs it ties DSPy, and on job postings a TF-IDF
+classifier still beats it.
+
+An independent research prototype, not affiliated with TypeSafe.
+
+## Installation
+
+```sh
+pip install rulesmith
+```
+
+Python 3.12 to 3.14. Extras for the games and benchmarks are listed in the
+[quickstart](docs/quickstart.md#installation).
+
+## Quickstart
+
+With a judge running ([Ruling](https://github.com/bradAGI/ruling) on a Mac,
+Cloudflare's Clef-flash on an NVIDIA GPU or a Mac, TypeSafe's hosted Jev, or any
+OpenAI-compatible chat API that returns log-probabilities), any OpenAI-compatible model to write the rules, and a
+labeled task such as [this one](examples/support-routing/task.json):
+
+```sh
+export OPENAI_API_KEY=local-no-auth   # a local model server takes any key
+rulesmith optimize task.json --backend ruling --max-workers 1 \
+  --reflection-model "openai/<model>" --reflection-base-url http://127.0.0.1:8080/v1 \
+  --output runs/my-task
+rulesmith run runs/my-task/plan.rules input.json --backend ruling
+```
+
+The [quickstart](docs/quickstart.md) walks through it with every command's output.
+
+## Beyond classification
+
+The same search plays games. Pitted against a league of other graphs, this one learned to
+hunt in Doom deathmatch and beat the champion 12-0, with no model calls at all
+([how](docs/doom.md)):
+
+![A searched rules graph hunting down the Doom champion it dethroned](docs/assets/doom/duel-hunter.gif)
+
+## Learn more
+
+- [Quickstart](docs/quickstart.md): installation, judges, and a task to a running graph.
+- [Notebook](notebooks/quickstart.ipynb): rules for spotting fake job postings, end to end in Colab with an OpenAI key.
+- [Local notebook](notebooks/local.ipynb): the same with Clef-flash judging on your Mac or NVIDIA GPU, free per call.
+- [Tasks](docs/tasks.md): the task file, drafting labels, structured inputs, costs, and graders.
+- [Rules](docs/rules.md): the language search writes, confidence, calibration, escalation, and extraction.
+- [Graph format](docs/graphs.md): the JSON behind each node, execution, and drawing a graph.
+- [Maps](docs/maps.md): many decisions as one map, searched a unit at a time.
+- [Optimization](docs/optimization.md): how search runs, its settings, and its limits.
+- [Command line](docs/cli.md): every command, including `serve` and `drift`, and the backends.
+- [Python](docs/python.md): running and building graphs from code.
+- [Benchmarks](docs/benchmarks.md): rulesmith against DSPy, TF-IDF and direct calls on five public datasets.
+- [Related work](docs/related-work.md): when to use rulesmith and when DSPy, and what came before.
+- [Doom](docs/doom.md) and [Chess](docs/chess.md): graphs that play.
+- [Claude Code skill](skills/rulesmith/SKILL.md): copy `skills/rulesmith` into `~/.claude/skills/`.
+
+## Development
+
+```sh
+uv sync --locked
+uv run pytest -q
+```
+
+## License
+
+[MIT](LICENSE).

rulesmith-0.1.0/benchmarks/charts.py

@@ -0,0 +1,312 @@
+"""The benchmark charts in the docs, drawn from the results files.
+
+    python3 benchmarks/charts.py
+
+Writes docs/assets/benchmarks.svg, accuracy against judge calls per row for rulesmith's search
+and the baselines on each dataset; docs/assets/escalation.svg, accuracy against the share of
+rows sent to the larger judge; and docs/assets/claim.svg, the README's headline: search against
+DSPy's typed classifier on the same tuned judge. Every number is a mean over the seeds in
+benchmarks/results; the tables in docs/benchmarks.md carry the spread and the tests of
+significance.
+"""
+
+import json
+from pathlib import Path
+from statistics import mean
+
+ROOT = Path(__file__).resolve().parents[1]
+RESULTS = ROOT / "benchmarks" / "results"
+ASSETS = ROOT / "docs" / "assets"
+
+# Where rules over fields help first, then where the input is one piece of free text.
+DATASETS = ("clothing", "jobs", "news", "trec", "banking77")
+# The baselines that ask a language model on every row; the best of them per dataset is the
+# fair comparison, so search is never shown beside a weak one.
+ASKING = {
+    "direct": "direct call",
+    "dspy": "DSPy",
+    "dspy-cot": "DSPy, reasoning",
+    "dspy-typed": "DSPy typed",
+}
+# The README's headline: where the input has fields, on the judge tuned on each seed's examples.
+CLAIM = (
+    ("jobs", "Job postings: scam or genuine?"),
+    ("clothing", "Clothing reviews: recommend or not?"),
+)
+ESCALATION = (("small", "4B alone"), ("large", "Clef-flash alone"), ("escalate", "escalating"))
+
+# rulesmith is the one hue; every baseline is context, in the neutral, and named beside its mark.
+STYLE = """
+.bg { fill: #fcfcfb } .ink { fill: #0b0b0b } .muted { fill: #52514e }
+.grid { stroke: #e4e3df } .axis { stroke: #b5b4ae }
+.us { fill: #2a78d6 } .them { fill: #6b6a66 } .ring { stroke: #fcfcfb }
+.mix { stroke: #6b6a66 }
+.halo { stroke: #fcfcfb; stroke-width: 4px; paint-order: stroke; stroke-linejoin: round }
+@media (prefers-color-scheme: dark) {
+  .bg { fill: #1a1a19 } .ink { fill: #ffffff } .muted { fill: #c3c2b7 }
+  .grid { stroke: #33322f } .axis { stroke: #57564f }
+  .us { fill: #3987e5 } .them { fill: #8f8e89 } .ring { stroke: #1a1a19 }
+  .mix { stroke: #8f8e89 }
+  .halo { stroke: #1a1a19 }
+}
+"""
+
+PANEL, WIDE, GAP, LEFT, TOP, PLOT = 176, 300, 62, 52, 74, 190
+FONT = 'font-family="system-ui, sans-serif" font-size="12"'
+
+
+def by_method(dataset: str) -> dict[str, tuple[float, float]]:
+    """Mean accuracy and judge calls per row of each method on a dataset, over its seeds."""
+    runs = json.loads((RESULTS / f"{dataset}.json").read_text())["runs"]
+    methods = {r["method"] for r in runs}
+    return {
+        m: (
+            mean(r["accuracy"] for r in runs if r["method"] == m),
+            mean(r["calls_per_example"] for r in runs if r["method"] == m),
+        )
+        for m in methods
+    }
+
+
+def comparison(found: dict[str, tuple[float, float]]) -> list[tuple[str, float, float, bool]]:
+    """What a panel shows: search, the best baseline that asks the model on every row, and the
+    trained classifier that never asks, as (name, calls, accuracy, is rulesmith)."""
+    best = max((m for m in ASKING if m in found), key=lambda m: found[m][0])
+    return [
+        ("TF-IDF", found["tfidf"][1], found["tfidf"][0], False),
+        (ASKING[best], found[best][1], found[best][0], False),
+        ("rulesmith", found["search"][1], found["search"][0], True),
+    ]
+
+
+def escalated() -> dict[str, list[tuple[str, float, float]]]:
+    """Per dataset, each way of judging as (name, larger-judge calls per row, accuracy)."""
+    runs = json.loads((RESULTS / "escalation.json").read_text())["runs"]
+    found = {}
+    for dataset in sorted({r["dataset"] for r in runs}):
+        found[dataset] = []
+        for graph, name in ESCALATION:
+            mine = [r for r in runs if r["dataset"] == dataset and r["graph"] == graph]
+            large = mean(r["calls_per_row"]["large"] for r in mine)
+            found[dataset].append((name, large, mean(r["accuracy"] for r in mine)))
+    return found
+
+
+def tuned(dataset: str, method: str) -> list[dict]:
+    """A method's runs on the tuned judge, seed by seed, so pooled rows line up across methods."""
+    runs = json.loads((RESULTS / "tuned" / f"{dataset}.json").read_text())["runs"]
+    return sorted((r for r in runs if r["method"] == method), key=lambda r: r["seed"])
+
+
+def p_value(p: float) -> str:
+    """A p-value as a reader writes it: 0.66, or 3×10⁻⁶ rather than 3.2e-06."""
+    if p >= 0.001:
+        return f"{p:.2g}"
+    mantissa, exponent = f"{p:.0e}".split("e")
+    raised = str(int(exponent)).translate(str.maketrans("-0123456789", "⁻⁰¹²³⁴⁵⁶⁷⁸⁹"))
+    return f"{mantissa}×10{raised}"
+
+
+def ticks(low: float, high: float) -> tuple[float, float, list[float]]:
+    """A y range padded to the nearest 0.05 around the values, and its gridlines."""
+    step = 0.05 if high - low < 0.25 else 0.1
+    bottom = max(0.0, (int(low / step) - (0 if low % step else 1)) * step)
+    top = min(1.0, (int(high / step) + 1) * step)
+    count = round((top - bottom) / step)
+    return bottom, top, [round(bottom + i * step, 2) for i in range(count + 1)]
+
+
+def labels(points: list[tuple[float, float]]) -> list[int]:
+    """Which way each point's label goes: up, unless a higher point sits near enough that the
+    labels would meet, in which case the lower one is labeled below."""
+    sides = [-1] * len(points)
+    for i, (x, y) in enumerate(points):
+        for j, (x2, y2) in enumerate(points):
+            if i != j and abs(x - x2) < 70 and 0 <= y - y2 < 26:
+                sides[i] = 1
+    return sides
+
+
+def panel(left: float, title: str, points, x_label: str, segment=None, size=PANEL) -> list[str]:
+    """One small multiple: its own y scale, x from 0 to 1, each point named beside it."""
+    values = [y for _, _, y, *_ in points]
+    bottom, top, grid = ticks(min(values), max(values))
+    x = lambda v: left + 10 + v * (size - 20)  # noqa: E731
+    y = lambda v: TOP + PLOT - (v - bottom) / (top - bottom) * PLOT  # noqa: E731
+    out = [f'<text x="{left}" y="{TOP - 14}" font-weight="600" class="ink">{title}</text>']
+    for g in grid:
+        out.append(
+            f'<line x1="{left}" y1="{y(g):.1f}" x2="{left + size}" y2="{y(g):.1f}" class="grid"/>'
+        )
+        out.append(
+            f'<text x="{left - 6}" y="{y(g) + 4:.1f}" text-anchor="end" class="muted">'
+            f"{g:.2f}</text>"
+        )
+    base = TOP + PLOT
+    out.append(f'<line x1="{left}" y1="{base}" x2="{left + size}" y2="{base}" class="axis"/>')
+    for v in (0, 0.5, 1):
+        out.append(
+            f'<text x="{x(v):.1f}" y="{base + 16}" text-anchor="middle" class="muted">{v:g}</text>'
+        )
+    out.append(
+        f'<text x="{left + size / 2}" y="{base + 34}" text-anchor="middle" class="muted">'
+        f"{x_label}</text>"
+    )
+    if segment:
+        (x1, y1), (x2, y2) = segment
+        out.append(
+            f'<line x1="{x(x1):.1f}" y1="{y(y1):.1f}" x2="{x(x2):.1f}" y2="{y(y2):.1f}" '
+            'class="mix" stroke-width="1.5" stroke-dasharray="4 4"/>'
+        )
+    placed = [(x(c), y(a)) for _, c, a, *_ in points]
+    for (name, calls, accuracy, *ours), (px, py), side in zip(
+        points, placed, labels(placed), strict=True
+    ):
+        kind = "us" if ours and ours[0] else "them"
+        anchor = "start" if px < left + 30 else "end" if px > left + size - 30 else "middle"
+        weight = ' font-weight="600"' if kind == "us" else ""
+        out.append(
+            f'<circle cx="{px:.1f}" cy="{py:.1f}" r="5" class="{kind} ring" stroke-width="2">'
+            f"<title>{name}: {accuracy:.3f} accuracy, {calls:.2f} calls per row</title></circle>"
+        )
+        ly = py - 10 if side < 0 else py + 19
+        out.append(
+            f'<text x="{px:.1f}" y="{ly:.1f}" text-anchor="{anchor}" class="ink halo"{weight}>'
+            f"{name} {accuracy:.3f}</text>"
+        )
+    return out
+
+
+def svg(width: int, height: int, title: str, subtitle: str, body: list[str]) -> str:
+    return "\n".join(
+        [
+            f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
+            f'viewBox="0 0 {width} {height}" {FONT}>',
+            f"<style>{STYLE}</style>",
+            f'<rect width="{width}" height="{height}" class="bg"/>',
+            f'<text x="16" y="24" font-size="14" font-weight="600" class="ink">{title}</text>',
+            f'<text x="16" y="42" class="muted">{subtitle}</text>',
+            *body,
+            "</svg>",
+            "",
+        ]
+    )
+
+
+def benchmarks_chart() -> str:
+    body = []
+    for i, dataset in enumerate(DATASETS):
+        shown = comparison(by_method(dataset))
+        body += panel(LEFT + i * (PANEL + GAP), dataset, shown, "judge calls per row")
+    width = LEFT + len(DATASETS) * (PANEL + GAP) - GAP + 24
+    return svg(
+        width,
+        TOP + PLOT + 52,
+        "Accuracy against judge calls per row, on 300 held-out rows",
+        "rulesmith's searched rules against the best baseline that asks on every row, and a "
+        "classifier that never asks. Means of three seeds; each panel has its own scale.",
+        body,
+    )
+
+
+def escalation_chart() -> str:
+    body = []
+    for i, (dataset, points) in enumerate(escalated().items()):
+        marked = [(name, calls, acc, name == "escalating") for name, calls, acc in points]
+        ends = [(c, a) for n, c, a in points if n != "escalating"]
+        body += panel(
+            LEFT + i * (WIDE + GAP), dataset, marked, "larger-judge calls per row", ends, WIDE
+        )
+    width = LEFT + 2 * (WIDE + GAP) - GAP + 24
+    return svg(
+        width,
+        TOP + PLOT + 52,
+        "Escalating to a larger judge only where the small one is unsure",
+        "Above the dashed line is better than sending that share of rows to the larger judge at "
+        "random. Means of three seeds.",
+        body,
+    )
+
+
+def bar(x: float, y: float, length: float, value: str, kind: str) -> list[str]:
+    """A bar from zero, and its value at the end of it in the text color."""
+    weight = ' font-weight="600"' if kind == "us" else ""
+    return [
+        f'<rect x="{x}" y="{y + 2}" width="{length:.1f}" height="18" rx="2" class="{kind}"/>',
+        f'<text x="{x + length + 6:.1f}" y="{y + 15}" class="ink"{weight}>{value}</text>',
+    ]
+
+
+def claim_chart() -> str:
+    from rulesmith.bench import mcnemar
+
+    width, left, accuracy_width, calls_left, calls_width = 820, 200, 360, 640, 110
+    body = [
+        f'<text x="{left}" y="{TOP}" class="muted" font-weight="600">accuracy</text>',
+        f'<text x="{calls_left}" y="{TOP}" class="muted" font-weight="600">'
+        "model calls per row</text>",
+    ]
+    y = TOP + 34
+    for dataset, title in CLAIM:
+        ours, theirs = tuned(dataset, "search"), tuned(dataset, "dspy-typed")
+        pooled = [[c for r in runs for c in r["correct"]] for runs in (ours, theirs)]
+        p = mcnemar(*pooled)
+        rows = [
+            (
+                "rulesmith",
+                mean(r["accuracy"] for r in ours),
+                mean(r["calls_per_example"] for r in ours),
+                "us",
+            ),
+            (
+                "DSPy typed classifier",
+                mean(r["accuracy"] for r in theirs),
+                mean(r["calls_per_example"] for r in theirs),
+                "them",
+            ),
+        ]
+        gain = (rows[0][1] - rows[1][1]) * 100
+        verdict = f"+{gain:.1f} points" if p < 0.05 and gain > 0 else "tie"
+        share = rows[0][2] / rows[1][2]
+        body.append(
+            f'<text x="16" y="{y}" class="ink" font-size="14" font-weight="600">{title}</text>'
+            f'<text x="{left + 230}" y="{y}" class="muted">{verdict} · p = {p_value(p)} · '
+            f"{share:.0%} of the calls</text>"
+        )
+        y += 14
+        for name, accuracy, calls, kind in rows:
+            weight = ' font-weight="600"' if kind == "us" else ""
+            body += [
+                f'<text x="{left - 10}" y="{y + 15}" text-anchor="end" class="ink"{weight}>'
+                f"{name}</text>",
+                *bar(left, y, accuracy * accuracy_width, f"{accuracy:.3f}", kind),
+                *bar(calls_left, y, calls * calls_width, f"{calls:.2f}", kind),
+            ]
+            y += 26
+        y += 30
+    body.append(
+        f'<text x="16" y="{y}" class="muted">Means of 3 seeds, 300 held-out rows each; p from '
+        "exact McNemar tests over the pooled rows. Bars start at zero.</text>"
+    )
+    body.append(
+        f'<text x="16" y="{y + 18}" class="muted">On one-sentence inputs (TREC, Banking77) '
+        "rulesmith ties DSPy; on job postings a TF-IDF classifier scores higher still.</text>"
+    )
+    return svg(
+        width,
+        y + 36,
+        "Same 4B judge, same 100 examples, a fraction of the calls",
+        "rulesmith against DSPy's typed classifier (GEPA + ReAnchor), the judge fine-tuned on each "
+        "seed's examples.",
+        body,
+    )
+
+
+def main() -> None:
+    (ASSETS / "benchmarks.svg").write_text(benchmarks_chart())
+    (ASSETS / "escalation.svg").write_text(escalation_chart())
+    (ASSETS / "claim.svg").write_text(claim_chart())
+
+
+if __name__ == "__main__":
+    main()

rulesmith-0.1.0/benchmarks/chess.sh

@@ -0,0 +1,86 @@
+#!/bin/zsh
+# The chess searches against Stockfish behind docs/assets/fitness.svg: from the greedy graph,
+# several search seeds at once, each with the same budget of games. A finished search is kept,
+# so running this again picks up where an interrupted run stopped.
+#
+#   benchmarks/chess.sh                     # three seeds, 1000 games each
+#   BUDGET=200 SEEDS="0" benchmarks/chess.sh
+#   kill $(cat runs/chess/driver.pid)       # stop it, and only it
+#
+# Needs: Stockfish at $STOCKFISH, a proposer (a hosted model with its key in the environment,
+# or a local server at $PROPOSER_URL), and Ruling at $JUDGE for any graph that asks it.
+set -u
+cd "${0:A:h}/.."
+
+HOURS=${HOURS:-6}
+BUDGET=${BUDGET:-1000}
+SEEDS=(${=SEEDS:-0 1 2})
+STOCKFISH=${STOCKFISH:-$HOME/.local/bin/stockfish}
+JUDGE=${JUDGE:-http://127.0.0.1:8010}
+PROPOSER_MODEL=${PROPOSER_MODEL:-openrouter/qwen/qwen3.8-27b}
+PROPOSER_URL=${PROPOSER_URL-}
+PROPOSER_OPTIONS=${PROPOSER_OPTIONS-'{"reasoning": {"max_tokens": 2048}}'}
+TASK=examples/chess/stockfish.json
+START=examples/chess/graphs/greedy.rules
+LOGS=runs/chess
+mkdir -p $LOGS benchmarks/results
+export OPENAI_API_KEY=${OPENAI_API_KEY:-local-no-auth}
+
+proposer=(--reflection-model $PROPOSER_MODEL --reflection-max-tokens 16384 --reflection-timeout 3600)
+[[ -n $PROPOSER_URL ]] && proposer+=(--reflection-base-url $PROPOSER_URL)
+[[ -n $PROPOSER_OPTIONS ]] && proposer+=(--reflection-options $PROPOSER_OPTIONS)
+judge=(--backend ruling --max-workers 1 --base-url $JUDGE --engine $STOCKFISH)
+
+deadline=$(( $(date +%s) + HOURS * 3600 ))
+note() { print -r -- "$(date '+%F %T') $*" | tee -a $LOGS/status.txt; }
+
+caffeinate -dimsu -w $$ &
+print $$ >$LOGS/driver.pid
+runs=()
+halt() {
+  trap - TERM INT
+  # Each search stops after its current round and keeps its checkpoint; running this script
+  # again resumes them. A second signal ends the searches at once.
+  note "stopping after the current round"
+  for search in $LOGS/*/search; do [[ -d $search ]] && touch $search/gepa.stop; done
+  trap 'pkill -f "^[^ ]*python3 [^ ]*/rulesmith (optimize|evaluate) examples/chess/"' TERM INT
+  [[ -n ${stopper:-} ]] && kill $stopper 2>/dev/null
+  wait $runs
+  rm -f $LOGS/driver.pid
+  exit 143
+}
+trap halt TERM INT
+
+search() {
+  local seed=$1 run=$LOGS/chess-$1
+  if [[ -f $run/report.json ]]; then
+    note "seed $seed: already done"
+  else
+    # A search that was interrupted resumes from its checkpoint; one that left none is set aside.
+    resume=()
+    if [[ -f $run/search/gepa_state.bin ]]; then resume=(--resume)
+    elif [[ -d $run ]]; then mv $run $run-failed-$(date +%s)
+    fi
+    note "seed $seed: searching, $BUDGET games"
+    uv run rulesmith optimize $TASK $START $judge --max-metric-calls $BUDGET --seed $seed \
+      $proposer $resume --output $run >>$LOGS/chess-$seed.log 2>&1 \
+      || { note "seed $seed: search failed, see $LOGS/chess-$seed.log"; return; }
+  fi
+  if [[ ! -f $run/test.json ]]; then
+    note "seed $seed: scoring the winner on test"
+    uv run rulesmith evaluate $TASK $run/plan.json $judge --split test --output $run/test.json \
+      >>$LOGS/chess-$seed.log 2>&1 || note "seed $seed: test failed, see $LOGS/chess-$seed.log"
+  fi
+  note "seed $seed: done"
+}
+
+# Games take a core each and share nothing, so the seeds run at once.
+for seed in $SEEDS; do search $seed & runs+=($!); done
+( sleep $(( deadline - $(date +%s) )); note "deadline reached"; kill -TERM $$ ) &
+stopper=$!
+wait $runs
+kill $stopper 2>/dev/null
+rm -f $LOGS/driver.pid
+uv run python3 benchmarks/fitness.py runs/doom runs/chess benchmarks/results/games.json \
+  docs/assets/chess-stockfish.svg --tasks chess \
+  && note "all finished; chart rebuilt" || note "chart not rebuilt; see above"