assessment-bench 0.1.0__tar.gz

assessment_bench-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.coverage
+bench-out/
+.env

assessment_bench-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Michael Borck
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assessment_bench-0.1.0/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: assessment-bench
+Version: 0.1.0
+Summary: Benchmark assessment approaches: pure-LLM marking vs the family's signal-based observations, with repeated runs and agreement statistics.
+Project-URL: Homepage, https://github.com/michael-borck/assessment-bench
+Author: Michael Borck
+License: MIT License
+        
+        Copyright (c) 2026 Michael Borck
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.11
+Requires-Dist: assessment-lens>=0.2.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: rich>=13.7.0
+Provides-Extra: analysers
+Requires-Dist: assessment-lens[analysers]>=0.2.0; extra == 'analysers'
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: llm
+Requires-Dist: anthropic>=0.40.0; extra == 'llm'
+Requires-Dist: openai>=1.12.0; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# assessment-bench
+
+Part of the [lens family](https://github.com/michael-borck/lens-analysers).
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Benchmark assessment approaches.** Run one cohort through competing
+assessment arms — pure-LLM marking (the baseline) and the family's
+signal-based observations (`assessment-lens`) — with repeated runs,
+consistency statistics, and agreement against human marks.
+**The bench measures; it never marks.**
+
+> `assessment-bench` is a *bench* (a measurement product), not an `-analyser`
+> and not a marking tool. It exists to answer research questions like: *how
+> consistent is LLM marking across repeated runs and providers?* and *which
+> deterministic signals actually track human judgement?*
+
+## What it does
+
+```
+experiment.yaml (rubric + cohort + arms)
+  ├─ llm arm(s)    : submission + rubric → provider → score   × repetitions
+  ├─ signals arm   : assessment-lens → evidence values        (deterministic, once)
+  └─ human marks   : optional ground-truth CSV
+        ↓
+result.json + runs.csv + signals.csv + agreement.csv
+  • per-submission consistency: mean / median / std-dev / CV / reliability
+  • agreement: Pearson & Spearman of every arm mean and every numeric signal
+    against the human marks
+```
+
+## Install
+
+```bash
+# from source (family layout)
+uv venv && source .venv/bin/activate
+uv pip install -e ".[dev]"
+
+# the signals arm needs the analyser stack (bundle-analyser CLI on PATH):
+uv pip install -e ".[analysers]"
+
+# LLM arms (Anthropic, OpenAI, Ollama, OpenRouter):
+uv pip install -e ".[llm]"      # + export ANTHROPIC_API_KEY / OPENAI_API_KEY / ...
+```
+
+## Quick start
+
+```bash
+assessment-bench init experiment.yaml   # commented example config
+# edit: point at your rubric.yaml + submissions/, choose arms
+assessment-bench run experiment.yaml -o out/
+```
+
+LLM arms specify provider **and** model per arm — comparing
+`claude-haiku-4-5` vs `gpt-4o-mini` vs a local `llama3.1` via Ollama is just
+three arms in one config.
+
+## Relationship to the family
+
+- **Analysers** generate deterministic signals (assessment-agnostic).
+- **assessment-lens** maps signals to a rubric as observations — never scores.
+- **assessment-bench** measures both approaches against human judgement. The
+  LLM arm produces scores *because that is the approach under test*; the bench
+  treats them as data points, not grades for students.
+
+## Status
+
+**v0.1 scaffold.** Working today:
+
+- ✅ Experiment config (YAML) → cohort discovery → arms → structured results
+- ✅ LLM arm: multi-provider (anthropic / openai / ollama / openrouter), repeated
+  runs, strict `SCORE: x/y` extraction with scaled fallback
+- ✅ Signals arm: one `assessment-lens` pass; raw evidence values consumed
+  (not the presence-based coverage)
+- ✅ Consistency stats (ported from the original Rust prototype) + Pearson/Spearman
+  agreement vs human marks
+- 📋 Hybrid arm (LLM marking with analyser signals in context) — next
+- 📋 HTTP service + desktop shell for non-technical researchers — planned
+
+## Development
+
+```bash
+pytest -v
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

assessment_bench-0.1.0/README.md

@@ -0,0 +1,89 @@
+# assessment-bench
+
+Part of the [lens family](https://github.com/michael-borck/lens-analysers).
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Benchmark assessment approaches.** Run one cohort through competing
+assessment arms — pure-LLM marking (the baseline) and the family's
+signal-based observations (`assessment-lens`) — with repeated runs,
+consistency statistics, and agreement against human marks.
+**The bench measures; it never marks.**
+
+> `assessment-bench` is a *bench* (a measurement product), not an `-analyser`
+> and not a marking tool. It exists to answer research questions like: *how
+> consistent is LLM marking across repeated runs and providers?* and *which
+> deterministic signals actually track human judgement?*
+
+## What it does
+
+```
+experiment.yaml (rubric + cohort + arms)
+  ├─ llm arm(s)    : submission + rubric → provider → score   × repetitions
+  ├─ signals arm   : assessment-lens → evidence values        (deterministic, once)
+  └─ human marks   : optional ground-truth CSV
+        ↓
+result.json + runs.csv + signals.csv + agreement.csv
+  • per-submission consistency: mean / median / std-dev / CV / reliability
+  • agreement: Pearson & Spearman of every arm mean and every numeric signal
+    against the human marks
+```
+
+## Install
+
+```bash
+# from source (family layout)
+uv venv && source .venv/bin/activate
+uv pip install -e ".[dev]"
+
+# the signals arm needs the analyser stack (bundle-analyser CLI on PATH):
+uv pip install -e ".[analysers]"
+
+# LLM arms (Anthropic, OpenAI, Ollama, OpenRouter):
+uv pip install -e ".[llm]"      # + export ANTHROPIC_API_KEY / OPENAI_API_KEY / ...
+```
+
+## Quick start
+
+```bash
+assessment-bench init experiment.yaml   # commented example config
+# edit: point at your rubric.yaml + submissions/, choose arms
+assessment-bench run experiment.yaml -o out/
+```
+
+LLM arms specify provider **and** model per arm — comparing
+`claude-haiku-4-5` vs `gpt-4o-mini` vs a local `llama3.1` via Ollama is just
+three arms in one config.
+
+## Relationship to the family
+
+- **Analysers** generate deterministic signals (assessment-agnostic).
+- **assessment-lens** maps signals to a rubric as observations — never scores.
+- **assessment-bench** measures both approaches against human judgement. The
+  LLM arm produces scores *because that is the approach under test*; the bench
+  treats them as data points, not grades for students.
+
+## Status
+
+**v0.1 scaffold.** Working today:
+
+- ✅ Experiment config (YAML) → cohort discovery → arms → structured results
+- ✅ LLM arm: multi-provider (anthropic / openai / ollama / openrouter), repeated
+  runs, strict `SCORE: x/y` extraction with scaled fallback
+- ✅ Signals arm: one `assessment-lens` pass; raw evidence values consumed
+  (not the presence-based coverage)
+- ✅ Consistency stats (ported from the original Rust prototype) + Pearson/Spearman
+  agreement vs human marks
+- 📋 Hybrid arm (LLM marking with analyser signals in context) — next
+- 📋 HTTP service + desktop shell for non-technical researchers — planned
+
+## Development
+
+```bash
+pytest -v
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

assessment_bench-0.1.0/pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "assessment-bench"
+version = "0.1.0"
+description = "Benchmark assessment approaches: pure-LLM marking vs the family's signal-based observations, with repeated runs and agreement statistics."
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.11"
+authors = [{ name = "Michael Borck" }]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "assessment-lens>=0.2.0",
+    "pydantic>=2.5.0",
+    "pyyaml>=6.0.0",
+    "rich>=13.7.0",
+]
+
+[project.optional-dependencies]
+# The pure-LLM marking arm. anthropic for Anthropic; openai also covers
+# Ollama / OpenRouter / any OpenAI-compatible endpoint via base_url.
+llm = [
+    "anthropic>=0.40.0",
+    "openai>=1.12.0",
+]
+# Pull the analyser stack into the same env so the signals arm runs for real
+# (assessment-lens shells out to the bundle-analyser CLI).
+analysers = [
+    "assessment-lens[analysers]>=0.2.0",
+]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+assessment-bench = "assessment_bench.cli:main"
+
+# Local dev: resolve family members from sibling checkouts. uv strips this from
+# the published wheel, which keeps the plain PyPI pins.
+[tool.uv.sources]
+assessment-lens = { path = "../assessment-lens", editable = true }
+
+[project.urls]
+Homepage = "https://github.com/michael-borck/assessment-bench"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/assessment_bench"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

assessment_bench-0.1.0/src/assessment_bench/__init__.py

@@ -0,0 +1,45 @@
+"""assessment-bench — benchmark assessment approaches for the lens family.
+
+Runs one cohort through competing assessment arms (pure-LLM marking as the
+baseline; assessment-lens signal observations as the approach under study),
+with repeated runs, consistency statistics, and agreement against human marks.
+**The bench measures; it never marks.**
+"""
+
+from .exceptions import AssessmentBenchError
+from .experiment import load_config, run_experiment
+from .models import (
+    Agreement,
+    ArmKind,
+    ArmOutcome,
+    ArmSpec,
+    ExperimentConfig,
+    ExperimentResult,
+    GradeRun,
+    ProviderName,
+    ProviderSpec,
+    RunStats,
+    SignalReading,
+)
+from .report import write_results
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Agreement",
+    "ArmKind",
+    "ArmOutcome",
+    "ArmSpec",
+    "AssessmentBenchError",
+    "ExperimentConfig",
+    "ExperimentResult",
+    "GradeRun",
+    "ProviderName",
+    "ProviderSpec",
+    "RunStats",
+    "SignalReading",
+    "__version__",
+    "load_config",
+    "run_experiment",
+    "write_results",
+]

assessment_bench-0.1.0/src/assessment_bench/arms.py

@@ -0,0 +1,122 @@
+"""The assessment arms under test.
+
+LLM arm — the approach the family deliberately moved away from (an LLM reading
+a submission and emitting a mark), kept here as the benchmark baseline. Prompt
+shape ports the original Rust prototype's Tier-1 design; the score comes from a
+strict trailing ``SCORE: x/y`` line with a permissive regex fallback.
+
+Signals arm — assessment-lens observations. Deterministic, so it runs once per
+cohort regardless of repetitions; the bench consumes raw evidence values (not
+the presence-based coverage column) and correlates each numeric signal with the
+human marks.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from assessment_lens.assess import assess
+from assessment_lens.rubric import load_rubric
+
+from . import providers
+from .models import ArmSpec, GradeRun, SignalReading
+
+# Submission text for the LLM arm. Plain-text formats are read directly;
+# .pdf/.docx go through the family's canonical extractor when installed.
+_PLAIN_TEXT_SUFFIXES = {".md", ".txt", ".py", ".js", ".ts", ".r", ".sql", ".csv"}
+_EXTRACTOR_SUFFIXES = {".pdf", ".docx", ".pptx"}
+
+_GRADE_SYSTEM = (
+    "You are an experienced university marker. Grade the submission against the "
+    "rubric. Be consistent and justify briefly. End your response with exactly "
+    "one line in the form 'SCORE: <number>/<max>' and nothing after it."
+)
+
+_SCORE_RE = re.compile(r"SCORE:\s*(\d+(?:\.\d+)?)\s*/\s*(\d+(?:\.\d+)?)", re.IGNORECASE)
+_FALLBACK_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(?:/|out of)\s*(\d+(?:\.\d+)?)")
+
+
+def read_submission_text(folder: Path) -> str:
+    """Concatenate the readable artefacts in one submission folder."""
+    parts: list[str] = []
+    for path in sorted(folder.rglob("*")):
+        if not path.is_file():
+            continue
+        suffix = path.suffix.lower()
+        if suffix in _PLAIN_TEXT_SUFFIXES:
+            parts.append(f"--- {path.name} ---\n{path.read_text(errors='replace')}")
+        elif suffix in _EXTRACTOR_SUFFIXES:
+            try:
+                from document_analyser import extract_text
+
+                parts.append(f"--- {path.name} ---\n{extract_text(path)}")
+            except ImportError:
+                parts.append(f"--- {path.name} --- (skipped: install the [analysers] extra to extract {suffix})")
+    return "\n\n".join(parts)
+
+
+def extract_score(response: str, max_score: float) -> tuple[float | None, float]:
+    """Pull (score, max) from a response; scale to max_score when the LLM used its own denominator."""
+    matches = _SCORE_RE.findall(response) or _FALLBACK_RE.findall(response)
+    if not matches:
+        return None, max_score
+    raw, denom = (float(v) for v in matches[-1])
+    if denom and denom != max_score:
+        raw = raw / denom * max_score
+    return raw, max_score
+
+
+def grade_prompt(rubric_text: str, submission_text: str, max_score: float) -> str:
+    return (
+        f"RUBRIC:\n{rubric_text}\n\n"
+        f"SUBMISSION:\n{submission_text}\n\n"
+        f"Grade the submission against the rubric out of {max_score:g}. "
+        f"Give 2-3 sentences of rationale, then the final 'SCORE: x/{max_score:g}' line."
+    )
+
+
+def run_llm_arm(
+    arm: ArmSpec,
+    submission_id: str,
+    submission_folder: Path,
+    rubric_text: str,
+    max_score: float,
+) -> list[GradeRun]:
+    """All repetitions of one LLM arm for one submission. Failures are recorded, not raised."""
+    assert arm.provider is not None  # validated by ArmSpec
+    text = read_submission_text(submission_folder)
+    prompt = grade_prompt(rubric_text, text, max_score)
+    runs: list[GradeRun] = []
+    for i in range(arm.repetitions):
+        run = GradeRun(submission_id=submission_id, arm_id=arm.id, run_index=i, max_score=max_score)
+        try:
+            response = providers.complete(prompt, system=_GRADE_SYSTEM, spec=arm.provider)
+            run.raw_response = response
+            run.score, _ = extract_score(response, max_score)
+            run.rationale = _SCORE_RE.sub("", response).strip()
+            if run.score is None:
+                run.error = "no SCORE line found in response"
+        except Exception as exc:  # one bad call must not kill a cohort run
+            run.error = str(exc)
+        runs.append(run)
+    return runs
+
+
+def run_signals_arm(arm: ArmSpec, rubric_path: Path, submissions_dir: Path) -> list[SignalReading]:
+    """One assessment-lens pass over the whole cohort -> flat evidence readings."""
+    rubric = load_rubric(rubric_path)
+    result = assess(rubric, submissions_dir)
+    readings: list[SignalReading] = []
+    for submission in result.submissions:
+        for observation in submission.observations:
+            for evidence in observation.evidence:
+                readings.append(
+                    SignalReading(
+                        submission_id=submission.submission_id,
+                        criterion_id=observation.criterion_id,
+                        signal=evidence.signal,
+                        value=evidence.value,
+                    )
+                )
+    return readings

assessment_bench-0.1.0/src/assessment_bench/cli.py

@@ -0,0 +1,70 @@
+"""assessment-bench CLI.
+
+  assessment-bench run experiment.yaml -o out/
+  assessment-bench init my-experiment.yaml
+"""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+import sys
+from pathlib import Path
+
+from rich.console import Console
+
+from .exceptions import AssessmentBenchError
+from .experiment import load_config, run_experiment
+from .report import write_results
+
+console = Console()
+
+_EXAMPLE = Path(__file__).parent / "data" / "example-experiment.yaml"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="assessment-bench",
+        description="Benchmark assessment approaches over one cohort: pure-LLM marking vs signal-based observation.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run_p = sub.add_parser("run", help="Run an experiment config over its cohort.")
+    run_p.add_argument("config", type=Path, help="Experiment YAML.")
+    run_p.add_argument("-o", "--out", type=Path, default=Path("bench-out"), help="Output folder.")
+
+    init_p = sub.add_parser("init", help="Write a commented example experiment config.")
+    init_p.add_argument("path", type=Path, nargs="?", default=Path("experiment.yaml"))
+
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "init":
+            if args.path.exists():
+                console.print(f"[red]refusing to overwrite {args.path}[/red]")
+                return 1
+            shutil.copy(_EXAMPLE, args.path)
+            console.print(f"✓ wrote {args.path} — edit it, then: assessment-bench run {args.path}")
+            return 0
+
+        config = load_config(args.config)
+        console.print(
+            f"[bold]{config.name}[/bold] — {len(config.arms)} arms, max score {config.max_score:g}"
+        )
+        result = run_experiment(config, progress=lambda msg: console.print(f"  {msg}"))
+        written = write_results(result, args.out)
+        console.print(f"✓ {len(result.submissions)} submissions → " + ", ".join(str(p) for p in written))
+        if result.agreements:
+            console.print("[bold]Agreement with human marks:[/bold]")
+            for a in sorted(result.agreements, key=lambda a: -(a.pearson or -2)):
+                console.print(f"  {a.measure}: r={a.pearson:.3f} rho={a.spearman:.3f} (n={a.n})"
+                              if a.pearson is not None and a.spearman is not None
+                              else f"  {a.measure}: undefined (n={a.n})")
+        return 0
+    except AssessmentBenchError as exc:
+        console.print(f"[red]error:[/red] {exc}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

assessment_bench-0.1.0/src/assessment_bench/data/example-experiment.yaml

@@ -0,0 +1,27 @@
+# assessment-bench experiment config.
+# Paths are relative to this file. One subfolder under `submissions` = one submission.
+name: "My experiment"
+rubric: rubric.yaml          # assessment-lens structured rubric (criteria + pinned signals)
+submissions: submissions/
+max_score: 100
+# Optional ground truth — enables agreement statistics (Pearson/Spearman):
+# human_marks: marks.csv     # CSV with header: submission_id,mark
+arms:
+  # The baseline under test: an LLM reads the submission + rubric and emits a mark.
+  - id: llm-haiku
+    kind: llm
+    repetitions: 3           # repeated runs -> consistency stats (mean/CV/reliability)
+    provider:
+      provider: anthropic    # anthropic | openai | ollama | openrouter
+      model: claude-haiku-4-5
+      temperature: 0.1
+  # Local model via Ollama (any OpenAI-compatible endpoint works via base_url):
+  # - id: llm-local
+  #   kind: llm
+  #   repetitions: 3
+  #   provider:
+  #     provider: ollama
+  #     model: llama3.1
+  # Signal-based observation via assessment-lens (deterministic; runs once):
+  - id: signals
+    kind: signals

assessment_bench-0.1.0/src/assessment_bench/exceptions.py

@@ -0,0 +1,5 @@
+"""assessment-bench exception hierarchy."""
+
+
+class AssessmentBenchError(Exception):
+    """Base for everything the bench raises on purpose."""

assessment_bench-0.1.0/src/assessment_bench/experiment.py

@@ -0,0 +1,141 @@
+"""Experiment runner — the bench's orchestration spine.
+
+One experiment = one rubric + one cohort + N arms. Each LLM arm runs per
+submission x repetitions; the signals arm runs once per cohort (deterministic).
+Afterwards, every arm's mean score and every numeric signal is correlated
+against the human marks (when provided). The bench measures; it never marks.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+import yaml
+
+from . import arms, stats
+from .exceptions import AssessmentBenchError
+from .models import (
+    Agreement,
+    ArmKind,
+    ArmOutcome,
+    ExperimentConfig,
+    ExperimentResult,
+)
+
+
+def load_config(path: Path) -> ExperimentConfig:
+    """Load an experiment YAML; relative paths resolve against the config's folder."""
+    raw = yaml.safe_load(Path(path).read_text())
+    config = ExperimentConfig.model_validate(raw)
+    base = Path(path).resolve().parent
+    config.rubric = (base / config.rubric).resolve()
+    config.submissions = (base / config.submissions).resolve()
+    if config.human_marks is not None:
+        config.human_marks = (base / config.human_marks).resolve()
+    return config
+
+
+def discover_submissions(submissions_dir: Path) -> list[Path]:
+    """One subfolder = one submission, mirroring assessment-lens's discovery."""
+    if not submissions_dir.is_dir():
+        raise AssessmentBenchError(f"submissions folder not found: {submissions_dir}")
+    folders = sorted(p for p in submissions_dir.iterdir() if p.is_dir() and not p.name.startswith("."))
+    if not folders:
+        raise AssessmentBenchError(f"no submission subfolders in {submissions_dir}")
+    return folders
+
+
+def load_human_marks(path: Path) -> dict[str, float]:
+    """CSV with a header row: submission_id,mark."""
+    marks: dict[str, float] = {}
+    with open(path, newline="") as f:
+        for row in csv.DictReader(f):
+            marks[row["submission_id"].strip()] = float(row["mark"])
+    return marks
+
+
+def _agreements(
+    result: ExperimentResult, marks: dict[str, float]
+) -> list[Agreement]:
+    """Correlate every arm mean and every numeric signal with the human marks."""
+    agreements: list[Agreement] = []
+
+    # Arm means (LLM arms): pair each submission's mean score with its mark.
+    by_arm: dict[str, dict[str, float]] = {}
+    for outcome in result.outcomes:
+        if outcome.stats is not None:
+            by_arm.setdefault(outcome.arm_id, {})[outcome.submission_id] = outcome.stats.mean
+    # Numeric signals (signals arm): one measure per dotted signal path.
+    by_signal: dict[str, dict[str, float]] = {}
+    for outcome in result.outcomes:
+        for reading in outcome.signals:
+            if isinstance(reading.value, bool):
+                value = float(reading.value)
+            elif isinstance(reading.value, (int, float)):
+                value = float(reading.value)
+            else:
+                continue
+            by_signal.setdefault(reading.signal, {})[reading.submission_id] = value
+
+    for measure, values in {**by_arm, **by_signal}.items():
+        paired = [(values[s], marks[s]) for s in values if s in marks]
+        if len(paired) < 2:
+            continue
+        xs, ys = [p[0] for p in paired], [p[1] for p in paired]
+        agreements.append(
+            Agreement(
+                measure=measure,
+                n=len(paired),
+                pearson=stats.pearson(xs, ys),
+                spearman=stats.spearman(xs, ys),
+            )
+        )
+    return agreements
+
+
+def run_experiment(config: ExperimentConfig, *, progress=None) -> ExperimentResult:
+    """Run every arm over the cohort and assemble the structured result.
+
+    ``progress`` is an optional callable(str) for CLI/UI status lines.
+    """
+    say = progress or (lambda _msg: None)
+    submissions = discover_submissions(config.submissions)
+    rubric_text = config.rubric.read_text()
+    result = ExperimentResult(
+        name=config.name,
+        max_score=config.max_score,
+        submissions=[s.name for s in submissions],
+    )
+
+    for arm in config.arms:
+        if arm.kind is ArmKind.SIGNALS:
+            say(f"arm {arm.id}: assessment-lens over {len(submissions)} submissions")
+            readings = arms.run_signals_arm(arm, config.rubric, config.submissions)
+            for folder in submissions:
+                result.outcomes.append(
+                    ArmOutcome(
+                        submission_id=folder.name,
+                        arm_id=arm.id,
+                        signals=[r for r in readings if r.submission_id == folder.name],
+                    )
+                )
+        else:
+            for folder in submissions:
+                say(f"arm {arm.id}: {folder.name} x{arm.repetitions}")
+                runs = arms.run_llm_arm(arm, folder.name, folder, rubric_text, config.max_score)
+                scores = [r.score for r in runs if r.score is not None]
+                result.outcomes.append(
+                    ArmOutcome(
+                        submission_id=folder.name,
+                        arm_id=arm.id,
+                        runs=runs,
+                        stats=stats.run_stats(scores),
+                    )
+                )
+
+    if config.human_marks is not None:
+        marks = load_human_marks(config.human_marks)
+        result.agreements = _agreements(result, marks)
+
+    return result

assessment_bench-0.1.0/src/assessment_bench/models.py

@@ -0,0 +1,136 @@
+"""Core data models for assessment-bench.
+
+The bench is the family's *measurement* layer: it runs the same cohort through
+competing assessment arms and reports consistency and agreement. The design
+rule that shapes these models: **the bench measures; it never marks.** An LLM
+arm produces scores because that is the approach under test — the bench treats
+those scores as data points, not as grades for students. Human marks, when
+provided, are the ground truth everything is compared against.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+# --- Experiment side (input) -------------------------------------------------
+class ProviderName(str, Enum):
+    ANTHROPIC = "anthropic"
+    OPENAI = "openai"
+    OLLAMA = "ollama"
+    OPENROUTER = "openrouter"
+
+
+class ProviderSpec(BaseModel):
+    """Which LLM serves an arm. base_url covers Ollama / any OpenAI-compatible host."""
+
+    provider: ProviderName
+    model: str
+    base_url: str | None = None
+    temperature: float = 0.1
+    max_tokens: int = 1500
+
+
+class ArmKind(str, Enum):
+    LLM = "llm"          # pure-LLM marking: submission + rubric -> score
+    SIGNALS = "signals"  # assessment-lens observations: deterministic evidence values
+
+
+class ArmSpec(BaseModel):
+    """One assessment approach under test."""
+
+    id: str
+    kind: ArmKind
+    repetitions: int = Field(default=1, ge=1, le=50)
+    provider: ProviderSpec | None = None  # required for kind=llm
+
+    def model_post_init(self, __context: object) -> None:
+        if self.kind is ArmKind.LLM and self.provider is None:
+            raise ValueError(f"arm '{self.id}': kind=llm requires a provider")
+
+
+class ExperimentConfig(BaseModel):
+    """One experiment: a rubric, a cohort, the arms to compare.
+
+    Paths are resolved relative to the config file's directory by ``load_config``.
+    """
+
+    name: str
+    rubric: Path
+    submissions: Path
+    max_score: float = 100.0
+    human_marks: Path | None = Field(
+        default=None,
+        description="Optional CSV (submission_id,mark) of human ground-truth marks.",
+    )
+    arms: list[ArmSpec] = Field(min_length=1)
+
+
+# --- Result side (output) ----------------------------------------------------
+class GradeRun(BaseModel):
+    """One LLM grading call. score=None means extraction failed (kept, not hidden)."""
+
+    submission_id: str
+    arm_id: str
+    run_index: int
+    score: float | None = None
+    max_score: float
+    rationale: str = ""
+    raw_response: str = ""
+    error: str = ""
+
+
+class RunStats(BaseModel):
+    """Consistency statistics over one arm's repeated runs for one submission."""
+
+    n: int
+    mean: float
+    median: float
+    std_dev: float = Field(description="Sample standard deviation (n-1).")
+    coefficient_of_variation: float
+    min: float
+    max: float
+    reliability: float = Field(
+        description="1 - CV, floored at 0. A rough 'how repeatable was this arm' index."
+    )
+
+
+class SignalReading(BaseModel):
+    """One deterministic evidence value from the signals arm."""
+
+    submission_id: str
+    criterion_id: str
+    signal: str
+    value: object | None = None
+
+
+class ArmOutcome(BaseModel):
+    """Everything one arm produced for one submission."""
+
+    submission_id: str
+    arm_id: str
+    runs: list[GradeRun] = Field(default_factory=list)
+    stats: RunStats | None = None
+    signals: list[SignalReading] = Field(default_factory=list)
+
+
+class Agreement(BaseModel):
+    """Correlation between one measure and the human marks."""
+
+    measure: str = Field(description="An arm id (mean score) or a dotted signal path.")
+    n: int
+    pearson: float | None = None
+    spearman: float | None = None
+
+
+class ExperimentResult(BaseModel):
+    """The source-of-truth structured result for one experiment run."""
+
+    name: str
+    max_score: float
+    submissions: list[str] = Field(default_factory=list)
+    outcomes: list[ArmOutcome] = Field(default_factory=list)
+    agreements: list[Agreement] = Field(default_factory=list)

assessment_bench-0.1.0/src/assessment_bench/providers.py

@@ -0,0 +1,114 @@
+"""Multi-provider LLM completion for the pure-LLM marking arm.
+
+Provider registry adapted from image-analyser's caption providers: Anthropic via
+its own SDK; OpenAI, OpenRouter, and Ollama through the openai SDK (the latter
+two are OpenAI-compatible endpoints reached via base_url). Key resolution
+follows the family pattern — env var first, then a minimal .env fallback.
+
+Everything here is opt-in and degradable: callers catch ``LLMUnavailable`` and
+the experiment records the failure instead of dying mid-cohort.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .exceptions import AssessmentBenchError
+from .models import ProviderName, ProviderSpec
+
+PROVIDER_KEYS = {
+    ProviderName.ANTHROPIC: "ANTHROPIC_API_KEY",
+    ProviderName.OPENAI: "OPENAI_API_KEY",
+    ProviderName.OPENROUTER: "OPENROUTER_API_KEY",
+    ProviderName.OLLAMA: None,  # local; no key
+}
+
+DEFAULT_BASE_URLS = {
+    ProviderName.OPENROUTER: "https://openrouter.ai/api/v1",
+    ProviderName.OLLAMA: "http://localhost:11434/v1",
+}
+
+
+class LLMUnavailable(AssessmentBenchError):
+    """The [llm] extra is not installed or no API key is configured."""
+
+
+def _load_env_file() -> None:
+    """Minimal .env loader (cwd upward) — no python-dotenv dependency."""
+    for parent in [Path.cwd(), *Path.cwd().parents]:
+        env_file = parent / ".env"
+        if env_file.exists():
+            try:
+                for line in env_file.read_text().splitlines():
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        key, value = line.split("=", 1)
+                        os.environ.setdefault(key.strip(), value.strip().strip("\"'"))
+            except OSError:
+                pass
+            return
+
+
+def get_api_key(provider: ProviderName) -> str | None:
+    env_var = PROVIDER_KEYS.get(provider)
+    if env_var is None:
+        return "unused"  # Ollama: openai SDK requires a non-empty key
+    if key := os.getenv(env_var):
+        return key
+    _load_env_file()
+    return os.getenv(env_var)
+
+
+def complete(prompt: str, *, system: str, spec: ProviderSpec) -> str:
+    """One marking-style completion against the arm's configured provider."""
+    api_key = get_api_key(spec.provider)
+    if not api_key:
+        raise LLMUnavailable(
+            f"No API key for {spec.provider.value} — set {PROVIDER_KEYS[spec.provider]} (env or .env)."
+        )
+
+    if spec.provider is ProviderName.ANTHROPIC:
+        return _complete_anthropic(prompt, system=system, spec=spec, api_key=api_key)
+    return _complete_openai_compatible(prompt, system=system, spec=spec, api_key=api_key)
+
+
+def _complete_anthropic(prompt: str, *, system: str, spec: ProviderSpec, api_key: str) -> str:
+    try:
+        import anthropic
+    except ImportError as exc:
+        raise LLMUnavailable(
+            "LLM arms need the [llm] extra: pip install 'assessment-bench[llm]'"
+        ) from exc
+    client = anthropic.Anthropic(api_key=api_key)
+    response = client.messages.create(
+        model=spec.model,
+        max_tokens=spec.max_tokens,
+        temperature=spec.temperature,
+        system=system,
+        messages=[{"role": "user", "content": prompt}],
+    )
+    return "".join(block.text for block in response.content if block.type == "text").strip()
+
+
+def _complete_openai_compatible(
+    prompt: str, *, system: str, spec: ProviderSpec, api_key: str
+) -> str:
+    try:
+        import openai
+    except ImportError as exc:
+        raise LLMUnavailable(
+            "LLM arms need the [llm] extra: pip install 'assessment-bench[llm]'"
+        ) from exc
+    base_url = spec.base_url or DEFAULT_BASE_URLS.get(spec.provider)
+    client = openai.OpenAI(api_key=api_key, base_url=base_url)
+    response = client.chat.completions.create(
+        model=spec.model,
+        max_tokens=spec.max_tokens,
+        temperature=spec.temperature,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    return (response.choices[0].message.content or "").strip()

assessment_bench-0.1.0/src/assessment_bench/report.py

@@ -0,0 +1,58 @@
+"""Result writers: one JSON source of truth + flat CSVs for spreadsheet people."""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+from .models import ExperimentResult
+
+
+def write_results(result: ExperimentResult, out_dir: Path) -> list[Path]:
+    """Write result.json, runs.csv, signals.csv, agreement.csv. Returns written paths."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    written: list[Path] = []
+
+    json_path = out_dir / "result.json"
+    json_path.write_text(json.dumps(result.model_dump(mode="json"), indent=2))
+    written.append(json_path)
+
+    runs_path = out_dir / "runs.csv"
+    with open(runs_path, "w", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow(["submission", "arm", "run", "score", "max_score", "error"])
+        for outcome in result.outcomes:
+            for run in outcome.runs:
+                writer.writerow(
+                    [run.submission_id, run.arm_id, run.run_index, run.score, run.max_score, run.error]
+                )
+    written.append(runs_path)
+
+    signals_path = out_dir / "signals.csv"
+    with open(signals_path, "w", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow(["submission", "arm", "criterion", "signal", "value"])
+        for outcome in result.outcomes:
+            for reading in outcome.signals:
+                writer.writerow(
+                    [
+                        reading.submission_id,
+                        outcome.arm_id,
+                        reading.criterion_id,
+                        reading.signal,
+                        json.dumps(reading.value, default=str),
+                    ]
+                )
+    written.append(signals_path)
+
+    if result.agreements:
+        agreement_path = out_dir / "agreement.csv"
+        with open(agreement_path, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(["measure", "n", "pearson", "spearman"])
+            for a in result.agreements:
+                writer.writerow([a.measure, a.n, a.pearson, a.spearman])
+        written.append(agreement_path)
+
+    return written

assessment_bench-0.1.0/src/assessment_bench/stats.py

@@ -0,0 +1,79 @@
+"""Consistency and agreement statistics.
+
+The run-level statistics (mean/median/sample std-dev/CV/reliability) are a port
+of the original AssessmentBench Rust aggregation engine — the best-validated
+concept in that prototype. Agreement (Pearson/Spearman against human marks) is
+new here: it is the bench's core research output. Pure stdlib, no numpy.
+"""
+
+from __future__ import annotations
+
+from .models import RunStats
+
+
+def run_stats(scores: list[float]) -> RunStats | None:
+    """Consistency statistics over one arm's repeated scores. None when empty."""
+    if not scores:
+        return None
+    n = len(scores)
+    mean = sum(scores) / n
+    ordered = sorted(scores)
+    median = (
+        ordered[n // 2]
+        if n % 2
+        else (ordered[n // 2 - 1] + ordered[n // 2]) / 2.0
+    )
+    if n > 1:
+        variance = sum((s - mean) ** 2 for s in scores) / (n - 1)
+        std_dev = variance**0.5
+    else:
+        std_dev = 0.0
+    cv = std_dev / mean if mean else 0.0
+    return RunStats(
+        n=n,
+        mean=mean,
+        median=median,
+        std_dev=std_dev,
+        coefficient_of_variation=cv,
+        min=ordered[0],
+        max=ordered[-1],
+        reliability=max(0.0, 1.0 - cv),
+    )
+
+
+def pearson(xs: list[float], ys: list[float]) -> float | None:
+    """Pearson r. None when undefined (n<2 or zero variance) — never faked as 0."""
+    n = len(xs)
+    if n != len(ys) or n < 2:
+        return None
+    mx = sum(xs) / n
+    my = sum(ys) / n
+    sxx = sum((x - mx) ** 2 for x in xs)
+    syy = sum((y - my) ** 2 for y in ys)
+    if sxx == 0 or syy == 0:
+        return None
+    sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
+    return sxy / (sxx**0.5 * syy**0.5)
+
+
+def _ranks(values: list[float]) -> list[float]:
+    """Average ranks (ties share the mean of their rank positions)."""
+    indexed = sorted(range(len(values)), key=lambda i: values[i])
+    ranks = [0.0] * len(values)
+    i = 0
+    while i < len(indexed):
+        j = i
+        while j + 1 < len(indexed) and values[indexed[j + 1]] == values[indexed[i]]:
+            j += 1
+        avg_rank = (i + j) / 2.0 + 1.0
+        for k in range(i, j + 1):
+            ranks[indexed[k]] = avg_rank
+        i = j + 1
+    return ranks
+
+
+def spearman(xs: list[float], ys: list[float]) -> float | None:
+    """Spearman rho = Pearson on average ranks. None when undefined."""
+    if len(xs) != len(ys) or len(xs) < 2:
+        return None
+    return pearson(_ranks(xs), _ranks(ys))

assessment_bench-0.1.0/tests/test_arms.py

@@ -0,0 +1,36 @@
+"""Score extraction — the riskiest parsing in the bench (the Rust prototype never tested its)."""
+
+import pytest
+
+from assessment_bench.arms import extract_score, grade_prompt
+
+
+def test_strict_score_line():
+    score, _ = extract_score("Good work overall.\nSCORE: 78/100", 100.0)
+    assert score == pytest.approx(78.0)
+
+
+def test_last_score_line_wins():
+    text = "If perfect this would be SCORE: 100/100, but...\nSCORE: 62.5/100"
+    score, _ = extract_score(text, 100.0)
+    assert score == pytest.approx(62.5)
+
+
+def test_scaled_to_max_score():
+    score, _ = extract_score("SCORE: 7/10", 100.0)
+    assert score == pytest.approx(70.0)
+
+
+def test_fallback_out_of_phrasing():
+    score, _ = extract_score("I would award 41 out of 50 for this.", 100.0)
+    assert score == pytest.approx(82.0)
+
+
+def test_no_score_returns_none():
+    score, _ = extract_score("This is thoughtful work with clear structure.", 100.0)
+    assert score is None
+
+
+def test_grade_prompt_carries_parts():
+    p = grade_prompt("RUB", "SUB", 50.0)
+    assert "RUB" in p and "SUB" in p and "SCORE: x/50" in p

assessment_bench-0.1.0/tests/test_models.py

@@ -0,0 +1,44 @@
+"""Config models: the experiment YAML is the bench's central contract."""
+
+import pytest
+from pydantic import ValidationError
+
+from assessment_bench.models import ArmKind, ArmSpec, ExperimentConfig
+
+
+def test_llm_arm_requires_provider():
+    with pytest.raises((ValidationError, ValueError)):
+        ArmSpec(id="bad", kind=ArmKind.LLM)
+
+
+def test_signals_arm_needs_no_provider():
+    arm = ArmSpec(id="signals", kind=ArmKind.SIGNALS)
+    assert arm.repetitions == 1
+
+
+def test_experiment_config_parses():
+    config = ExperimentConfig.model_validate(
+        {
+            "name": "t",
+            "rubric": "rubric.yaml",
+            "submissions": "subs/",
+            "arms": [
+                {
+                    "id": "llm",
+                    "kind": "llm",
+                    "repetitions": 3,
+                    "provider": {"provider": "ollama", "model": "llama3.1"},
+                },
+                {"id": "signals", "kind": "signals"},
+            ],
+        }
+    )
+    assert config.max_score == 100.0
+    assert config.arms[0].provider.base_url is None
+
+
+def test_experiment_config_requires_an_arm():
+    with pytest.raises(ValidationError):
+        ExperimentConfig.model_validate(
+            {"name": "t", "rubric": "r.yaml", "submissions": "s/", "arms": []}
+        )

assessment_bench-0.1.0/tests/test_stats.py

@@ -0,0 +1,44 @@
+"""Stats: the ported Rust aggregation math + the new agreement correlations."""
+
+import pytest
+
+from assessment_bench.stats import pearson, run_stats, spearman
+
+
+def test_run_stats_basic():
+    s = run_stats([80.0, 85.0, 90.0])
+    assert s.n == 3
+    assert s.mean == pytest.approx(85.0)
+    assert s.median == pytest.approx(85.0)
+    assert s.std_dev == pytest.approx(5.0)  # sample std-dev (n-1)
+    assert s.coefficient_of_variation == pytest.approx(5.0 / 85.0)
+    assert s.min == 80.0 and s.max == 90.0
+    assert s.reliability == pytest.approx(1.0 - 5.0 / 85.0)
+
+
+def test_run_stats_even_median_and_single():
+    assert run_stats([1.0, 2.0, 3.0, 4.0]).median == pytest.approx(2.5)
+    single = run_stats([70.0])
+    assert single.std_dev == 0.0 and single.reliability == 1.0
+    assert run_stats([]) is None
+
+
+def test_pearson_perfect_and_inverse():
+    assert pearson([1, 2, 3], [10, 20, 30]) == pytest.approx(1.0)
+    assert pearson([1, 2, 3], [30, 20, 10]) == pytest.approx(-1.0)
+
+
+def test_pearson_undefined_not_faked():
+    assert pearson([1, 1, 1], [1, 2, 3]) is None  # zero variance
+    assert pearson([1], [1]) is None  # n < 2
+
+
+def test_spearman_monotonic_nonlinear():
+    # Monotonic but nonlinear: rho is 1 even where r is not.
+    xs = [1.0, 2.0, 3.0, 4.0]
+    ys = [1.0, 10.0, 100.0, 1000.0]
+    assert spearman(xs, ys) == pytest.approx(1.0)
+
+
+def test_spearman_ties_average_ranks():
+    assert spearman([1.0, 1.0, 2.0], [1.0, 1.0, 2.0]) == pytest.approx(1.0)