cattykit 0.1.2__tar.gz → 0.1.3__tar.gz

{cattykit-0.1.2 → cattykit-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cattykit
-Version: 0.1.2
+Version: 0.1.3
 Summary: A framework for implementing artificial intelligence models using components from the cognitive models of the Fluid Analogies Research Group
 Author-email: George Alexander Wright <me@georgeaw.com>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE

{cattykit-0.1.2 → cattykit-0.1.3}/cattykit/__init__.py

@@ -1,8 +1,9 @@
-from .experiments import run_experiment
+from .experiments import ExperimentResult, run_experiment
 from .models import available_models, install_model, load_model, model_info
 
 __all__ = [
     "available_models",
+    "ExperimentResult",
     "install_model",
     "load_model",
     "model_info",

cattykit-0.1.3/cattykit/experiments/__init__.py

@@ -0,0 +1,25 @@
+"""Public experiment-running and statistical-analysis API."""
+
+from .reproduction import reproduction_summary, run_reproduction
+from .runner import ExperimentResult, run_experiment
+from .statistics import (
+    chi_square_survival_function,
+    mean_absolute_error,
+    summarize_runs,
+    total_variation_distance,
+    two_sided_p_value,
+    z_statistic,
+)
+
+__all__ = [
+    "ExperimentResult",
+    "chi_square_survival_function",
+    "mean_absolute_error",
+    "run_experiment",
+    "run_reproduction",
+    "reproduction_summary",
+    "summarize_runs",
+    "total_variation_distance",
+    "two_sided_p_value",
+    "z_statistic",
+]

cattykit-0.1.3/cattykit/experiments/reproduction.py

@@ -0,0 +1,226 @@
+"""Comparison of repeated model runs with reference behaviour."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from .runner import run_experiment
+from .statistics import total_variation_distance, z_statistic
+
+
+def run_reproduction(
+    model_name: str,
+    gold_behaviour: dict[str, pd.DataFrame | list[dict[str, Any]]],
+    iterations: int,
+    output_file: str | Path | None = None,
+) -> tuple[pd.DataFrame, dict[str, Any]]:
+    """Run a model and compare its standard summaries with reference behaviour.
+
+    ``gold_behaviour`` maps each problem to rows using the schema returned by
+    :func:`summarize_runs`, excluding its ``problem`` column.  The comparison
+    contains one row for each reference answer and each problem's ``Total``
+    row.  This makes the routine suitable for any Cattykit model that records
+    the standard answer, temperature, and codelet-count measurements.
+    """
+    observed = run_experiment(
+        model_name, list(gold_behaviour), iterations, verbose=False
+    ).summary
+    rows: list[dict[str, object]] = []
+    for problem, behaviour in gold_behaviour.items():
+        expected = _as_dataframe(behaviour)
+        actual = observed.loc[observed["problem"] == problem]
+        actual_answers = actual.loc[actual["solution"] != "Total"]
+        expected_answers = expected.loc[expected["solution"] != "Total"]
+        rows.append(
+            _comparison_row(
+                problem=problem,
+                solution="Total",
+                observed=_total_row(actual, problem, "observed"),
+                expected=_total_row(expected, problem, "reference"),
+                tv_distance=total_variation_distance(
+                    dict(
+                        zip(
+                            actual_answers["solution"],
+                            actual_answers["frequency"],
+                            strict=True,
+                        )
+                    ),
+                    dict(
+                        zip(
+                            expected_answers["solution"],
+                            expected_answers["frequency"],
+                            strict=True,
+                        )
+                    ),
+                ),
+                codelets_z_stat=_z_statistic(
+                    _total_row(actual, problem, "observed"),
+                    _total_row(expected, problem, "reference"),
+                    "mean_codelets_run",
+                    "codelets_standard_error",
+                ),
+            )
+        )
+        for expected_answer in expected_answers.itertuples(index=False):
+            matches = actual_answers.loc[
+                actual_answers["solution"] == expected_answer.solution
+            ]
+            actual_answer = None if matches.empty else matches.iloc[0]
+            rows.append(
+                _comparison_row(
+                    problem=problem,
+                    solution=str(expected_answer.solution),
+                    observed=actual_answer,
+                    expected=expected_answer._asdict(),
+                    temperature_z_stat=(
+                        None
+                        if actual_answer is None
+                        else _z_statistic(
+                            actual_answer,
+                            expected_answer._asdict(),
+                            "mean_temperature",
+                            "temperature_standard_error",
+                        )
+                    ),
+                )
+            )
+    comparison = pd.DataFrame(rows)
+    if output_file is not None:
+        comparison.to_csv(output_file, index=False)
+    return comparison, reproduction_summary(comparison)
+
+
+def reproduction_summary(comparison: pd.DataFrame) -> dict[str, Any]:
+    """Summarize the generic error measures in a reproduction comparison.
+
+    Temperature errors are weighted by the reference answer frequency.  The
+    maximum temperature metric is the maximum of the frequency-weighted mean
+    errors for each problem, rather than the error of a single answer.
+    """
+    totals = comparison.loc[comparison["solution"] == "Total"]
+    answer_rows = comparison.loc[comparison["solution"] != "Total"]
+    temperature_rows = answer_rows.loc[
+        answer_rows["mean_temperature_difference"].notna()
+        & answer_rows["expected_frequency"].notna()
+    ].copy()
+    temperature_rows["absolute_temperature_error"] = temperature_rows[
+        "mean_temperature_difference"
+    ].abs()
+    temperature_mean_error = _frequency_weighted_mean(temperature_rows)
+    problem_temperature_mean_errors = {
+        problem: _frequency_weighted_mean(rows)
+        for problem, rows in temperature_rows.groupby("problem", sort=False)
+    }
+    codelets_errors = totals["codelets_relative_error_magnitude"].dropna()
+    return {
+        "mean_total_variation_distance": totals["tv_distance"].mean(),
+        "max_total_variation_distance": totals["tv_distance"].max(),
+        "mean_temperature_absolute_error_magnitude": temperature_mean_error,
+        "max_temperature_absolute_error_magnitude": max(
+            problem_temperature_mean_errors.values(), default=0.0
+        ),
+        "temperature_absolute_error_magnitude_by_problem": (
+            problem_temperature_mean_errors
+        ),
+        "mean_codelets_relative_error_magnitude": codelets_errors.mean(),
+        "max_codelets_relative_error_magnitude": codelets_errors.max(),
+        "codelets_relative_error_magnitude_quantiles": {
+            percentile: codelets_errors.quantile(percentile / 100)
+            for percentile in range(10, 101, 10)
+        },
+    }
+
+
+def _frequency_weighted_mean(rows: pd.DataFrame) -> float:
+    """Return the reference-frequency-weighted temperature error for rows."""
+    total_frequency = rows["expected_frequency"].sum()
+    if total_frequency == 0:
+        return 0.0
+    return float(
+        (rows["absolute_temperature_error"] * rows["expected_frequency"]).sum()
+        / total_frequency
+    )
+
+
+def _as_dataframe(behaviour: pd.DataFrame | list[dict[str, Any]]) -> pd.DataFrame:
+    if isinstance(behaviour, pd.DataFrame):
+        return behaviour.copy()
+    return pd.DataFrame(behaviour)
+
+
+def _total_row(results: pd.DataFrame, problem: str, description: str) -> pd.Series:
+    totals = results.loc[results["solution"] == "Total"]
+    if len(totals) != 1:
+        raise ValueError(
+            f"Expected exactly one Total row for {description} behaviour of "
+            f"{problem!r}, found {len(totals)}."
+        )
+    return totals.iloc[0]
+
+
+def _z_statistic(
+    observed: pd.Series | dict[str, Any],
+    expected: pd.Series | dict[str, Any],
+    mean_column: str,
+    standard_error_column: str,
+) -> float | None:
+    values = (
+        observed[mean_column],
+        expected[mean_column],
+        observed[standard_error_column],
+        expected[standard_error_column],
+    )
+    return None if any(pd.isna(value) for value in values) else z_statistic(*values)
+
+
+def _comparison_row(
+    *,
+    problem: str,
+    solution: str,
+    observed: pd.Series | None,
+    expected: pd.Series | dict[str, Any],
+    tv_distance: float | None = None,
+    codelets_z_stat: float | None = None,
+    temperature_z_stat: float | None = None,
+) -> dict[str, object]:
+    row: dict[str, object] = {
+        "problem": problem,
+        "solution": solution,
+        "tv_distance": tv_distance,
+        "codelets_z_stat": codelets_z_stat,
+        "temperature_z_stat": temperature_z_stat,
+    }
+    for field in (
+        "frequency",
+        "mean_codelets_run",
+        "codelets_standard_error",
+        "mean_temperature",
+        "temperature_standard_error",
+    ):
+        actual = (
+            0 if observed is None and field == "frequency" else _value(observed, field)
+        )
+        reference = _value(expected, field)
+        row[f"observed_{field}"] = actual
+        row[f"expected_{field}"] = reference
+        row[f"{field}_difference"] = (
+            actual - reference if actual is not None and reference is not None else None
+        )
+    observed_codelets = row["observed_mean_codelets_run"]
+    expected_codelets = row["expected_mean_codelets_run"]
+    row["codelets_relative_error_magnitude"] = (
+        abs(observed_codelets - expected_codelets) / abs(expected_codelets)
+        if observed_codelets is not None and expected_codelets not in (None, 0)
+        else None
+    )
+    return row
+
+
+def _value(row: pd.Series | dict[str, Any] | None, field: str) -> object | None:
+    if row is None:
+        return None
+    value = row[field]
+    return None if pd.isna(value) else value

cattykit-0.1.3/cattykit/experiments/runner.py

@@ -0,0 +1,122 @@
+"""Execution of repeatable model experiments."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from shutil import get_terminal_size
+from typing import Any
+
+import pandas as pd
+
+from ..logging import CattycamLogger, NullLogger
+from ..models import load_model
+from .statistics import SUMMARY_COLUMNS, summarize_runs
+
+RAW_RESULT_COLUMNS = ["problem", "random_seed", "answer", "codelets_run", "temperature"]
+SummaryFunction = Callable[[list[dict[str, Any]], str], pd.DataFrame]
+ProgressCallback = Callable[[int, int, str], None]
+
+
+class RunningAnswerDisplay:
+    """Render an updating terminal view of a running experiment summary."""
+
+    def __init__(self) -> None:
+        self.lines_printed = 0
+
+    def update(self, results: pd.DataFrame) -> None:
+        table = results.drop(columns="problem").to_string(index=False)
+        if self.lines_printed:
+            print(f"\033[{self.lines_printed}A", end="")
+            for _ in range(self.lines_printed):
+                print("\033[2K")
+            print(f"\033[{self.lines_printed}A", end="")
+        lines = table.splitlines()
+        for line in lines:
+            print(f"\033[2K{line}")
+        width = get_terminal_size(fallback=(80, 24)).columns
+        self.lines_printed = sum(max(1, (len(line) + width - 1) // width) for line in lines)
+
+
+@dataclass
+class ExperimentResult:
+    """Aggregate statistics and the individual runs that produced them."""
+
+    summary: pd.DataFrame
+    raw_runs: pd.DataFrame
+
+    def save_summary_csv(self, path: str | Path) -> None:
+        self.summary.to_csv(path, index=False)
+
+    def save_raw_runs_csv(self, path: str | Path) -> None:
+        self.raw_runs.to_csv(path, index=False)
+
+
+def run_experiment(
+    model_name: str,
+    problems: list[str],
+    iterations: int,
+    logging_db: str | Path | None = None,
+    verbose: bool = True,
+    record_attributes: list[str] | None = None,
+    summary_function: SummaryFunction = summarize_runs,
+    progress_callback: ProgressCallback | None = None,
+) -> ExperimentResult:
+    """Run every problem once per seed with configurable recordings and summaries."""
+    record_attributes = record_attributes or []
+    reserved = set(RAW_RESULT_COLUMNS)
+    duplicates = reserved.intersection(record_attributes)
+    if duplicates:
+        raise ValueError(
+            f"record_attributes duplicate standard fields: {sorted(duplicates)}"
+        )
+    interactive = verbose and sys.stdout.isatty()
+    summaries, raw_runs = [], []
+    logger = (
+        CattycamLogger(logging_db, model_name) if logging_db else NullLogger(model_name)
+    )
+    try:
+        for problem_number, problem in enumerate(problems, start=1):
+            display = RunningAnswerDisplay() if interactive else None
+            runs = []
+            for seed in range(iterations):
+                model = load_model(model_name, config={"seed": seed}, logger=logger)
+                try:
+                    answer = model.solve(problem)
+                    run: dict[str, Any] = {
+                        "solution": answer,
+                        "codelets_run": model.coderack.number_of_codelets_run,
+                        "temperature": model.temperature,
+                    }
+                    run.update(
+                        {
+                            attribute: getattr(model, attribute)
+                            for attribute in record_attributes
+                        }
+                    )
+                    runs.append(run)
+                    raw_runs.append(
+                        {
+                            "problem": problem,
+                            "random_seed": seed,
+                            "answer": answer,
+                            **{key: run[key] for key in run if key != "solution"},
+                        }
+                    )
+                    if display:
+                        display.update(summary_function(runs, problem))
+                finally:
+                    model.close()
+            summaries.append(summary_function(runs, problem))
+            if progress_callback is not None:
+                progress_callback(problem_number, len(problems), problem)
+    finally:
+        logger.close()
+    return ExperimentResult(
+        pd.concat(summaries, ignore_index=True)
+        if summaries
+        else pd.DataFrame(columns=SUMMARY_COLUMNS),
+        pd.DataFrame(raw_runs, columns=[*RAW_RESULT_COLUMNS, *record_attributes]),
+    )

cattykit-0.1.3/cattykit/experiments/statistics.py

@@ -0,0 +1,171 @@
+from collections.abc import Iterable
+from math import erfc, exp, inf, isfinite, lgamma, log, sqrt
+
+import pandas as pd
+
+SUMMARY_COLUMNS = [
+    "problem",
+    "solution",
+    "frequency",
+    "mean_codelets_run",
+    "codelets_standard_error",
+    "mean_temperature",
+    "temperature_standard_error",
+]
+
+
+def mean_absolute_error(
+    observed: Iterable[float],
+    expected: Iterable[float],
+) -> float:
+    """Return the mean absolute difference between equal-length measurements."""
+    pairs = list(zip(observed, expected, strict=True))
+    return (
+        sum(abs(actual - reference) for actual, reference in pairs) / len(pairs)
+        if pairs
+        else 0.0
+    )
+
+
+def chi_square_survival_function(statistic: float, degrees_of_freedom: int) -> float:
+    """Return the chi-square survival probability without a SciPy dependency.
+
+    This evaluates the regularized upper incomplete gamma function at
+    ``statistic / 2``.
+    """
+    if degrees_of_freedom < 1:
+        return 1.0
+    if statistic <= 0:
+        return 1.0
+    if not isfinite(statistic):
+        return 0.0 if statistic == inf else float("nan")
+    return _regularized_gamma_q(degrees_of_freedom / 2, statistic / 2)
+
+
+def _regularized_gamma_q(shape: float, value: float) -> float:
+    """Evaluate the regularized upper incomplete gamma function."""
+    epsilon = 1e-14
+    minimum = 1e-300
+    if value < shape + 1:
+        term = 1 / shape
+        total = term
+        for index in range(1, 10_000):
+            term *= value / (shape + index)
+            total += term
+            if abs(term) <= abs(total) * epsilon:
+                break
+        return 1 - total * exp(-value + shape * log(value) - lgamma(shape))
+
+    denominator = value + 1 - shape
+    continued_fraction = 1 / minimum
+    reciprocal = 1 / denominator
+    result = reciprocal
+    for index in range(1, 10_000):
+        numerator = -index * (index - shape)
+        denominator += 2
+        reciprocal = numerator * reciprocal + denominator
+        if abs(reciprocal) < minimum:
+            reciprocal = minimum
+        continued_fraction = denominator + numerator / continued_fraction
+        if abs(continued_fraction) < minimum:
+            continued_fraction = minimum
+        reciprocal = 1 / reciprocal
+        delta = reciprocal * continued_fraction
+        result *= delta
+        if abs(delta - 1) <= epsilon:
+            break
+    return exp(-value + shape * log(value) - lgamma(shape)) * result
+
+
+def total_variation_distance(
+    observed: dict[str, int],
+    expected: dict[str, int],
+) -> float:
+    """Return total variation distance between two discrete distributions."""
+    observed_total = sum(observed.values())
+    expected_total = sum(expected.values())
+    if observed_total == 0 or expected_total == 0:
+        raise ValueError("Distributions must contain at least one observation.")
+    answers = observed.keys() | expected.keys()
+    return 0.5 * sum(
+        abs(
+            observed.get(answer, 0) / observed_total
+            - expected.get(answer, 0) / expected_total
+        )
+        for answer in answers
+    )
+
+
+def z_statistic(
+    observed_mean: float,
+    expected_mean: float,
+    observed_standard_error: float,
+    expected_standard_error: float,
+) -> float:
+    """Return the z statistic for two means with independent errors."""
+    standard_error = sqrt(observed_standard_error**2 + expected_standard_error**2)
+    difference = observed_mean - expected_mean
+    if standard_error == 0:
+        if difference == 0:
+            return 0.0
+        return float("inf") if difference > 0 else float("-inf")
+    return difference / standard_error
+
+
+def two_sided_p_value(z_score: float) -> float:
+    """Return the two-sided normal-distribution p value for a z statistic."""
+    return erfc(abs(z_score) / sqrt(2))
+
+
+def summarize_runs(
+    runs: list[dict[str, str | float | int | None]],
+    problem: str,
+) -> pd.DataFrame:
+    """Summarize individual runs, including an overall totals row."""
+    data = pd.DataFrame(runs)
+    if data.empty:
+        return pd.DataFrame(columns=SUMMARY_COLUMNS)
+    grouped = data.groupby(
+        "solution",
+        sort=False,
+    )
+    summary = grouped.agg(
+        frequency=("solution", "size"),
+        mean_codelets_run=("codelets_run", "mean"),
+        codelets_standard_error=("codelets_run", "sem"),
+        mean_temperature=("temperature", "mean"),
+        temperature_standard_error=("temperature", "sem"),
+    ).reset_index()
+    summary = summary.sort_values(
+        "frequency",
+        ascending=False,
+        kind="stable",
+    )
+    error_columns = [
+        "codelets_standard_error",
+        "temperature_standard_error",
+    ]
+    summary[error_columns] = summary[error_columns].fillna(0.0)
+    totals = pd.DataFrame(
+        [
+            {
+                "solution": "Total",
+                "frequency": len(data),
+                "mean_codelets_run": (data["codelets_run"].mean()),
+                "codelets_standard_error": (data["codelets_run"].sem()),
+                "mean_temperature": (data["temperature"].mean()),
+                "temperature_standard_error": (data["temperature"].sem()),
+            }
+        ]
+    )
+    totals[error_columns] = totals[error_columns].fillna(0.0)
+    summary = pd.concat(
+        [summary, totals],
+        ignore_index=True,
+    )
+    summary.insert(
+        0,
+        "problem",
+        problem,
+    )
+    return summary[SUMMARY_COLUMNS]

{cattykit-0.1.2 → cattykit-0.1.3}/cattykit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cattykit
-Version: 0.1.2
+Version: 0.1.3
 Summary: A framework for implementing artificial intelligence models using components from the cognitive models of the Fluid Analogies Research Group
 Author-email: George Alexander Wright <me@georgeaw.com>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE

{cattykit-0.1.2 → cattykit-0.1.3}/cattykit.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ README.md
 pyproject.toml
 cattykit/__init__.py
 cattykit/cli.py
-cattykit/experiments.py
 cattykit/py.typed
 cattykit.egg-info/PKG-INFO
 cattykit.egg-info/SOURCES.txt
@@ -11,6 +10,10 @@ cattykit.egg-info/dependency_links.txt
 cattykit.egg-info/entry_points.txt
 cattykit.egg-info/requires.txt
 cattykit.egg-info/top_level.txt
+cattykit/experiments/__init__.py
+cattykit/experiments/reproduction.py
+cattykit/experiments/runner.py
+cattykit/experiments/statistics.py
 cattykit/logging/__init__.py
 cattykit/logging/cattycam_logger.py
 cattykit/logging/identifiers.py

{cattykit-0.1.2 → cattykit-0.1.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cattykit"
-version = "0.1.2"
+version = "0.1.3"
 description = "A framework for implementing artificial intelligence models using components from the cognitive models of the Fluid Analogies Research Group"
 readme = "README.md"
 authors = [{name="George Alexander Wright", email="me@georgeaw.com"}]

cattykit-0.1.2/cattykit/experiments.py

@@ -1,602 +0,0 @@
-import sys
-from math import erfc, exp, inf, isfinite, lgamma, log, sqrt
-from pathlib import Path
-from shutil import get_terminal_size
-
-import pandas as pd
-
-from .logging import CattycamLogger, NullLogger
-from .models import load_model
-
-
-SUMMARY_COLUMNS = [
-    "problem",
-    "solution",
-    "frequency",
-    "mean_codelets_run",
-    "codelets_standard_error",
-    "mean_temperature",
-    "temperature_standard_error",
-]
-
-
-def run_experiment(
-    model_name: str,
-    problems: list[str],
-    iterations: int,
-    logging_db: str | Path | None = None,
-    verbose: bool = True,
-) -> pd.DataFrame:
-    """Run the model repeatedly on each problem.
-    The iteration count is used as the random seed given to the model, so the
-    same random seeds are used for each problem.
-    Returns one results DataFrame containing one row per solution plus a
-    totals row for each problem.
-    """
-    interactive = verbose and sys.stdout.isatty()
-    result_frames: list[pd.DataFrame] = []
-    logger = (
-        CattycamLogger(logging_db, model_name)
-        if logging_db is not None
-        else NullLogger(model_name)
-    )
-    try:
-        for problem_number, problem in enumerate(problems):
-            if interactive:
-                if problem_number:
-                    print(_terminal_rule("="))
-                print(problem)
-                print(_terminal_rule("-"))
-                display = RunningAnswerDisplay()
-            runs: list[dict[str, str | float | int | None]] = []
-            for seed in range(iterations):
-                model = load_model(
-                    model_name,
-                    config={"seed": seed},
-                    logger=logger,
-                )
-                try:
-                    answer = model.solve(problem)
-                    runs.append(
-                        {
-                            "solution": answer,
-                            "codelets_run": model.coderack.number_of_codelets_run,
-                            "temperature": model.temperature,
-                        }
-                    )
-                    if interactive:
-                        display.update(summarize_runs(runs, problem))
-                finally:
-                    model.close()
-            result_frames.append(summarize_runs(runs, problem))
-    finally:
-        logger.close()
-    if not result_frames:
-        return pd.DataFrame(columns=SUMMARY_COLUMNS)
-    return pd.concat(result_frames, ignore_index=True)
-
-
-def run_reproduction(
-    model_name: str,
-    gold_behaviour: dict[str, pd.DataFrame | list[dict]],
-    iterations: int,
-    verbose: bool = True,
-    output_file: str | None = None,
-) -> tuple[pd.DataFrame, dict[str, object]]:
-    """Compare current model behaviour with previously published behaviour.
-    ``gold_behaviour`` maps each problem to a summary table with the same
-    columns produced by ``summarize_runs``, except for ``problem``.
-    The returned table has paired observed and expected columns for every
-    expected solution and totals row. Differences are observed minus expected.
-    """
-    problems = list(gold_behaviour.keys())
-    observed = run_experiment(
-        model_name,
-        problems,
-        iterations,
-        verbose=False,
-    )
-    gold = _gold_behaviour_dataframe(gold_behaviour)
-    comparison_rows: list[dict[str, str | int | float | None]] = []
-    for problem in problems:
-        observed_problem = observed[observed["problem"] == problem]
-        gold_problem = gold[gold["problem"] == problem]
-        observed_solutions = observed_problem[observed_problem["solution"] != "Total"]
-        gold_solutions = gold_problem[gold_problem["solution"] != "Total"]
-        observed_distribution = dict(
-            zip(
-                observed_solutions["solution"],
-                observed_solutions["frequency"],
-            )
-        )
-        gold_distribution = dict(
-            zip(
-                gold_solutions["solution"],
-                gold_solutions["frequency"],
-            )
-        )
-        tv_distance = total_variation_distance(
-            observed_distribution,
-            gold_distribution,
-        )
-        observed_total = _total_row(
-            observed_problem,
-            problem,
-            "observed",
-        )
-        gold_total = _total_row(
-            gold_problem,
-            problem,
-            "gold",
-        )
-        codelets_z = z_statistic(
-            observed_mean=observed_total["mean_codelets_run"],
-            expected_mean=gold_total["mean_codelets_run"],
-            observed_standard_error=observed_total["codelets_standard_error"],
-            expected_standard_error=gold_total["codelets_standard_error"],
-        )
-        comparison_rows.append(
-            _comparison_row(
-                problem=problem,
-                solution="Total",
-                observed=observed_total,
-                expected=gold_total,
-                tv_distance=tv_distance,
-                codelets_z_stat=codelets_z,
-            )
-        )
-        for gold_row in gold_solutions.itertuples(index=False):
-            observed_match = observed_solutions[
-                observed_solutions["solution"] == gold_row.solution
-            ]
-            if observed_match.empty:
-                observed_row = None
-                temperature_z = None
-            else:
-                observed_row = observed_match.iloc[0]
-                temperature_z = z_statistic(
-                    observed_mean=observed_row["mean_temperature"],
-                    expected_mean=gold_row.mean_temperature,
-                    observed_standard_error=observed_row["temperature_standard_error"],
-                    expected_standard_error=(gold_row.temperature_standard_error),
-                )
-            comparison_rows.append(
-                _comparison_row(
-                    problem=problem,
-                    solution=gold_row.solution,
-                    observed=observed_row,
-                    expected=gold_row._asdict(),
-                    temperature_z_stat=temperature_z,
-                )
-            )
-    comparison = pd.DataFrame(
-        comparison_rows,
-        columns=[
-            "problem",
-            "solution",
-            "observed_frequency",
-            "expected_frequency",
-            "frequency_difference",
-            "observed_mean_codelets_run",
-            "expected_mean_codelets_run",
-            "mean_codelets_run_difference",
-            "codelets_relative_error_magnitude",
-            "observed_codelets_standard_error",
-            "expected_codelets_standard_error",
-            "codelets_standard_error_difference",
-            "observed_mean_temperature",
-            "expected_mean_temperature",
-            "mean_temperature_difference",
-            "observed_temperature_standard_error",
-            "expected_temperature_standard_error",
-            "temperature_standard_error_difference",
-            "tv_distance",
-            "codelets_z_stat",
-            "temperature_z_stat",
-        ],
-    )
-    summary = _reproduction_summary(comparison)
-    if verbose:
-        print(
-            comparison.to_string(
-                index=False,
-                na_rep="-",
-                formatters={
-                    "tv_distance": lambda x: f"{x:.4f}",
-                    "codelets_z_stat": lambda x: f"{x:.2f}",
-                    "codelets_relative_error_magnitude": lambda x: f"{x:.1%}",
-                    "temperature_z_stat": lambda x: f"{x:.2f}",
-                },
-            )
-        )
-        for key, value in summary.items():
-            print(f"{key}: {value}")
-    if output_file:
-        comparison.to_csv(output_file, index=False)
-    return comparison, summary
-
-
-def _comparison_row(
-    *,
-    problem: str,
-    solution: str,
-    observed: pd.Series | dict[str, object] | None,
-    expected: pd.Series | dict[str, object] | None,
-    tv_distance: float | None = None,
-    codelets_z_stat: float | None = None,
-    temperature_z_stat: float | None = None,
-) -> dict[str, str | float | int | None]:
-    """Build one observed-versus-expected comparison row."""
-    row: dict[str, str | float | int | None] = {
-        "problem": problem,
-        "solution": solution,
-        "tv_distance": tv_distance,
-        "codelets_z_stat": codelets_z_stat,
-        "temperature_z_stat": temperature_z_stat,
-    }
-    for field in (
-        "frequency",
-        "mean_codelets_run",
-        "codelets_standard_error",
-        "mean_temperature",
-        "temperature_standard_error",
-    ):
-        observed_value = _row_value(observed, field)
-        expected_value = _row_value(expected, field)
-        row[f"observed_{field}"] = observed_value
-        row[f"expected_{field}"] = expected_value
-        row[f"{field}_difference"] = _difference(
-            observed_value,
-            expected_value,
-        )
-    row["codelets_relative_error_magnitude"] = _relative_error_magnitude(
-        row["observed_mean_codelets_run"],
-        row["expected_mean_codelets_run"],
-    )
-    return row
-
-
-def _row_value(
-    row: pd.Series | dict[str, object] | None,
-    field: str,
-) -> float | int | None:
-    if row is None:
-        return 0 if field == "frequency" else None
-    value = row.get(field)
-    return None if pd.isna(value) else value
-
-
-def _difference(
-    observed: float | int | None,
-    expected: float | int | None,
-) -> float | int | None:
-    if observed is None or expected is None:
-        return None
-    return observed - expected
-
-
-def _relative_error_magnitude(
-    observed: float | int | None,
-    expected: float | int | None,
-) -> float | None:
-    """Return the magnitude of error relative to the expected value."""
-    if observed is None or expected is None or expected == 0:
-        return None
-    return abs(observed - expected) / expected
-
-
-def _gold_behaviour_dataframe(
-    gold_behaviour: dict[str, pd.DataFrame | list[dict]],
-) -> pd.DataFrame:
-    """Combine per-problem gold summaries into one DataFrame."""
-    frames: list[pd.DataFrame] = []
-    for problem, behaviour in gold_behaviour.items():
-        frame = (
-            behaviour.copy()
-            if isinstance(behaviour, pd.DataFrame)
-            else pd.DataFrame(behaviour)
-        )
-        frame.insert(0, "problem", problem)
-        frames.append(frame)
-    if not frames:
-        return pd.DataFrame(columns=SUMMARY_COLUMNS)
-    return pd.concat(frames, ignore_index=True)
-
-
-def _total_row(
-    results: pd.DataFrame,
-    problem: str,
-    description: str,
-) -> pd.Series:
-    """Return the single totals row for a problem."""
-    total = results[results["solution"] == "Total"]
-    if len(total) != 1:
-        raise ValueError(
-            f"Expected exactly one Total row for {description} behaviour "
-            f"of {problem!r}, found {len(total)}."
-        )
-    return total.iloc[0]
-
-
-def _reproduction_summary(comparison: pd.DataFrame) -> dict[str, object]:
-    """Calculate aggregate reproduction statistics from a comparison table."""
-    tv_distances = comparison["tv_distance"].dropna().tolist()
-    codelets_z_scores = comparison["codelets_z_stat"].dropna().tolist()
-    codelets_relative_error_magnitudes = comparison[
-        "codelets_relative_error_magnitude"
-    ].dropna().tolist()
-    temperature_z_scores = comparison.loc[
-        comparison["expected_frequency"] >= 10,
-        "temperature_z_stat",
-    ].dropna().tolist()
-    temperature_absolute_error_magnitudes = comparison.loc[
-        comparison["expected_frequency"] >= 10,
-        "mean_temperature_difference",
-    ].dropna().abs().tolist()
-    return {
-        "mean_total_variation_distance": _mean(tv_distances),
-        "max_total_variation_distance": max(tv_distances, default=0.0),
-        "mean_codelets_relative_error_magnitude": _mean(
-            codelets_relative_error_magnitudes,
-        ),
-        "max_codelets_relative_error_magnitude": max(
-            codelets_relative_error_magnitudes,
-            default=0.0,
-        ),
-        "codelets_relative_error_magnitude_quantiles": _deciles(
-            codelets_relative_error_magnitudes,
-        ),
-        "mean_temperature_absolute_error_magnitude": _mean(
-            temperature_absolute_error_magnitudes,
-        ),
-        "max_temperature_absolute_error_magnitude": max(
-            temperature_absolute_error_magnitudes,
-            default=0.0,
-        ),
-        "temperature_absolute_error_magnitude_quantiles": _deciles(
-            temperature_absolute_error_magnitudes,
-        ),
-        **_z_score_summary("codelets", codelets_z_scores),
-        **_z_score_summary("temperature", temperature_z_scores),
-    }
-
-
-def _z_score_summary(prefix: str, z_scores: list[float]) -> dict[str, float]:
-    """Return chi-square and descriptive statistics for independent z scores."""
-    statistic = sum(z_score**2 for z_score in z_scores)
-    return {
-        f"{prefix}_chi_square_statistic": statistic,
-        f"{prefix}_chi_square_p_value": chi_square_survival_function(
-            statistic,
-            degrees_of_freedom=len(z_scores),
-        ),
-        f"{prefix}_rms_z_score": sqrt(statistic / len(z_scores)) if z_scores else 0.0,
-        f"{prefix}_max_absolute_z_score": max(
-            (abs(z_score) for z_score in z_scores),
-            default=0.0,
-        ),
-    }
-
-
-def _mean(values: list[float]) -> float:
-    return sum(values) / len(values) if values else 0.0
-
-
-def _deciles(values: list[float]) -> dict[int, float]:
-    """Return the 10th through 100th percentiles, keyed by percentile."""
-    if not values:
-        return {percentile: 0.0 for percentile in range(10, 101, 10)}
-    ordered = sorted(values)
-    result = {}
-    for percentile in range(10, 101, 10):
-        position = (len(ordered) - 1) * percentile / 100
-        lower = int(position)
-        upper = min(lower + 1, len(ordered) - 1)
-        fraction = position - lower
-        result[percentile] = ordered[lower] + fraction * (
-            ordered[upper] - ordered[lower]
-        )
-    return result
-
-
-def chi_square_survival_function(
-    statistic: float,
-    degrees_of_freedom: int,
-) -> float:
-    """Return ``P(X >= statistic)`` for a chi-square random variable.
-
-    This is the regularized upper incomplete gamma function evaluated at
-    ``statistic / 2`` and avoids making SciPy a runtime dependency.
-    """
-    if degrees_of_freedom < 1:
-        return 1.0
-    if statistic <= 0:
-        return 1.0
-    if not isfinite(statistic):
-        return 0.0 if statistic == inf else float("nan")
-    return _regularized_gamma_q(degrees_of_freedom / 2, statistic / 2)
-
-
-def _regularized_gamma_q(shape: float, value: float) -> float:
-    """Evaluate the regularized upper incomplete gamma function."""
-    epsilon = 1e-14
-    minimum = 1e-300
-    if value < shape + 1:
-        term = 1 / shape
-        total = term
-        for index in range(1, 10_000):
-            term *= value / (shape + index)
-            total += term
-            if abs(term) <= abs(total) * epsilon:
-                break
-        return 1 - total * exp(-value + shape * log(value) - lgamma(shape))
-
-    denominator = value + 1 - shape
-    continued_fraction = 1 / minimum
-    reciprocal = 1 / denominator
-    result = reciprocal
-    for index in range(1, 10_000):
-        numerator = -index * (index - shape)
-        denominator += 2
-        reciprocal = numerator * reciprocal + denominator
-        if abs(reciprocal) < minimum:
-            reciprocal = minimum
-        continued_fraction = denominator + numerator / continued_fraction
-        if abs(continued_fraction) < minimum:
-            continued_fraction = minimum
-        reciprocal = 1 / reciprocal
-        delta = reciprocal * continued_fraction
-        result *= delta
-        if abs(delta - 1) <= epsilon:
-            break
-    return exp(-value + shape * log(value) - lgamma(shape)) * result
-
-
-def total_variation_distance(
-    observed: dict[str, int],
-    expected: dict[str, int],
-) -> float:
-    """Return total variation distance between two discrete distributions."""
-    observed_total = sum(observed.values())
-    expected_total = sum(expected.values())
-    if observed_total == 0 or expected_total == 0:
-        raise ValueError("Distributions must contain at least one observation.")
-    answers = observed.keys() | expected.keys()
-    return 0.5 * sum(
-        abs(
-            observed.get(answer, 0) / observed_total
-            - expected.get(answer, 0) / expected_total
-        )
-        for answer in answers
-    )
-
-
-def z_statistic(
-    observed_mean: float,
-    expected_mean: float,
-    observed_standard_error: float,
-    expected_standard_error: float,
-) -> float:
-    """Return the z statistic for two means with independent errors."""
-    standard_error = sqrt(observed_standard_error**2 + expected_standard_error**2)
-    difference = observed_mean - expected_mean
-    if standard_error == 0:
-        if difference == 0:
-            return 0.0
-        return float("inf") if difference > 0 else float("-inf")
-    return difference / standard_error
-
-
-def two_sided_p_value(z_score: float) -> float:
-    """Return the two-sided normal-distribution p value for a z statistic."""
-    return erfc(abs(z_score) / sqrt(2))
-
-
-def _terminal_rule(character: str) -> str:
-    """Return a full-width terminal separator."""
-    return character * get_terminal_size(fallback=(80, 24)).columns
-
-
-class RunningAnswerDisplay:
-    def __init__(self):
-        self.lines_printed = 0
-
-    def update(self, results: pd.DataFrame) -> None:
-        table = (
-            results.drop(columns="problem")
-            .rename(
-                columns={
-                    "solution": "Solution",
-                    "frequency": "Runs",
-                    "mean_codelets_run": "Mean codelets",
-                    "codelets_standard_error": "Codelets SE",
-                    "mean_temperature": "Mean temp",
-                    "temperature_standard_error": "Temp SE",
-                }
-            )
-            .to_string(
-                index=False,
-                formatters={
-                    "Mean codelets": "{:.0f}".format,
-                    "Codelets SE": "{:.1f}".format,
-                    "Mean temp": "{:.2f}".format,
-                    "Temp SE": "{:.4f}".format,
-                },
-            )
-        )
-        if self.lines_printed:
-            print(
-                f"\033[{self.lines_printed}A",
-                end="",
-            )
-            for _ in range(self.lines_printed):
-                print("\033[2K")
-            print(
-                f"\033[{self.lines_printed}A",
-                end="",
-            )
-        lines = table.splitlines()
-        for line in lines:
-            print(f"\033[2K{line}")
-        terminal_width = get_terminal_size(fallback=(80, 24)).columns
-        self.lines_printed = sum(
-            max(
-                1,
-                (len(line) + terminal_width - 1) // terminal_width,
-            )
-            for line in lines
-        )
-
-
-def summarize_runs(
-    runs: list[dict[str, str | float | int | None]],
-    problem: str,
-) -> pd.DataFrame:
-    """Summarize individual runs, including an overall totals row."""
-    data = pd.DataFrame(runs)
-    if data.empty:
-        return pd.DataFrame(columns=SUMMARY_COLUMNS)
-    grouped = data.groupby(
-        "solution",
-        sort=False,
-    )
-    summary = grouped.agg(
-        frequency=("solution", "size"),
-        mean_codelets_run=("codelets_run", "mean"),
-        codelets_standard_error=("codelets_run", "sem"),
-        mean_temperature=("temperature", "mean"),
-        temperature_standard_error=("temperature", "sem"),
-    ).reset_index()
-    summary = summary.sort_values(
-        "frequency",
-        ascending=False,
-        kind="stable",
-    )
-    error_columns = [
-        "codelets_standard_error",
-        "temperature_standard_error",
-    ]
-    summary[error_columns] = summary[error_columns].fillna(0.0)
-    totals = pd.DataFrame(
-        [
-            {
-                "solution": "Total",
-                "frequency": len(data),
-                "mean_codelets_run": (data["codelets_run"].mean()),
-                "codelets_standard_error": (data["codelets_run"].sem()),
-                "mean_temperature": (data["temperature"].mean()),
-                "temperature_standard_error": (data["temperature"].sem()),
-            }
-        ]
-    )
-    totals[error_columns] = totals[error_columns].fillna(0.0)
-    summary = pd.concat(
-        [summary, totals],
-        ignore_index=True,
-    )
-    summary.insert(
-        0,
-        "problem",
-        problem,
-    )
-    return summary[SUMMARY_COLUMNS]