aria-code 0.114.0__tar.gz → 0.115.0__tar.gz

{aria_code-0.114.0/src/aria_code.egg-info → aria_code-0.115.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aria-code
-Version: 0.114.0
+Version: 0.115.0
 Summary: Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/artheras/aria-code
@@ -171,7 +171,7 @@ Numbers you can check, client data kept apart, actions a person approves.</p>
 <p align="center">
   <a href="https://www.npmjs.com/package/@artheras/aria-code"><img src="https://img.shields.io/npm/v/@artheras/aria-code/latest?style=flat-square&logo=npm&label=npm" alt="npm version"></a>
   <!-- Release automation pins PyPI: the default endpoint still selects legacy 4.x. -->
-  <a href="https://pypi.org/project/aria-code/0.114.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.114.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
+  <a href="https://pypi.org/project/aria-code/0.115.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.115.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
   <a href="https://github.com/artheras/aria-code/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/artheras/aria-code/ci.yml?branch=main&style=flat-square&logo=githubactions&label=CI" alt="CI status"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-64748b?style=flat-square" alt="Apache License 2.0"></a>
 </p>

{aria_code-0.114.0 → aria_code-0.115.0}/README.md

@@ -15,7 +15,7 @@ Numbers you can check, client data kept apart, actions a person approves.</p>
 <p align="center">
   <a href="https://www.npmjs.com/package/@artheras/aria-code"><img src="https://img.shields.io/npm/v/@artheras/aria-code/latest?style=flat-square&logo=npm&label=npm" alt="npm version"></a>
   <!-- Release automation pins PyPI: the default endpoint still selects legacy 4.x. -->
-  <a href="https://pypi.org/project/aria-code/0.114.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.114.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
+  <a href="https://pypi.org/project/aria-code/0.115.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.115.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
   <a href="https://github.com/artheras/aria-code/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/artheras/aria-code/ci.yml?branch=main&style=flat-square&logo=githubactions&label=CI" alt="CI status"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-64748b?style=flat-square" alt="Apache License 2.0"></a>
 </p>

{aria_code-0.114.0 → aria_code-0.115.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aria-code"
-version = "0.114.0"
+version = "0.115.0"
 description = "Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{aria_code-0.114.0 → aria_code-0.115.0}/src/aria_code/_version.py

@@ -17,4 +17,4 @@ from __future__ import annotations
 
 __all__ = ["__version__"]
 
-__version__ = "0.114.0"
+__version__ = "0.115.0"

aria_code-0.115.0/src/aria_code/evals/gap.py

@@ -0,0 +1,93 @@
+"""In-sample minus out-of-sample: public tasks against their private twins.
+
+A public task can end up in a model's training data, and a tuned model can
+learn the public bank instead of the skill. Either way it scores well on the
+public task and worse on a twin it has never seen: the same skill, with
+different data, numbers and wording, kept in a private repository. SWE-bench
+Pro reads leakage the same way, from the gap between its public and held-out
+splits; a quant reads overfitting from the gap between in-sample and
+out-of-sample returns.
+
+A holdout suite names each twin's public task with ``twin_of``. This pairs
+them, takes each side's pass rate over its repeats, and reports the mean gap
+(public minus twin) with a bootstrap 95% interval. An interval above zero is
+a warning that the public score overstates the skill.
+
+    python -m aria_code.evals.gap --public evals-*.json --holdout holdout.json --suites holdout/suites/*.yaml
+
+``--markdown`` prints counts and rates only, never a private task id, so the
+section can go into a public job summary.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Iterable
+
+from aria_code.training.compare import bootstrap_ci, rates
+
+
+def twins(suite_files: Iterable[Path]) -> dict[str, str]:
+    """Holdout task id -> the public task it is a twin of."""
+    import yaml
+
+    pairs = {}
+    for suite_file in suite_files:
+        doc = yaml.safe_load(Path(suite_file).read_text(encoding="utf-8")) or {}
+        pairs.update({str(t["id"]): str(t["twin_of"]) for t in doc.get("tasks") or [] if t.get("twin_of")})
+    return pairs
+
+
+def gap(public_reports: Iterable[Path], holdout_reports: Iterable[Path], pairs: dict[str, str]) -> dict:
+    public, _, _ = rates(public_reports)
+    holdout, _, _ = rates(holdout_reports)
+    rows = {h: (public[p], holdout[h]) for h, p in pairs.items() if p in public and h in holdout}
+    diffs = [pub - twin for pub, twin in rows.values()]
+    lo, hi = bootstrap_ci(diffs)
+    mean = sum(diffs) / len(diffs) if diffs else 0.0
+    return {
+        "pairs": len(rows),
+        "public_pass_rate": round(sum(p for p, _ in rows.values()) / len(rows), 4) if rows else None,
+        "twin_pass_rate": round(sum(t for _, t in rows.values()) / len(rows), 4) if rows else None,
+        "mean_gap": round(mean, 4),
+        "ci95": [round(lo, 4), round(hi, 4)],
+        "public_overstates": lo > 0,
+        "unpaired": sorted(h for h in pairs if h not in rows),
+    }
+
+
+def markdown(result: dict) -> str:
+    """The gap for a public summary: rates and the interval, no task ids."""
+    if not result["pairs"]:
+        return "### Public vs holdout\nNo twin pairs were scored on both sides.\n"
+    lo, hi = result["ci95"]
+    verdict = ("The public score overstates the skill: the interval is above 0." if result["public_overstates"]
+               else "No significant gap: the interval includes 0 or is below it.")
+    return "\n".join([
+        "### Public vs holdout",
+        f"{result['pairs']} twin pairs. Public {result['public_pass_rate']:.0%}, "
+        f"holdout {result['twin_pass_rate']:.0%}; gap {result['mean_gap']:+.0%} "
+        f"(95% CI {lo:+.0%} to {hi:+.0%}).",
+        verdict,
+        *([f"{len(result['unpaired'])} twins had no scored pair."] if result["unpaired"] else []),
+        "",
+    ])
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="Gap between public tasks and their private twins.")
+    ap.add_argument("--public", nargs="+", type=Path, required=True)
+    ap.add_argument("--holdout", nargs="+", type=Path, required=True)
+    ap.add_argument("--suites", nargs="+", type=Path, required=True,
+                    help="the holdout suite YAML files (their tasks carry twin_of)")
+    ap.add_argument("--markdown", action="store_true", help="counts and rates only, safe for a public summary")
+    args = ap.parse_args(argv)
+    result = gap(args.public, args.holdout, twins(args.suites))
+    print(markdown(result) if args.markdown else json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{aria_code-0.114.0 → aria_code-0.115.0}/src/aria_code/evals/summary.py

@@ -1,6 +1,11 @@
 """Turn eval reports into a Markdown scoreboard, and decide whether the run is healthy.
 
-    python3 -m aria_code.evals.summary evals-core.json evals-operations.json
+    python3 -m aria_code.evals.summary evals-core.json evals-operations.json [--private evals-holdout.json]
+
+Each suite's score comes with the mean per-task pass rate and its standard
+error, so two runs can be told apart from noise. A report passed after
+``--private`` (the holdout suites, kept in a private repository) is shown as
+counts only: no task ids, details or logs, because the summary is public.
 
 The suites ran in CI only with ``--check``: every fixture still starts red,
 but no model ever attempted them, so nothing measured whether Aria can fix
@@ -15,22 +20,50 @@ A task the model gets wrong is a result, not a broken build: the exit code is
 from __future__ import annotations
 
 import json
+import math
 import sys
+from collections import defaultdict
 from pathlib import Path
 
 
-def summarise(reports: list[dict], model: str = "") -> tuple[str, int]:
+def mean_sem(report: dict) -> tuple[float, float, int] | None:
+    """Mean per-task pass rate over scored attempts, its standard error, and the task count."""
+    passed: dict[str, int] = defaultdict(int)
+    scored: dict[str, int] = defaultdict(int)
+    for r in report.get("results") or []:
+        if str(r.get("outcome", "")).lower() in ("pass", "fail"):
+            scored[str(r.get("task_id"))] += 1
+            passed[str(r.get("task_id"))] += str(r.get("outcome")).lower() == "pass"
+    rates = [passed[t] / scored[t] for t in scored]
+    if not rates:
+        return None
+    mean = sum(rates) / len(rates)
+    if len(rates) < 2:
+        return mean, 0.0, 1
+    var = sum((x - mean) ** 2 for x in rates) / (len(rates) - 1)
+    return mean, math.sqrt(var / len(rates)), len(rates)
+
+
+def summarise(reports: list[dict], model: str = "", private: list[dict] | None = None) -> tuple[str, int]:
     """Markdown for the reports, and the exit code the run should have."""
     lines = [f"## Aria eval scoreboard{f' · {model}' if model else ''}", ""]
     unhealthy = 0
-    for report in reports:
+    for report in [*reports, *(private or [])]:
+        is_private = any(report is p for p in private or [])
         suite = report.get("suite", "?")
-        lines.append(f"### {suite}: {report.get('passed', 0)}/{report.get('scored', 0)} "
-                     f"({report.get('pass_rate', 0):.0%})")
+        lines.append(f"### {suite}{' (holdout)' if is_private else ''}: "
+                     f"{report.get('passed', 0)}/{report.get('scored', 0)} ({report.get('pass_rate', 0):.0%})")
+        stats = mean_sem(report)
+        if stats:
+            mean, sem, n = stats
+            lines.append(f"Mean task pass rate {mean:.0%} ± {sem:.0%} (SEM over {n} task{'s' if n != 1 else ''}).")
         extras = [f"{report[k]} {k}" for k in ("errored", "invalid") if report.get(k)]
         if extras:
             lines.append(f"Not scored: {', '.join(extras)}.")
         unhealthy += int(report.get("errored", 0)) + int(report.get("invalid", 0))
+        if is_private:
+            lines += ["Task ids, details and logs are not published for holdout suites.", ""]
+            continue
         lines += ["", "| Task | Outcome | Seconds | Detail |", "|---|---|---:|---|"]
         for result in report.get("results", []):
             detail = str(result.get("detail") or "").replace("|", "\\|")[:120]
@@ -56,17 +89,26 @@ def main(argv: list[str] | None = None) -> int:
     model = ""
     if args[:1] == ["--model"] and len(args) > 1:
         model, args = args[1], args[2:]
-    reports = []
-    for name in args:
-        path = Path(name)
-        if path.exists():
-            reports.append(json.loads(path.read_text(encoding="utf-8")))
-        else:
-            print(f"(no report at {name})", file=sys.stderr)
-    if not reports:
+    public_names, private_names = args, []
+    if "--private" in args:
+        i = args.index("--private")
+        public_names, private_names = args[:i], args[i + 1:]
+
+    def load(names: list[str]) -> list[dict]:
+        out = []
+        for name in names:
+            path = Path(name)
+            if path.exists():
+                out.append(json.loads(path.read_text(encoding="utf-8")))
+            else:
+                print(f"(no report at {name})", file=sys.stderr)
+        return out
+
+    reports, private = load(public_names), load(private_names)
+    if not reports and not private:
         print("no eval reports to summarise", file=sys.stderr)
         return 1
-    text, code = summarise(reports, model)
+    text, code = summarise(reports, model, private)
     print(text)
     return code
 

{aria_code-0.114.0 → aria_code-0.115.0/src/aria_code.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aria-code
-Version: 0.114.0
+Version: 0.115.0
 Summary: Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/artheras/aria-code
@@ -171,7 +171,7 @@ Numbers you can check, client data kept apart, actions a person approves.</p>
 <p align="center">
   <a href="https://www.npmjs.com/package/@artheras/aria-code"><img src="https://img.shields.io/npm/v/@artheras/aria-code/latest?style=flat-square&logo=npm&label=npm" alt="npm version"></a>
   <!-- Release automation pins PyPI: the default endpoint still selects legacy 4.x. -->
-  <a href="https://pypi.org/project/aria-code/0.114.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.114.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
+  <a href="https://pypi.org/project/aria-code/0.115.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.115.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
   <a href="https://github.com/artheras/aria-code/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/artheras/aria-code/ci.yml?branch=main&style=flat-square&logo=githubactions&label=CI" alt="CI status"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-64748b?style=flat-square" alt="Apache License 2.0"></a>
 </p>

{aria_code-0.114.0 → aria_code-0.115.0}/src/aria_code.egg-info/SOURCES.txt

@@ -337,6 +337,7 @@ src/aria_code/domain/finance_formulas.py
 src/aria_code/domain/portfolio_ledger.py
 src/aria_code/domain/strategy_vault.py
 src/aria_code/evals/__init__.py
+src/aria_code/evals/gap.py
 src/aria_code/evals/harness.py
 src/aria_code/evals/runner.py
 src/aria_code/evals/summary.py
@@ -594,6 +595,7 @@ tests/test_english_output_group.py
 tests/test_enterprise_tool_honesty.py
 tests/test_env_switch_docs.py
 tests/test_eval_hard_suite.py
+tests/test_eval_holdout.py
 tests/test_eval_summary.py
 tests/test_eval_task_bank.py
 tests/test_eval_trajectories.py

aria_code-0.115.0/tests/test_eval_holdout.py

@@ -0,0 +1,102 @@
+"""Private holdout suites: scored beside the public ones, never published.
+
+Twins in a private repository (the same skill as a public task, different
+data) show whether the public score is inflated by leakage or by tuning on
+the public bank. The run pages of this repository are public, so a holdout
+suite may show up in them only as counts.
+"""
+
+from __future__ import annotations
+
+import json
+
+import yaml
+
+from aria_code.evals.gap import gap, markdown, twins
+from aria_code.evals.summary import main, mean_sem, summarise
+
+
+def report(suite: str, outcomes: dict[str, list[str]]) -> dict:
+    results = [{"task_id": t, "outcome": o, "seconds": 10, "detail": f"secret detail of {t}",
+                "log_tail": [f"secret log of {t}"]} for t, os_ in outcomes.items() for o in os_]
+    passed = sum(r["outcome"] == "pass" for r in results)
+    return {"suite": suite, "passed": passed, "scored": len(results), "pass_rate": passed / len(results),
+            "errored": 0, "invalid": 0, "results": results}
+
+
+def write(tmp_path, name, data):
+    path = tmp_path / name
+    path.write_text(json.dumps(data) if not name.endswith(".yaml") else yaml.safe_dump(data))
+    return path
+
+
+def test_mean_and_standard_error_over_task_pass_rates():
+    mean, sem, n = mean_sem(report("core", {"a": ["pass", "pass"], "b": ["pass", "fail"], "c": ["fail", "fail"]}))
+    assert n == 3 and mean == 0.5
+    assert abs(sem - 0.5 / 3 ** 0.5) < 1e-9          # sd of (1, .5, 0) is .5
+    assert mean_sem({"results": [{"task_id": "a", "outcome": "error"}]}) is None
+
+
+def test_the_scoreboard_carries_the_standard_error():
+    text, _ = summarise([report("core", {"a": ["pass"], "b": ["fail"]})])
+    assert "Mean task pass rate 50% ± 50% (SEM over 2 tasks)." in text
+
+
+def test_a_private_report_shows_counts_and_nothing_else():
+    private = report("holdout-ops", {"twin-secret-task": ["pass", "fail"]})
+    text, code = summarise([report("core", {"a": ["pass"]})], "m", [private])
+    assert code == 0
+    assert "### holdout-ops (holdout): 1/2 (50%)" in text
+    for leak in ("twin-secret-task", "detail of twin", "log of twin"):
+        assert leak not in text
+    assert "| a | pass |" in text                    # the public suite keeps its table
+
+
+def test_the_cli_takes_private_reports_after_a_flag(tmp_path, capsys):
+    pub = write(tmp_path, "evals-core.json", report("core", {"a": ["pass"]}))
+    priv = write(tmp_path, "holdout.json", report("holdout", {"twin-x": ["fail"]}))
+    assert main(["--model", "m", str(pub), "--private", str(priv)]) == 0
+    out = capsys.readouterr().out
+    assert "### holdout (holdout): 0/1" in out and "twin-x" not in out
+
+
+def test_twins_pair_holdout_tasks_with_their_public_tasks(tmp_path):
+    suite = write(tmp_path, "holdout.yaml", {"suite": "holdout", "tasks": [
+        {"id": "h1", "twin_of": "a"}, {"id": "h2", "twin_of": "b"}, {"id": "h3"}]})
+    assert twins([suite]) == {"h1": "a", "h2": "b"}
+
+
+def test_a_public_score_that_does_not_carry_over_is_flagged(tmp_path):
+    public = [write(tmp_path, "pub.json", report("core", {f"p{i}": ["pass"] * 3 for i in range(8)}))]
+    holdout = [write(tmp_path, "hold.json", report("holdout", {f"h{i}": ["fail"] * 3 for i in range(8)}))]
+    result = gap(public, holdout, {f"h{i}": f"p{i}" for i in range(8)})
+    assert result["pairs"] == 8 and result["mean_gap"] == 1.0 and result["public_overstates"]
+    text = markdown(result)
+    assert "gap +100%" in text and "overstates" in text
+    assert "h0" not in text and "p0" not in text
+
+
+def test_equal_scores_are_not_called_a_gap(tmp_path):
+    outcomes = {"x": ["pass", "fail"], "y": ["pass", "pass"], "z": ["fail", "fail"]}
+    public = [write(tmp_path, "pub.json", report("core", outcomes))]
+    holdout = [write(tmp_path, "hold.json", report("holdout", {f"h-{t}": o for t, o in outcomes.items()}))]
+    result = gap(public, holdout, {f"h-{t}": t for t in outcomes} | {"h-missing": "nope"})
+    assert result["mean_gap"] == 0 and not result["public_overstates"]
+    assert result["unpaired"] == ["h-missing"]
+    assert "No significant gap" in markdown(result) and "1 twins had no scored pair." in markdown(result)
+
+
+def test_the_workflow_keeps_the_holdout_private():
+    text = open(".github/workflows/evals.yml").read()
+    job = yaml.safe_load(text)["jobs"]["evals"]
+    steps = {s.get("name", s.get("uses")): s for s in job["steps"]}
+    checkout = steps["Check out the holdout suites"]
+    assert "env.HOLDOUT == 'true'" in checkout["if"]
+    assert checkout["with"]["repository"] == "artheras/aria-evals-holdout"
+    assert checkout["with"]["ssh-key"] == "${{ secrets.HOLDOUT_DEPLOY_KEY }}"
+    assert checkout["with"]["persist-credentials"] is False
+    run = steps["Run the holdout suites"]["run"]
+    assert "--trajectories" not in run and "> /dev/null 2>&1" in run   # no task ids in the public log
+    assert "--private" in steps["Scoreboard"]["run"] and "--markdown" in steps["Scoreboard"]["run"]
+    upload = next(s for s in job["steps"] if str(s.get("uses", "")).startswith("actions/upload-artifact"))
+    assert ".holdout" not in upload["with"]["path"]