aria-code 0.125.0__tar.gz → 0.127.0__tar.gz

{aria_code-0.125.0/src/aria_code.egg-info → aria_code-0.127.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aria-code
-Version: 0.125.0
+Version: 0.127.0
 Summary: Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/artheras/aria-code
@@ -172,7 +172,7 @@ Numbers you can check, client data kept apart, actions a person approves.</p>
 <p align="center">
   <a href="https://www.npmjs.com/package/@artheras/aria-code"><img src="https://img.shields.io/npm/v/@artheras/aria-code/latest?style=flat-square&logo=npm&label=npm" alt="npm version"></a>
   <!-- Release automation pins PyPI: the default endpoint still selects legacy 4.x. -->
-  <a href="https://pypi.org/project/aria-code/0.125.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.125.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
+  <a href="https://pypi.org/project/aria-code/0.127.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.127.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
   <a href="https://github.com/artheras/aria-code/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/artheras/aria-code/ci.yml?branch=main&style=flat-square&logo=githubactions&label=CI" alt="CI status"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-64748b?style=flat-square" alt="Apache License 2.0"></a>
 </p>
@@ -281,6 +281,7 @@ Install extras only for features you use; see [pyproject.toml](pyproject.toml) f
 | Topic | Start here |
 | --- | --- |
 | Architecture and runtime | [Architecture](docs/architecture.md) |
+| Opt-in Rust CLI prototype and migration stages | [Rust CLI migration](docs/rust-cli-migration.md) |
 | Tool permissions and data boundaries | [Safety notes](docs/aria-code-safety.md) |
 | Warehouse agents and their read-only data contract | [Warehouse ERP agents](docs/warehouse-erp-agents.md) |
 | Strategy and backtesting workflows | [Strategy workspace](docs/strategy_workspace.md) |

{aria_code-0.125.0 → aria_code-0.127.0}/README.md

@@ -15,7 +15,7 @@ Numbers you can check, client data kept apart, actions a person approves.</p>
 <p align="center">
   <a href="https://www.npmjs.com/package/@artheras/aria-code"><img src="https://img.shields.io/npm/v/@artheras/aria-code/latest?style=flat-square&logo=npm&label=npm" alt="npm version"></a>
   <!-- Release automation pins PyPI: the default endpoint still selects legacy 4.x. -->
-  <a href="https://pypi.org/project/aria-code/0.125.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.125.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
+  <a href="https://pypi.org/project/aria-code/0.127.0/"><img src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpypi.org%2Fpypi%2Faria-code%2F0.127.0%2Fjson&amp;query=%24.info.version&amp;prefix=v&amp;style=flat-square&amp;logo=pypi&amp;label=PyPI&amp;color=blue&amp;cacheSeconds=300" alt="PyPI version"></a>
   <a href="https://github.com/artheras/aria-code/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/artheras/aria-code/ci.yml?branch=main&style=flat-square&logo=githubactions&label=CI" alt="CI status"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-64748b?style=flat-square" alt="Apache License 2.0"></a>
 </p>
@@ -124,6 +124,7 @@ Install extras only for features you use; see [pyproject.toml](pyproject.toml) f
 | Topic | Start here |
 | --- | --- |
 | Architecture and runtime | [Architecture](docs/architecture.md) |
+| Opt-in Rust CLI prototype and migration stages | [Rust CLI migration](docs/rust-cli-migration.md) |
 | Tool permissions and data boundaries | [Safety notes](docs/aria-code-safety.md) |
 | Warehouse agents and their read-only data contract | [Warehouse ERP agents](docs/warehouse-erp-agents.md) |
 | Strategy and backtesting workflows | [Strategy workspace](docs/strategy_workspace.md) |

{aria_code-0.125.0 → aria_code-0.127.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aria-code"
-version = "0.125.0"
+version = "0.127.0"
 description = "Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/__init__.py

@@ -66,12 +66,20 @@ class _OneModulePerFile(_importlib_abc.MetaPathFinder, _importlib_abc.Loader):
     def __init__(self, root: str) -> None:
         self._own: dict = {}
         self.root = _os.path.normcase(root) + _os.sep
-        self.names = {
-            entry[:-3] if entry.endswith(".py") else entry
-            for entry in _os.listdir(root)
-            if entry != "__init__.py" and not entry.startswith(("_", "."))
-            and (entry.endswith(".py") or _os.path.isfile(_os.path.join(root, entry, "__init__.py")))
-        }
+        if not getattr(_sys, "frozen", False) and _os.path.isdir(root):
+            self.names = {
+                entry[:-3] if entry.endswith(".py") else entry
+                for entry in _os.listdir(root)
+                if entry != "__init__.py" and not entry.startswith(("_", "."))
+                and (entry.endswith(".py") or _os.path.isfile(_os.path.join(root, entry, "__init__.py")))
+            }
+        else:
+            # Frozen Python modules live in PYZ, not a physical source tree.
+            # PyInstaller's pkgutil hook enumerates that archive even when the
+            # package directory is absent, or contains only bundled data files.
+            from pkgutil import iter_modules
+            self.names = {module.name for module in iter_modules([root])
+                          if not module.name.startswith(("_", "."))}
 
     def find_spec(self, fullname, path=None, target=None):
         if fullname.partition(".")[0] not in self.names:

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/_version.py

@@ -17,4 +17,4 @@ from __future__ import annotations
 
 __all__ = ["__version__"]
 
-__version__ = "0.125.0"
+__version__ = "0.127.0"

aria_code-0.127.0/src/aria_code/apps/cli/native_bridge.py

@@ -0,0 +1,118 @@
+"""Experimental, workspace-confined JSON-RPC worker for the Rust CLI.
+
+The host selects the workspace and approves one named write via process flags.
+Request parameters never establish permissions. Existing CLI handlers retain
+their diff, change-store and verification behavior; stdout is protocol only.
+"""
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+MAX_MESSAGE = 1_048_576
+READ_TOOLS = frozenset({"read_file", "list_files", "search_code"})
+WRITE_TOOLS = frozenset({"write_file", "edit_file"})
+
+
+def error(code: int, message: str, request_id: Any = None) -> dict:
+    return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
+
+
+def execute(name: str, arguments: dict, workspace: Path, approved_tool: str | None) -> dict:
+    from aria_code.runtime.approval import ApprovalDecision
+    from aria_code.runtime.tool_executor import ToolExecutor
+    from aria_code.runtime.tool_policy import check_tool_policy
+
+    if name not in READ_TOOLS | WRITE_TOOLS:
+        return {"success": False, "error": f"Tool not exposed by native bridge: {name}"}
+    verdict = check_tool_policy(name)
+    if verdict == "deny":
+        return {"success": False, "error": "Blocked by persistent tool policy"}
+    approved = name in WRITE_TOOLS and name == approved_tool
+    if verdict == "ask" and not approved:
+        return {"success": False, "error": "Tool requires explicit approval; use the interactive Python CLI"}
+    if name in WRITE_TOOLS and not approved:
+        return {"success": False, "error": "Writes require host --approve-write for this invocation"}
+    def handler(params: dict) -> dict:
+        # Load UI-dependent write code only after ToolExecutor grants the path.
+        if name in READ_TOOLS:
+            from aria_code.apps.cli.tools import file_tools
+            return getattr(file_tools, "tool_" + name)(params)
+        from aria_code.apps.cli.tools import write_tools
+        cli = write_tools._ac()
+        console = getattr(cli, "console", None)
+        previous = console.file if console is not None else None
+        try:
+            if console is not None:
+                console.file = sys.stderr
+            return getattr(write_tools, "tool_" + name)(params)
+        finally:
+            if console is not None:
+                console.file = previous
+    executor = ToolExecutor(
+        {name: (handler,)},
+        config={"workspace_root": str(workspace), "permission_mode": "workspace-write" if approved else "read-only"},
+        execution_context=lambda: {"_workspace_restricted": True},
+    )
+    decision = ApprovalDecision.allow(user_approved=True, tool_scope=name) if approved else None
+    return executor.execute_local(name, arguments, approval=decision)
+
+
+def dispatch(request: Any, workspace: Path, approved_tool: str | None = None) -> dict | None:
+    if not isinstance(request, dict):
+        return error(-32600, "Expected a single JSON-RPC request object")
+    request_id = request.get("id")
+    valid_id = request_id is None or isinstance(request_id, str) or type(request_id) is int
+    if request.get("jsonrpc") != "2.0" or not isinstance(request.get("method"), str) or not valid_id:
+        return error(-32600, "Invalid JSON-RPC request")
+    # This bridge only executes requests with an ID. Notifications never cause
+    # file changes and never produce a response, including unknown methods.
+    if "id" not in request:
+        return None
+    if request["method"] != "tools.call":
+        return error(-32601, "Method not found", request_id)
+    params = request.get("params")
+    if not isinstance(params, dict) or not isinstance(params.get("name"), str) or not isinstance(params.get("arguments"), dict):
+        return error(-32602, "Expected name and arguments object", request_id)
+    try:
+        with contextlib.redirect_stdout(sys.stderr):
+            result = execute(params["name"], params["arguments"], workspace, approved_tool)
+        return {"jsonrpc": "2.0", "id": request_id, "result": result}
+    except Exception as exc:
+        return error(-32603, f"Python tool failed: {exc}", request_id)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--workspace", type=Path, required=True)
+    parser.add_argument("--approve-tool", choices=sorted(WRITE_TOOLS))
+    args = parser.parse_args()
+    workspace = args.workspace.expanduser().resolve(strict=True)
+    if not workspace.is_dir():
+        parser.error("workspace must be a directory")
+    # One request per process bounds trace memory and approval lifetime. Read
+    # size is bounded before JSON decoding; no unbounded readline allocation.
+    raw = sys.stdin.buffer.readline(MAX_MESSAGE + 1)
+    if len(raw) > MAX_MESSAGE:
+        response = error(-32600, "Request exceeds 1 MiB")
+    else:
+        try:
+            request = json.loads(raw)
+        except (ValueError, UnicodeDecodeError):
+            response = error(-32700, "Parse error")
+        else:
+            response = dispatch(request, workspace, args.approve_tool)
+    if response is not None:
+        encoded = json.dumps(response, ensure_ascii=True, allow_nan=False).encode("ascii")
+        if len(encoded) + 1 > MAX_MESSAGE:
+            encoded = json.dumps(error(-32603, "Response exceeds 1 MiB", response["id"])).encode("ascii")
+        sys.stdout.buffer.write(encoded + b"\n")
+        sys.stdout.buffer.flush()
+
+
+if __name__ == "__main__":
+    main()

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/apps/cli/tools/write_tools.py

@@ -302,7 +302,7 @@ def _write_policy_confirm(p: pathlib.Path, content: str, existed: bool) -> tuple
     if has_rich and console:
         console.print()
         if existed:
-            old_content = p.read_text(errors="replace")
+            old_content = p.read_text(encoding="utf-8", errors="replace")
             diff = list(difflib.unified_diff(
                 old_content.splitlines(keepends=True),
                 content.splitlines(keepends=True),
@@ -723,7 +723,7 @@ def tool_edit_file(params: dict) -> dict:
         if not _is_safe(p, params):
             return {"success": False, "error": f"Access denied: path '{p}' is outside allowed directories"}
 
-        content = p.read_text(errors="replace")
+        content = p.read_text(encoding="utf-8", errors="replace")
         before_mode = _stat.S_IMODE(p.stat().st_mode)
         if content.count(old_str) == 0:
             preview = "\n".join(content.splitlines()[:10])
@@ -845,7 +845,7 @@ def tool_multi_edit(params: dict) -> dict:
         if not _is_safe(p, params):
             return {"success": False, "error": f"Access denied: path '{p}' is outside allowed directories"}
 
-        content = p.read_text(errors="replace")
+        content = p.read_text(encoding="utf-8", errors="replace")
         before_mode = _stat.S_IMODE(p.stat().st_mode)
         working = content
         applied_count = 0

aria_code-0.127.0/src/aria_code/evals/behavior.py

@@ -0,0 +1,113 @@
+"""Behaviour checks: how the agent went about a task, from its tool calls.
+
+A task's ``verify`` says whether the work is right. Some protocol features
+are about how it gets there — look up what depends on a function before
+changing it, edit a definition by name instead of rewriting the file — and a
+correct result says nothing about them. A task may list ``behavior`` checks;
+they are judged on the run's JSONL events (``aria-code -p … --format jsonl``,
+``ARIA_EVENTS_FULL=1``) and reported next to the outcome, never changing it::
+
+    behavior:
+      - called_before: {tool: impact_analysis, before: [edit_file, write_file], path: pricing.py}
+      - param_used: {tool: edit_file, param: symbol}
+      - not_called: write_file
+
+``called`` / ``not_called``: the tool (or any of a list) ran at least once /
+never. ``called_before``: the first call to ``tool`` came before the first
+call to any ``before`` tool (default: the edit tools) whose path, command or
+targets mention ``path``. ``param_used``: some call to ``tool`` set ``param``.
+A label may be given with ``label:``.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+CHECKS = ("called", "not_called", "called_before", "param_used")
+EDIT_TOOLS = ("edit_file", "write_file", "multi_edit", "apply_patch", "notebook_edit")
+
+
+@dataclass(frozen=True)
+class Call:
+    index: int
+    tool: str
+    params: dict
+
+    def mentions(self, fragment: str) -> bool:
+        if not fragment:
+            return True
+        text = " ".join(str(self.params.get(k) or "") for k in ("path", "file", "command", "targets"))
+        return fragment in text
+
+
+def calls(events_text: str) -> list[Call]:
+    """The tool calls in a run's JSONL output, in order; other lines are skipped."""
+    found: list[Call] = []
+    for line in str(events_text or "").splitlines():
+        try:
+            event = json.loads(line)
+        except ValueError:
+            continue
+        if isinstance(event, dict) and event.get("type") == "tool.started":
+            name = str(event.get("tool") or "").rsplit("__", 1)[-1]
+            found.append(Call(len(found), name, dict(event.get("params") or {})))
+    return found
+
+
+def _names(value: Any) -> tuple[str, ...]:
+    return tuple(str(v) for v in value) if isinstance(value, (list, tuple)) else (str(value),)
+
+
+def validate(raw: Any) -> dict:
+    """One check from a suite file, or ValueError naming what is wrong with it."""
+    if not isinstance(raw, dict):
+        raise ValueError("a behavior check is a mapping")
+    kinds = [key for key in raw if key in CHECKS]
+    unknown = set(raw) - set(CHECKS) - {"label"}
+    if unknown or len(kinds) != 1:
+        raise ValueError(f"a behavior check needs exactly one of {', '.join(CHECKS)}; got {sorted(raw)}")
+    kind = kinds[0]
+    spec = raw[kind]
+    if kind in ("called_before", "param_used") and not (isinstance(spec, dict) and spec.get("tool")):
+        raise ValueError(f"{kind} needs a tool")
+    if kind == "param_used" and not spec.get("param"):
+        raise ValueError("param_used needs a param")
+    return {"type": kind, "spec": spec, "label": str(raw.get("label") or "")}
+
+
+def describe(check: dict) -> str:
+    return check.get("label") or f"{check['type']}: {check['spec']}"
+
+
+def judge(check: dict, run: list[Call]) -> tuple[bool, str]:
+    kind, spec = check["type"], check["spec"]
+    if kind in ("called", "not_called"):
+        tools = _names(spec.get("tool") if isinstance(spec, dict) else spec)
+        hits = [c for c in run if c.tool in tools]
+        ok = bool(hits) if kind == "called" else not hits
+        return ok, f"{len(hits)} call(s) to {'/'.join(tools)}"
+    if kind == "called_before":
+        tools, later = _names(spec["tool"]), _names(spec.get("before", EDIT_TOOLS))
+        path = str(spec.get("path") or "")
+        look = next((c.index for c in run if c.tool in tools), None)
+        edit = next((c.index for c in run if c.tool in later and c.mentions(path)), None)
+        if edit is None:
+            return False, f"no {'/'.join(later)} call on {path or 'any file'}"
+        if look is None:
+            return False, f"{'/'.join(tools)} never called"
+        return look < edit, f"{run[look].tool} at call {look}, first edit at call {edit}"
+    tools, param = _names(spec["tool"]), str(spec["param"])
+    hits = [c for c in run if c.tool in tools]
+    used = [c for c in hits if c.params.get(param) not in (None, "")]
+    return bool(used), f"{len(used)}/{len(hits)} {'/'.join(tools)} call(s) set {param}"
+
+
+def evaluate(checks: Iterable[dict], events_text: str) -> tuple[dict, ...]:
+    run = calls(events_text)
+    results = []
+    for check in checks:
+        passed, detail = judge(check, run)
+        results.append({"check": describe(check), "passed": passed, "detail": detail})
+    return tuple(results)

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/evals/harness.py

@@ -139,6 +139,9 @@ class TaskSpec:
     # an answer key when it sits in the workspace. gemini-3.5-flash passed 23
     # of the first 24 tasks with every grader in view.
     hidden: tuple[str, ...] = ()
+    # How the agent should go about it, judged on its tool calls and reported
+    # beside the outcome without changing it. See evals/behavior.py.
+    behavior: tuple[dict, ...] = ()
 
     @classmethod
     def from_dict(cls, data: dict) -> "TaskSpec":
@@ -161,9 +164,16 @@ class TaskSpec:
             ),
             allow_green_start=bool(data.get("allow_green_start", False)),
             hidden=tuple(str(g) for g in (data.get("hidden") or ())),
+            behavior=tuple(_validate_behavior(c) for c in (data.get("behavior") or ())),
         )
 
 
+def _validate_behavior(raw) -> dict:
+    from .behavior import validate
+
+    return validate(raw)
+
+
 _SECRET = re.compile(
     r"(?i)\b(api[_-]?key|token|password|secret)(\s*[=:]\s*)\S+"
     r"|\b(?:bearer)\s+\S+"
@@ -199,6 +209,8 @@ class TaskResult:
     # this the two are indistinguishable in a report, and they call for
     # opposite investigations.
     changed: tuple[str, ...] = ()
+    # Behaviour checks on the agent's tool calls: ({check, passed, detail}, …).
+    behavior: tuple = ()
 
     @property
     def counted(self) -> bool:
@@ -217,6 +229,7 @@ class TaskResult:
             # Why it did not pass. The first CI run's report said only "the
             # agent did not complete (exit 1)"; the reason was in the job log.
             "log_tail": _log_tail(self.log) if self.outcome != PASS else [],
+            **({"behavior": [dict(b) for b in self.behavior]} if self.behavior else {}),
         }
 
 
@@ -284,9 +297,20 @@ class SuiteResult:
             "scored": self.scored,
             "by_tag": self.by_tag(),
             "per_task": self.per_task(),
+            **({"behavior": self.behavior()} if any(r.behavior for r in self.results) else {}),
             "results": [r.to_dict() for r in self.results],
         }
 
+    def behavior(self) -> dict:
+        """Passes over runs for each behaviour check, keyed ``task: check``."""
+        out: dict = {}
+        for result in self.results:
+            for item in result.behavior:
+                stats = out.setdefault(f"{result.task_id}: {item['check']}", {"passed": 0, "runs": 0})
+                stats["runs"] += 1
+                stats["passed"] += bool(item["passed"])
+        return out
+
     def merge(self, other: "SuiteResult") -> None:
         """Fold another run of the same suite into this one."""
         self.results.extend(other.results)
@@ -318,6 +342,9 @@ class SuiteResult:
             parts.append(f"{self.invalid} invalid")
         if self.errored:
             parts.append(f"{self.errored} error")
+        checks = [item for r in self.results for item in r.behavior]
+        if checks:
+            parts.append(f"behaviour {sum(bool(i['passed']) for i in checks)}/{len(checks)}")
         return " · ".join(parts)
 
 
@@ -539,10 +566,12 @@ def run_task(
     )
     scratch.mkdir(parents=True, exist_ok=True)
 
+    behaviour: list = []   # filled once the agent has had its turn
+
     def _result(outcome: str, **kwargs) -> TaskResult:
         return TaskResult(
             task_id=task.id, outcome=outcome,
-            seconds=time.time() - started, tags=task.tags, **kwargs,
+            seconds=time.time() - started, tags=task.tags, behavior=tuple(behaviour), **kwargs,
         )
 
     # Everything below runs inside this try so the scratch directory is removed
@@ -596,6 +625,12 @@ def run_task(
             # A solver crash is not the agent failing the task; scoring it as
             # a fail would blame the model for a harness or provider outage.
             return _result(ERROR, detail=f"solver raised: {exc}")
+        # Only when the solver ran an agent: the pre-flight solver returns
+        # nothing, and "write_file was never called" would read as a pass.
+        if task.behavior and outcome is not None:
+            from .behavior import evaluate
+
+            behaviour.extend(evaluate(task.behavior, str(getattr(outcome, "stdout", "") or "")))
 
         # ── the score ─────────────────────────────────────────────────────
         # The check runs no matter what the solver reported, because the check

aria_code-0.127.0/src/aria_code/evals/review_probe.py

@@ -0,0 +1,132 @@
+"""Score the independent reviewer on fixed changes: false positives and misses.
+
+    python -m aria_code.evals.review_probe evals/review/cases.yaml --model <id> [--repeat 3] [--report out.json]
+
+The suites measure the builder; the reviewer (runtime/review.py) blocks or
+passes the builder's diff, and nothing measured whether it is right to. Each
+case is a goal and a diff with the verdict it should get: ``expect: pass``
+for a correct change it must not block (a false positive costs a repair
+round and the user's trust), ``expect: block`` for one that misses the goal
+(a miss lets a wrong change through), optionally with ``mentions_any`` — the
+finding must name one of these.
+
+A reviewer that errors or answers out of format is an ERROR, not a verdict,
+like an agent that never ran. The exit code is 1 only for errors.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Awaitable, Callable
+
+import yaml
+
+PASS, FAIL, ERROR = "pass", "fail", "error"
+
+
+@dataclass(frozen=True)
+class Case:
+    id: str
+    goal: str
+    diff: str
+    expect: str                      # "pass" | "block"
+    mentions_any: tuple[str, ...] = ()
+
+
+def load_cases(path: str | Path) -> list[Case]:
+    data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
+    cases = []
+    for raw in data.get("cases") or []:
+        expect = str(raw.get("expect") or "")
+        if expect not in ("pass", "block") or not raw.get("goal") or not raw.get("diff"):
+            raise ValueError(f"case {raw.get('id')!r}: needs goal, diff and expect: pass|block")
+        cases.append(Case(id=str(raw["id"]), goal=str(raw["goal"]), diff=str(raw["diff"]), expect=expect,
+                          mentions_any=tuple(str(m) for m in raw.get("mentions_any") or ())))
+    return cases
+
+
+def score(case: Case, report: Any) -> tuple[str, str]:
+    """(outcome, detail) for one reviewer answer."""
+    if getattr(report, "verdict", "") == "error":
+        return ERROR, f"reviewer error: {getattr(report, 'error', '')}"
+    blocking = list(getattr(report, "blocking", ()) or ())
+    if case.expect == "pass":
+        if blocking:
+            return FAIL, f"false positive: {len(blocking)} blocking — {blocking[0].issue[:120]}"
+        return PASS, "not blocked"
+    if not blocking:
+        return FAIL, "missed: no blocking finding"
+    text = " ".join(f"{getattr(f, 'file', '')} {getattr(f, 'issue', '')}" for f in blocking).lower()
+    if case.mentions_any and not any(m.lower() in text for m in case.mentions_any):
+        return FAIL, f"blocked, but not for {' / '.join(case.mentions_any)}: {blocking[0].issue[:120]}"
+    return PASS, f"blocked: {blocking[0].issue[:120]}"
+
+
+Review = Callable[..., Awaitable[Any]]
+
+
+async def run_cases(cases: list[Case], review: Review, *, repeat: int = 1) -> list[dict]:
+    results = []
+    for attempt in range(1, max(1, repeat) + 1):
+        for case in cases:
+            try:
+                report = await review(goal=case.goal, diff=case.diff)
+                outcome, detail = score(case, report)
+            except Exception as exc:  # a provider outage is not the reviewer's verdict
+                outcome, detail = ERROR, f"{type(exc).__name__}: {exc}"
+            results.append({"case": case.id, "expect": case.expect, "attempt": attempt,
+                            "outcome": outcome, "detail": detail})
+    return results
+
+
+def summary(results: list[dict]) -> dict:
+    def rate(expect: str) -> dict:
+        scored = [r for r in results if r["expect"] == expect and r["outcome"] != ERROR]
+        return {"passed": sum(r["outcome"] == PASS for r in scored), "scored": len(scored)}
+
+    return {"correct_changes_not_blocked": rate("pass"), "wrong_changes_blocked": rate("block"),
+            "errors": sum(r["outcome"] == ERROR for r in results), "results": results}
+
+
+def _gate_review(model: str) -> Review:
+    from aria_code.aria_cli import load_config
+    from aria_code.apps.cli.providers.runtime_bridge import build_review_gate
+
+    config = {**load_config(), "review_gate": True}
+    if model:
+        config["model"] = model
+    gate = build_review_gate(config, model=config.get("model", ""), api_url=config.get("api_url"),
+                             ollama_url=config.get("ollama_url", "http://localhost:11434"))
+    return gate.run
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="review_probe", description=__doc__.splitlines()[0])
+    parser.add_argument("cases")
+    parser.add_argument("--model", default="")
+    parser.add_argument("--repeat", type=int, default=1)
+    parser.add_argument("--report", default="")
+    args = parser.parse_args(argv)
+
+    cases = load_cases(args.cases)
+    results = asyncio.run(run_cases(cases, _gate_review(args.model), repeat=args.repeat))
+    for r in results:
+        icon = {PASS: "✓", FAIL: "✗"}.get(r["outcome"], "!")
+        print(f"  {icon} {r['case']} (expect {r['expect']}) — {r['detail']}")
+    data = summary(results)
+    ok, wrong = data["correct_changes_not_blocked"], data["wrong_changes_blocked"]
+    print(f"\nreviewer: correct changes not blocked {ok['passed']}/{ok['scored']} · "
+          f"wrong changes blocked {wrong['passed']}/{wrong['scored']}"
+          + (f" · {data['errors']} error" if data["errors"] else ""))
+    if args.report:
+        Path(args.report).write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    return 1 if data["errors"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/evals/runner.py

@@ -100,7 +100,7 @@ def check_only_solver(prompt: str, workspace: Path) -> None:
 
 
 def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = False,
-                       trajectories: "Trajectories | None" = None):
+                       trajectories: "Trajectories | None" = None, events: bool = False):
     """Drive the real Aria agent over one task, in the task's own workspace.
 
     The agent is invoked as a subprocess in headless mode rather than in this
@@ -123,9 +123,10 @@ def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = Fal
             "--dangerously-skip-permissions",
             "--no-banner",
         ]
-        if trajectories is not None:
+        if trajectories is not None or events:
             # One JSON event per tool call on stdout; the screen output stays
-            # on stderr, which is what a failed task's log tail shows.
+            # on stderr, which is what a failed task's log tail shows. Behaviour
+            # checks read these events too.
             command.extend(["--format", "jsonl"])
         if model:
             command.extend(["--model", model])
@@ -133,7 +134,7 @@ def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = Fal
             command.append("--local")
         try:
             env = None
-            if trajectories is not None:
+            if trajectories is not None or events:
                 import os
                 env = {**os.environ, "ARIA_EVENTS_FULL": "1"}
             done = subprocess.run(
@@ -176,6 +177,8 @@ def _print_result(result: TaskResult) -> None:
     if result.detail:
         line += f"  — {result.detail}"
     print(line)
+    for item in result.behavior:
+        print(f"      {'✓' if item['passed'] else '✗'} {item['check']} — {item['detail']}")
     if result.outcome == FAIL and result.changed:
         print(f"      changed: {', '.join(result.changed)}")
     if result.outcome in (FAIL, INVALID, ERROR) and result.log:
@@ -224,6 +227,7 @@ def main(argv: list[str] | None = None) -> int:
                     if args.trajectories and not args.check else None)
     solver = check_only_solver if args.check else build_agent_solver(
         model=args.model, timeout=args.solve_timeout, local=args.local, trajectories=trajectories,
+        events=any(task.behavior for task in tasks),
     )
     mode = "pre-flight only" if args.check else f"agent{f' ({args.model})' if args.model else ''}"
     print(f"\n{name} — {len(tasks)} task(s), {mode}\n")

{aria_code-0.125.0 → aria_code-0.127.0}/src/aria_code/evals/summary.py

@@ -75,6 +75,10 @@ def summarise(reports: list[dict], model: str = "", private: list[dict] | None =
                 body = "\n".join(tail).replace("```", "` ` `")
                 lines += ["", f"<details><summary>{result.get('task_id')}: last lines of the log</summary>",
                           "", "```", body, "```", "</details>"]
+        checks = report.get("behavior") or {}
+        if checks:
+            lines += ["", "| Behaviour check | Passed |", "|---|---:|"]
+            lines += [f"| {name.replace('|', '/')} | {s['passed']}/{s['runs']} |" for name, s in checks.items()]
         by_tag = report.get("by_tag") or {}
         if by_tag:
             tags = ", ".join(f"{tag} {s['passed']}/{s['scored']}" for tag, s in by_tag.items() if s.get("scored"))