aria-code 0.104.0__tar.gz → 0.105.0__tar.gz

{aria_code-0.104.0/src/aria_code.egg-info → aria_code-0.105.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aria-code
-Version: 0.104.0
+Version: 0.105.0
 Summary: Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/artheras/aria-code

{aria_code-0.104.0 → aria_code-0.105.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aria-code"
-version = "0.104.0"
+version = "0.105.0"
 description = "Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{aria_code-0.104.0 → aria_code-0.105.0}/src/aria_code/_version.py

@@ -17,4 +17,4 @@ from __future__ import annotations
 
 __all__ = ["__version__"]
 
-__version__ = "0.104.0"
+__version__ = "0.105.0"

{aria_code-0.104.0 → aria_code-0.105.0}/src/aria_code/apps/cli/exec_events.py

@@ -17,6 +17,7 @@ or a command's token have no place in a CI log.
 from __future__ import annotations
 
 import json
+import re
 import time
 from typing import Any, TextIO
 
@@ -30,7 +31,52 @@ def json_safe(value: Any) -> Any:
     return json.loads(json.dumps(value, ensure_ascii=False, default=str))
 
 
+RESULT_CHARS = 20_000
+
+
+def _full() -> bool:
+    """ARIA_EVENTS_FULL=1: keep whole parameters and tool results (eval trajectories).
+
+    The default view is for logs and CI: short, file bodies by size. A
+    trajectory to learn from needs what was written and what came back.
+    """
+    import os
+
+    return os.environ.get("ARIA_EVENTS_FULL", "").strip() in ("1", "true", "yes")
+
+
+_SECRET = re.compile(
+    r"(?i)\b(api[_-]?key|token|password|passwd|secret)(\s*[=:]\s*)[^\s'\"]+"
+    r"|\b(bearer|basic)(\s+)[A-Za-z0-9._~+/=-]+"
+    r"|\b(?:sk|pk|rk)-[A-Za-z0-9_-]{8,}|\bgh[pousr]_[A-Za-z0-9]{12,}|\bAIza[0-9A-Za-z_-]{20,}"
+    r"|\bya29\.[0-9A-Za-z_-]+"
+)
+
+
+def _mask_text(text: str) -> str:
+    """Secret-shaped values masked; whitespace, indentation and newlines kept."""
+    def repl(match: "re.Match") -> str:
+        if match.group(1):
+            return f"{match.group(1)}{match.group(2)}***"
+        if match.group(3):
+            return f"{match.group(3)}{match.group(4)}***"
+        return "***"
+    return _SECRET.sub(repl, str(text))
+
+
+def _mask_value(value: Any) -> Any:
+    if isinstance(value, str):
+        return _mask_text(value)
+    if isinstance(value, dict):
+        return {k: _mask_value(v) for k, v in value.items() if not str(k).startswith("_")}
+    if isinstance(value, list):
+        return [_mask_value(v) for v in value]
+    return value
+
+
 def _param_view(params: dict) -> dict:
+    if _full():
+        return _mask_value(json_safe(dict(params or {})))
     view = {}
     for key, value in (params or {}).items():
         if str(key).startswith("_"):
@@ -77,6 +123,10 @@ class ExecEvents:
         for key in ("exit_code", "path", "process_id", "change_id"):
             if key in data:
                 fields[key] = data[key]
+        if _full():
+            masked = _mask_value(json_safe(result))
+            body = json.dumps(masked, ensure_ascii=False)
+            fields["result"] = masked if len(body) <= RESULT_CHARS else body[:RESULT_CHARS] + "…"
         self.emit("tool.completed", **fields)
 
 

{aria_code-0.104.0 → aria_code-0.105.0}/src/aria_code/evals/runner.py

@@ -14,6 +14,7 @@ catches it on the commit that caused it rather than a month later.
 from __future__ import annotations
 
 import argparse
+import json
 import sys
 from pathlib import Path
 
@@ -47,6 +48,48 @@ class _TimedOut:
         self.stdout = ""
 
 
+class Trajectories:
+    """Each attempt's events as JSONL, closed with the outcome the check gave.
+
+    The material a model learns an agent's job from is the sequence, not the
+    final answer: which file it read, what it ran, what came back, what it
+    changed, and whether the check then passed. `-p --format jsonl` already
+    emits that; this keeps one file per task attempt and, once the suite is
+    scored, appends an `eval.outcome` line, so passing runs can be selected.
+    """
+
+    def __init__(self, root: Path, suite: str, model: str) -> None:
+        self.root = Path(root) / suite
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.model = model
+        self._files: dict[str, list[Path]] = {}
+
+    def record(self, task_id: str, prompt: str, stdout: str) -> Path:
+        attempt = len(self._files.get(task_id, [])) + 1
+        path = self.root / f"{task_id}-{attempt}.jsonl"
+        header = json.dumps({"type": "eval.task", "task_id": task_id, "attempt": attempt,
+                             "model": self.model, "prompt": prompt}, ensure_ascii=False)
+        body = "\n".join(line for line in str(stdout or "").splitlines() if line.startswith("{"))
+        path.write_text(header + "\n" + (body + "\n" if body else ""), encoding="utf-8")
+        self._files.setdefault(task_id, []).append(path)
+        return path
+
+    def close(self, results) -> None:
+        """Append each attempt's outcome, in the order the attempts ran."""
+        seen: dict[str, int] = {}
+        for result in results:
+            files = self._files.get(result.task_id, [])
+            index = seen.get(result.task_id, 0)
+            seen[result.task_id] = index + 1
+            if index >= len(files):
+                continue
+            line = json.dumps({"type": "eval.outcome", "outcome": result.outcome, "detail": result.detail,
+                               "changed": list(result.changed), "seconds": round(result.seconds, 2)},
+                              ensure_ascii=False)
+            with files[index].open("a", encoding="utf-8") as handle:
+                handle.write(line + "\n")
+
+
 def check_only_solver(prompt: str, workspace: Path) -> None:
     """A solver that does nothing, so only the pre-flight is exercised.
 
@@ -56,7 +99,8 @@ def check_only_solver(prompt: str, workspace: Path) -> None:
     return None
 
 
-def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = False):
+def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = False,
+                       trajectories: "Trajectories | None" = None):
     """Drive the real Aria agent over one task, in the task's own workspace.
 
     The agent is invoked as a subprocess in headless mode rather than in this
@@ -79,19 +123,31 @@ def build_agent_solver(*, model: str = "", timeout: int = 900, local: bool = Fal
             "--dangerously-skip-permissions",
             "--no-banner",
         ]
+        if trajectories is not None:
+            # One JSON event per tool call on stdout; the screen output stays
+            # on stderr, which is what a failed task's log tail shows.
+            command.extend(["--format", "jsonl"])
         if model:
             command.extend(["--model", model])
         if local:
             command.append("--local")
         try:
-            return subprocess.run(
+            env = None
+            if trajectories is not None:
+                import os
+                env = {**os.environ, "ARIA_EVENTS_FULL": "1"}
+            done = subprocess.run(
                 command,
                 cwd=str(workspace),
                 capture_output=True,
                 text=True,
                 timeout=timeout,
                 check=False,
+                env=env,
             )
+            if trajectories is not None:
+                trajectories.record(workspace.name, prompt, done.stdout)
+            return done
         except subprocess.TimeoutExpired:
             # A truncated run is not a measured failure.
             #
@@ -137,6 +193,8 @@ def main(argv: list[str] | None = None) -> int:
                         help="run the suite N times; pass@1 is then passes/attempts (default 1)")
     parser.add_argument("--report", default="", help="write the scoreboard to this JSON path")
     parser.add_argument("--keep", action="store_true", help="keep task workspaces for inspection")
+    parser.add_argument("--trajectories", default="",
+                        help="write each attempt's events (JSONL) and outcome under this folder")
     parser.add_argument("--scratch", default="", help="scratch root (implies --keep)")
     args = parser.parse_args(argv)
 
@@ -155,8 +213,10 @@ def main(argv: list[str] | None = None) -> int:
         print(f"suite {name} has no tasks", file=sys.stderr)
         return 2
 
+    trajectories = (Trajectories(Path(args.trajectories), name, args.model)
+                    if args.trajectories and not args.check else None)
     solver = check_only_solver if args.check else build_agent_solver(
-        model=args.model, timeout=args.solve_timeout, local=args.local,
+        model=args.model, timeout=args.solve_timeout, local=args.local, trajectories=trajectories,
     )
     mode = "pre-flight only" if args.check else f"agent{f' ({args.model})' if args.model else ''}"
     print(f"\n{name} — {len(tasks)} task(s), {mode}\n")
@@ -182,6 +242,9 @@ def main(argv: list[str] | None = None) -> int:
         else:
             suite.merge(run)
     assert suite is not None
+    if trajectories is not None:
+        trajectories.close(suite.results)
+        print(f"trajectories → {trajectories.root}")
 
     print(f"\n{suite.summary_line()}")
     if repeats > 1:

{aria_code-0.104.0 → aria_code-0.105.0/src/aria_code.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aria-code
-Version: 0.104.0
+Version: 0.105.0
 Summary: Open-source AI agent for coding, financial analysis and logistics operations: numbers you can check, client data kept apart, actions a person approves
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/artheras/aria-code

{aria_code-0.104.0 → aria_code-0.105.0}/src/aria_code.egg-info/SOURCES.txt

@@ -585,6 +585,7 @@ tests/test_english_output_group.py
 tests/test_enterprise_tool_honesty.py
 tests/test_env_switch_docs.py
 tests/test_eval_summary.py
+tests/test_eval_trajectories.py
 tests/test_evals_harness.py
 tests/test_event_import_roots.py
 tests/test_exec_events.py

aria_code-0.105.0/tests/test_eval_trajectories.py

@@ -0,0 +1,81 @@
+"""Each eval attempt is kept as a trajectory: the prompt, every tool call and result, the outcome.
+
+That sequence is what an agent model learns from. `-p --format jsonl` emits
+it; in full mode (ARIA_EVENTS_FULL=1, set by the eval runner) parameters and
+tool results are kept whole, with secret-shaped values masked and indentation
+intact.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+from aria_code.apps.cli.exec_events import ExecEvents
+from aria_code.evals.runner import Trajectories
+
+
+def test_full_mode_keeps_code_and_results(monkeypatch):
+    monkeypatch.setenv("ARIA_EVENTS_FULL", "1")
+    out = io.StringIO()
+    events = ExecEvents(out)
+    events.tool_started("write_file", {"path": "a.py", "content": "def f():\n    return 1\n" * 300, "_run_id": "r"})
+    events.tool_completed("run_command", {"success": True, "data": {"stdout": "token=abc123secret\n5\n",
+                                                                   "exit_code": 0}})
+    started, completed = [json.loads(line) for line in out.getvalue().splitlines()]
+    assert started["params"]["content"].startswith("def f():\n    return 1\n")
+    assert len(started["params"]["content"]) == len("def f():\n    return 1\n") * 300
+    assert "_run_id" not in started["params"]
+    assert completed["result"]["data"]["stdout"] == "token=***\n5\n"
+
+
+def test_the_default_view_is_unchanged(monkeypatch):
+    monkeypatch.delenv("ARIA_EVENTS_FULL", raising=False)
+    out = io.StringIO()
+    events = ExecEvents(out)
+    events.tool_started("write_file", {"path": "a.py", "content": "x" * 5000})
+    events.tool_completed("run_command", {"success": True, "data": {"stdout": "5"}})
+    started, completed = [json.loads(line) for line in out.getvalue().splitlines()]
+    assert started["params"]["content"] == "<5000 chars>" and "result" not in completed
+
+
+def test_trajectories_are_one_file_per_attempt_closed_with_the_outcome(tmp_path):
+    store = Trajectories(tmp_path, "operations", "google/gemini-3.5-flash")
+    events = '{"type": "turn.started"}\nnoise from the screen\n{"type": "turn.completed", "success": true}\n'
+    first = store.record("inventory-reorder", "write reorder.json", events)
+    second = store.record("inventory-reorder", "write reorder.json", events)
+    store.close([
+        SimpleNamespace(task_id="inventory-reorder", outcome="fail", detail="exited 1", changed=(), seconds=17.0),
+        SimpleNamespace(task_id="inventory-reorder", outcome="pass", detail="", changed=("reorder.json",),
+                        seconds=39.0),
+    ])
+    assert first.name == "inventory-reorder-1.jsonl" and second.name == "inventory-reorder-2.jsonl"
+    lines = [json.loads(line) for line in Path(second).read_text().splitlines()]
+    assert lines[0] == {"type": "eval.task", "task_id": "inventory-reorder", "attempt": 2,
+                        "model": "google/gemini-3.5-flash", "prompt": "write reorder.json"}
+    assert [l["type"] for l in lines] == ["eval.task", "turn.started", "turn.completed", "eval.outcome"]
+    assert lines[-1]["outcome"] == "pass" and lines[-1]["changed"] == ["reorder.json"]
+    assert json.loads(Path(first).read_text().splitlines()[-1])["outcome"] == "fail"
+
+
+def test_the_runner_asks_for_events_and_full_mode(monkeypatch, tmp_path):
+    import subprocess
+
+    from aria_code.evals import runner
+
+    seen = {}
+
+    def fake_run(command, **kwargs):
+        seen["command"], seen["env"] = command, kwargs.get("env")
+        return SimpleNamespace(returncode=0, stdout='{"type": "turn.completed"}\n', stderr="")
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    store = Trajectories(tmp_path, "core", "m")
+    workspace = tmp_path / "off-by-one"
+    workspace.mkdir()
+    runner.build_agent_solver(model="m", trajectories=store)("fix it", workspace)
+    assert seen["command"][seen["command"].index("--format") + 1] == "jsonl"
+    assert seen["env"]["ARIA_EVENTS_FULL"] == "1"
+    assert (tmp_path / "core" / "off-by-one-1.jsonl").exists()