agentx-python 0.8.8__tar.gz → 0.8.10__tar.gz

{agentx_python-0.8.8 → agentx_python-0.8.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.8
+Version: 0.8.10
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -193,7 +193,7 @@ client.evaluations.run(
 
 See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet).
 
-On self-host, a finalized run can also **gate a CI job**: `report.gate(fail_under=7, no_regression=True)` checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
+On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
 
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
 

{agentx_python-0.8.8 → agentx_python-0.8.10}/README.md

@@ -129,7 +129,7 @@ client.evaluations.run(
 
 See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet).
 
-On self-host, a finalized run can also **gate a CI job**: `report.gate(fail_under=7, no_regression=True)` checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
+On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
 
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
 

{agentx_python-0.8.8 → agentx_python-0.8.10}/agentx/cli.py

@@ -4,18 +4,21 @@ Evaluate, Monitor), published separately at github.com/AgentX-ai/AgentX-trace-ev
 wrapping a Bun-compiled TypeScript engine, not Python). That compiled engine binary is tens of
 megabytes; most `pip install agentx-python` installs are just this SDK talking to the hosted
 AgentX SaaS and would never touch it, so it isn't bundled in this package. Instead, this command
-downloads the matching release into ~/.agentx/bin the first time it's needed (mirroring
+downloads the release into ~/.agentx/bin the first time it's needed (mirroring
 AgentX-trace-eval's own install.sh) and then hands off to the real `agentx-server` binary.
 
-The installed release tag is stamped in ~/.agentx/bin/.version. The launcher never silently
-re-downloads, but it re-installs when you ask: `--update` fetches the newest release (engine +
-dashboard), and setting AGENTX_TRACE_EVAL_VERSION to a tag other than the stamped one switches
-to that release. When running "latest", a quick fail-open check against GitHub tells you if a
-newer release exists.
+Versioning: each SDK release pins the engine release it was tested against
+(agentx.version.ENGINE_VERSION); the launcher installs that pin and converges to it - if the
+stamped install (~/.agentx/bin/.version) differs from the pin, it re-installs, so upgrading the
+SDK upgrades the engine to the matching pair. `--update` force-reinstalls the resolved version
+(recovering a broken or pre-stamp install). Set AGENTX_TRACE_EVAL_VERSION to a tag or `latest`
+to override the pin; in `latest` mode the launcher trusts whatever is installed, prints a
+fail-open notice when GitHub has something newer, and `--update` fetches it.
 
 Usage:
     agentx-trace-eval --dev
     agentx-trace-eval --update --dev
+    AGENTX_TRACE_EVAL_VERSION=latest agentx-trace-eval --update --dev
     agentx-trace-eval --port 5000 --db-url postgres://...
 """
 
@@ -32,6 +35,8 @@ from typing import Optional, Tuple
 
 import requests
 
+from agentx.version import ENGINE_VERSION
+
 REPO = "AgentX-ai/AgentX-trace-eval"
 INSTALL_DIR = Path(os.environ.get("AGENTX_INSTALL_DIR", str(Path.home() / ".agentx" / "bin")))
 _BIN_NAMES = ("agentx", "agentx-server", "agentx-engine")
@@ -114,11 +119,17 @@ def _latest_release_tag() -> Optional[str]:
         return None
 
 
+def resolve_version() -> str:
+    """The engine version this launcher should be running: the AGENTX_TRACE_EVAL_VERSION
+    override (a tag, or `latest`) when set, else the SDK's tested pin."""
+    return os.environ.get("AGENTX_TRACE_EVAL_VERSION") or ENGINE_VERSION
+
+
 def _install(version: str = "latest") -> None:
     os_name, arch = _platform_tag()
     INSTALL_DIR.mkdir(parents=True, exist_ok=True)
 
-    print(f"agentx-trace-eval: downloading agentx ({os_name}/{arch})...", file=sys.stderr)
+    print(f"agentx-trace-eval: downloading agentx {version} ({os_name}/{arch})...", file=sys.stderr)
     with tempfile.TemporaryDirectory() as tmp:
         tmp_dir = Path(tmp)
         archive = tmp_dir / "agentx.tar.gz"
@@ -178,23 +189,30 @@ def _install(version: str = "latest") -> None:
         _extract_tar(web_archive, web_dir)
 
 
-def ensure_installed(version: str = "latest", force: bool = False) -> Path:
-    """Downloads agentx-server (+ its engine and dashboard) into ~/.agentx/bin when it's missing
+def ensure_installed(version: Optional[str] = None, force: bool = False) -> Path:
+    """Installs agentx-server (+ its engine and dashboard) into ~/.agentx/bin when it's missing
     there, when `force` is set, or when `version` is a specific tag that differs from the
-    installed one. Returns the path to the agentx-server executable. Set AGENTX_INSTALL_DIR to
-    change where this looks/installs; set AGENTX_TRACE_EVAL_VERSION to pin a release tag."""
+    stamped install - the convergence that makes an SDK upgrade carry its tested engine along.
+    `version` defaults to resolve_version() (the AGENTX_TRACE_EVAL_VERSION override, else the
+    SDK's pin); pass "latest" to trust whatever is installed and only fetch when missing.
+    Returns the path to the agentx-server executable. Set AGENTX_INSTALL_DIR to change where
+    this looks/installs."""
+    if version is None:
+        version = resolve_version()
     server_path = INSTALL_DIR / "agentx-server"
-    installed = _stamped_version()
-    pin_changed = version != "latest" and installed is not None and installed != version
-    if force or pin_changed or not server_path.exists():
+    stamped = _stamped_version()
+    pin_mismatch = version != "latest" and stamped != version
+    if force or pin_mismatch or not server_path.exists():
         _install(version=version)
     return server_path
 
 
 def _maybe_print_update_notice(installed: Optional[str]) -> None:
+    """Only relevant in `latest` mode, where nothing converges automatically - a quick fail-open
+    check tells the user when the world has moved on. (Pinned mode needs no notice: a pin
+    mismatch re-installs instead of nagging.)"""
     latest = _latest_release_tag()
     if installed is None:
-        # Pre-stamp install (or a wiped stamp): age unknown, so always point at --update.
         print(
             "agentx-trace-eval: installed engine version unknown"
             + (f" (newest release is {latest})" if latest else "")
@@ -215,7 +233,7 @@ def main() -> None:
     # --update belongs to this launcher, not to agentx-server - consume it before the handoff.
     args = [a for a in args if a != "--update"]
 
-    version = os.environ.get("AGENTX_TRACE_EVAL_VERSION", "latest")
+    version = resolve_version()
     server_path = ensure_installed(version=version, force=force_update)
     if not server_path.exists():
         raise SystemExit(f"agentx-trace-eval: {server_path} still missing after install, giving up")

{agentx_python-0.8.8 → agentx_python-0.8.10}/agentx/evaluations/client.py

@@ -37,6 +37,9 @@ _RETRY_BACKOFF = [1.0, 2.0, 4.0]
 # wait out the whole job on one connection. Matches EvaluationRunContext.analyze()'s own
 # default timeout.
 _SELF_HOST_ANALYZE_TIMEOUT = 1800
+# Batch result submission scores each result synchronously inside the request (one judge call
+# per result on the sync path) - a big batch on a slow judge legitimately takes minutes.
+_SELF_HOST_SCORING_TIMEOUT = 900
 
 
 class AgentXEvaluationsError(Exception):
@@ -344,7 +347,15 @@ class EvaluationsClient:
             "batchId": batch_id,
             "results": [_result_to_payload(r) for r in results],
         }
-        data = self._request("POST", f"/runs/{run_id}/results", json=payload)
+        # Scoring is synchronous inside this request (a judge call per result - the runner's
+        # spinner says "~60s+" for a reason), so the default 30s timeout + silent backoff loop
+        # re-POSTed the batch WHILE the first submission was still scoring, and the two raced
+        # into the engine's (run_id, idempotency_key) unique constraint. Long timeout, no
+        # transport retry - the runner's own logged retry-once is the retry layer here.
+        data = self._request(
+            "POST", f"/runs/{run_id}/results", json=payload,
+            timeout=_SELF_HOST_SCORING_TIMEOUT, retry=False,
+        )
         return BatchAppendResponse(**data)
 
     def finalize_run(self, run_id: str) -> Dict[str, Any]:

agentx_python-0.8.10/agentx/version.py

@@ -0,0 +1,7 @@
+VERSION = "0.8.10"
+
+# The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
+# installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
+# every published SDK names a known-good engine+dashboard pair. Users can override with
+# AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
+ENGINE_VERSION = "v0.3.1"

{agentx_python-0.8.8 → agentx_python-0.8.10}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.8
+Version: 0.8.10
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -193,7 +193,7 @@ client.evaluations.run(
 
 See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet).
 
-On self-host, a finalized run can also **gate a CI job**: `report.gate(fail_under=7, no_regression=True)` checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
+On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
 
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
 

agentx_python-0.8.10/tests/test_cli_launcher.py

@@ -0,0 +1,117 @@
+"""The agentx-trace-eval launcher's version pinning: each SDK release names the engine release
+it was tested against (agentx.version.ENGINE_VERSION) and the launcher converges the local
+install to it, so `pip install -U agentx-python` upgrades the engine to the matching pair.
+Before this, the launcher downloaded "latest" once and then ran that binary forever - a July
+install silently served a July engine months later, with no stamp and no way to update."""
+
+import pytest
+
+from agentx import cli
+from agentx.version import ENGINE_VERSION
+
+
+@pytest.fixture
+def install_dir(tmp_path, monkeypatch):
+    monkeypatch.setattr(cli, "INSTALL_DIR", tmp_path)
+    return tmp_path
+
+
+def _fake_install(calls):
+    def fake(version="latest"):
+        calls.append(version)
+
+    return fake
+
+
+def test_engine_pin_is_a_release_tag():
+    # The pin the SDK ships must look like a real trace-eval release tag (v<series>.<n>), not
+    # "latest" - "latest" as a pin would silently disable the whole tested-pair contract.
+    assert cli.ENGINE_VERSION == ENGINE_VERSION
+    import re
+
+    assert re.fullmatch(r"v\d+\.\d+\.\d+", ENGINE_VERSION)
+
+
+def test_resolve_version_prefers_env_override(monkeypatch):
+    monkeypatch.delenv("AGENTX_TRACE_EVAL_VERSION", raising=False)
+    assert cli.resolve_version() == ENGINE_VERSION
+    monkeypatch.setenv("AGENTX_TRACE_EVAL_VERSION", "latest")
+    assert cli.resolve_version() == "latest"
+    monkeypatch.setenv("AGENTX_TRACE_EVAL_VERSION", "v9.9.9")
+    assert cli.resolve_version() == "v9.9.9"
+
+
+def test_tag_resolved_from_redirected_release_url():
+    url = "https://github.com/AgentX-ai/AgentX-trace-eval/releases/download/v0.3.1/agentx_darwin_arm64.tar.gz"
+    assert cli._tag_from_release_url(url) == "v0.3.1"
+    assert cli._tag_from_release_url("https://example.com/not-a-release") is None
+
+
+def test_install_matching_pin_is_not_redownloaded(install_dir, monkeypatch):
+    (install_dir / "agentx-server").write_text("bin")
+    (install_dir / ".version").write_text(ENGINE_VERSION + "\n")
+    calls = []
+    monkeypatch.setattr(cli, "_install", _fake_install(calls))
+    cli.ensure_installed(version=ENGINE_VERSION)
+    assert calls == []
+
+
+def test_stamp_differing_from_pin_converges(install_dir, monkeypatch):
+    # The SDK-upgrade path: binaries exist but were installed for an older pin (or pre-stamp,
+    # where .version is missing entirely) - the launcher re-installs the pinned pair.
+    (install_dir / "agentx-server").write_text("bin")
+    (install_dir / ".version").write_text("v0.2.0\n")
+    calls = []
+    monkeypatch.setattr(cli, "_install", _fake_install(calls))
+    cli.ensure_installed(version=ENGINE_VERSION)
+    assert calls == [ENGINE_VERSION]
+
+    (install_dir / ".version").unlink()
+    calls.clear()
+    cli.ensure_installed(version=ENGINE_VERSION)
+    assert calls == [ENGINE_VERSION]
+
+
+def test_latest_mode_trusts_existing_install(install_dir, monkeypatch):
+    (install_dir / "agentx-server").write_text("bin")
+    (install_dir / ".version").write_text("v0.2.0\n")
+    calls = []
+    monkeypatch.setattr(cli, "_install", _fake_install(calls))
+    cli.ensure_installed(version="latest")
+    assert calls == []
+
+
+def test_force_reinstalls_over_existing(install_dir, monkeypatch):
+    (install_dir / "agentx-server").write_text("bin")
+    (install_dir / ".version").write_text(ENGINE_VERSION + "\n")
+    calls = []
+    monkeypatch.setattr(cli, "_install", _fake_install(calls))
+    cli.ensure_installed(version=ENGINE_VERSION, force=True)
+    assert calls == [ENGINE_VERSION]
+
+
+def test_update_notice_names_both_versions(install_dir, monkeypatch, capsys):
+    monkeypatch.setattr(cli, "_latest_release_tag", lambda: "v0.9.0")
+    cli._maybe_print_update_notice("v0.3.1")
+    err = capsys.readouterr().err
+    assert "v0.3.1" in err and "v0.9.0" in err and "--update" in err
+
+
+def test_update_notice_handles_unstamped_install(install_dir, monkeypatch, capsys):
+    monkeypatch.setattr(cli, "_latest_release_tag", lambda: None)
+    cli._maybe_print_update_notice(None)
+    assert "--update" in capsys.readouterr().err
+
+
+def test_update_notice_is_silent_when_current(install_dir, monkeypatch, capsys):
+    monkeypatch.setattr(cli, "_latest_release_tag", lambda: "v0.3.1")
+    cli._maybe_print_update_notice("v0.3.1")
+    assert capsys.readouterr().err == ""
+
+
+def test_latest_release_tag_fails_open(monkeypatch):
+    def boom(*args, **kwargs):
+        raise OSError("offline")
+
+    monkeypatch.setattr(cli.requests, "get", boom)
+    assert cli._latest_release_tag() is None

agentx_python-0.8.8/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.8.8"

agentx_python-0.8.8/tests/test_cli_launcher.py

@@ -1,89 +0,0 @@
-"""The agentx-trace-eval launcher's staleness fixes: before these, ensure_installed downloaded
-once and then ran that binary forever - a July install silently served a July engine months
-later, with no version stamp, no way to ask for an update, and the dashboard fetched from
-releases/latest independently of the engine (so the two halves of one install could skew)."""
-
-import io
-import sys
-
-import pytest
-
-from agentx import cli
-
-
-@pytest.fixture
-def install_dir(tmp_path, monkeypatch):
-    monkeypatch.setattr(cli, "INSTALL_DIR", tmp_path)
-    return tmp_path
-
-
-def _fake_install(calls):
-    def fake(version="latest"):
-        calls.append(version)
-
-    return fake
-
-
-def test_tag_resolved_from_redirected_release_url():
-    url = "https://github.com/AgentX-ai/AgentX-trace-eval/releases/download/v0.3.0/agentx_darwin_arm64.tar.gz"
-    assert cli._tag_from_release_url(url) == "v0.3.0"
-    assert cli._tag_from_release_url("https://example.com/not-a-release") is None
-
-
-def test_existing_install_is_not_redownloaded(install_dir, monkeypatch):
-    (install_dir / "agentx-server").write_text("bin")
-    calls = []
-    monkeypatch.setattr(cli, "_install", _fake_install(calls))
-    cli.ensure_installed(version="latest")
-    assert calls == []
-
-
-def test_force_reinstalls_over_existing(install_dir, monkeypatch):
-    (install_dir / "agentx-server").write_text("bin")
-    calls = []
-    monkeypatch.setattr(cli, "_install", _fake_install(calls))
-    cli.ensure_installed(version="latest", force=True)
-    assert calls == ["latest"]
-
-
-def test_changed_version_pin_reinstalls(install_dir, monkeypatch):
-    (install_dir / "agentx-server").write_text("bin")
-    (install_dir / ".version").write_text("v0.2.0\n")
-    calls = []
-    monkeypatch.setattr(cli, "_install", _fake_install(calls))
-
-    # Same pin: no download. Different pin: download. "latest": trusts what's there.
-    cli.ensure_installed(version="v0.2.0")
-    assert calls == []
-    cli.ensure_installed(version="v0.3.0")
-    assert calls == ["v0.3.0"]
-    calls.clear()
-    cli.ensure_installed(version="latest")
-    assert calls == []
-
-
-def test_update_notice_names_both_versions(install_dir, monkeypatch, capsys):
-    monkeypatch.setattr(cli, "_latest_release_tag", lambda: "v0.9.0")
-    cli._maybe_print_update_notice("v0.2.0")
-    err = capsys.readouterr().err
-    assert "v0.2.0" in err and "v0.9.0" in err and "--update" in err
-
-
-def test_update_notice_handles_unstamped_install(install_dir, monkeypatch, capsys):
-    monkeypatch.setattr(cli, "_latest_release_tag", lambda: None)
-    cli._maybe_print_update_notice(None)
-    assert "--update" in capsys.readouterr().err
-
-
-def test_update_notice_is_silent_when_current(install_dir, monkeypatch, capsys):
-    monkeypatch.setattr(cli, "_latest_release_tag", lambda: "v0.2.0")
-    cli._maybe_print_update_notice("v0.2.0")
-    assert capsys.readouterr().err == ""
-
-
-def test_latest_release_tag_fails_open(monkeypatch):
-    def boom(*args, **kwargs):
-        raise OSError("offline")
-
-    monkeypatch.setattr(cli.requests, "get", boom)
-    assert cli._latest_release_tag() is None