agentx-python 0.6.38__tar.gz → 0.8.1__tar.gz

{agentx_python-0.6.38 → agentx_python-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.38
+Version: 0.8.1
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.38 → agentx_python-0.8.1}/agentx/evaluations/client.py

@@ -17,6 +17,7 @@ from agentx.evaluations.models import (
     EvaluationSettings,
     EvaluationSubject,
     ModelInfo,
+    PairwiseComparison,
     Prompt,
     Report,
 )
@@ -541,6 +542,63 @@ class EvaluationsClient:
             "POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload
         )
 
+    # ------------------------------------------------------------------
+    # Head-to-head (pairwise) judging. Rides the /evaluate dialect via _api_root(), same
+    # precedent as get_report and the prompt loop above.
+    # ------------------------------------------------------------------
+
+    def compare_pairwise(
+        self,
+        run_a_id: str,
+        run_b_id: str,
+        *,
+        criteria: Optional[str] = None,
+        judge_model: Optional[str] = None,
+        both_orders: bool = False,
+    ) -> PairwiseComparison:
+        """Ask a judge which of two runs answered each question better.
+
+        Both runs must be of the same dataset. Absolute ratings answer "is this above the bar";
+        this answers "did the change help", which is the question a diff between two runs is
+        actually asking. The judge sees the two answers as "Answer 1"/"Answer 2" with the order
+        alternating case by case, so it never learns which run is the candidate.
+
+        ``both_orders=True`` judges every pair twice with the sides swapped. It doubles the judge
+        cost and is the only real defense against position bias: a pair whose winner reverses is
+        recorded as a tie, and the batch reports its ``flip_rate``.
+
+        ``criteria`` and ``judge_model`` default to the dataset's own evaluation criteria and
+        judge model, so a head-to-head grades on the same terms a normal run of it does.
+        """
+        payload: Dict[str, Any] = {"runAId": run_a_id, "runBId": run_b_id}
+        if criteria:
+            payload["criteria"] = criteria
+        if judge_model:
+            payload["judgeModel"] = judge_model
+        if both_orders:
+            payload["bothOrders"] = True
+        response = self._request("POST", "/evaluate/runs/pairwise", json=payload, base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def get_pairwise(self, batch_id: str) -> PairwiseComparison:
+        """Read back a stored head-to-head by its batch id."""
+        response = self._request("GET", f"/evaluate/runs/pairwise/{batch_id}", base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def list_pairwise(
+        self, *, run_a_id: Optional[str] = None, run_b_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        params: Dict[str, Any] = {}
+        if run_a_id:
+            params["runAId"] = run_a_id
+        if run_b_id:
+            params["runBId"] = run_b_id
+        response = self._request(
+            "GET", "/evaluate/runs/pairwise", params=params or None, base=self._api_root
+        )
+        return response.get("comparisons", [])
+
     # ------------------------------------------------------------------
     # Tool schema registry (same version-scoped propose/publish loop as prompts)
     # ------------------------------------------------------------------

{agentx_python-0.6.38 → agentx_python-0.8.1}/agentx/evaluations/models.py

@@ -742,3 +742,64 @@ class AnalysisStatus(BaseModel):
     @property
     def is_terminal(self) -> bool:
         return self.status in ("completed", "partially_failed", "failed")
+
+
+class PairwiseSummary(BaseModel):
+    """Batch-level verdict of a head-to-head comparison (``compare_pairwise``).
+
+    ``winner`` is "a", "b", or "tie" - a dead heat is reported as a tie rather than broken
+    arbitrarily. ``flip_rate`` is only populated for a ``both_orders=True`` comparison: it is the
+    share of cases whose winner reversed when the two answers were swapped, which is position
+    bias rather than quality. A high flip rate means the batch is inconclusive, so it is reported
+    instead of being folded away."""
+
+    total: int = 0
+    a_wins: int = Field(default=0, alias="aWins")
+    b_wins: int = Field(default=0, alias="bWins")
+    ties: int = 0
+    winner: str = "tie"
+    flip_rate: Optional[float] = Field(default=None, alias="flipRate")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseCase(BaseModel):
+    """One question's verdict. ``presented_first`` records which run's answer the judge read
+    first, because that is the confound pairwise judging exists to control for."""
+
+    id: Optional[str] = Field(default=None, alias="_id")
+    question_index: Optional[int] = Field(default=None, alias="questionIndex")
+    query: Optional[str] = None
+    winner: str = "tie"
+    presented_first: str = Field(default="a", alias="presentedFirst")
+    flipped: bool = False
+    justification: Optional[str] = None
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseComparison(BaseModel):
+    """A full head-to-head between two runs of the same dataset.
+
+    ``skipped`` names the cases that could not be judged (one side produced no answer, or the
+    batch hit the server's per-comparison cap) with the reason - a comparison that quietly
+    dropped half the dataset would read as a clean sweep."""
+
+    batch_id: str = Field(alias="batchId")
+    run_a_id: str = Field(alias="runAId")
+    run_b_id: str = Field(alias="runBId")
+    both_orders: bool = Field(default=False, alias="bothOrders")
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+    summary: PairwiseSummary = Field(default_factory=PairwiseSummary)
+    cases: List[PairwiseCase] = Field(default_factory=list)
+    skipped: List[Dict[str, Any]] = Field(default_factory=list)
+    created_at: Optional[str] = Field(default=None, alias="createdAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"

{agentx_python-0.6.38 → agentx_python-0.8.1}/agentx/evaluations/runner.py

@@ -446,6 +446,20 @@ class EvaluationsRunner:
         EvaluationsClient.simulate_conversation for parameters."""
         return self._client.simulate_conversation(**kwargs)
 
+    def compare_pairwise(self, run_a_id: str, run_b_id: str, **kwargs):
+        """Head-to-head judging between two runs of the same dataset - "which answer is better"
+        rather than two absolute scores. See EvaluationsClient.compare_pairwise for the options,
+        and agentx.testing.assert_pairwise for the pytest-side check."""
+        return self._client.compare_pairwise(run_a_id, run_b_id, **kwargs)
+
+    def get_pairwise(self, batch_id: str):
+        """Read back a stored head-to-head by its batch id."""
+        return self._client.get_pairwise(batch_id)
+
+    def list_pairwise(self, **kwargs) -> list:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        return self._client.list_pairwise(**kwargs)
+
     def get_run(self, run_id: str) -> dict:
         """Run summary + per-result rows by id, without needing the EvaluationRunContext that
         created it (e.g. from a separate process)."""

agentx_python-0.8.1/agentx/testing.py

@@ -0,0 +1,165 @@
+"""pytest-friendly assertions over evaluation runs.
+
+The DeepEval-style dev loop (``assert_test`` inside a pytest suite) on top of AgentX's existing
+run + CI-gate primitives: run the evaluation however you like, then make the test fail with a
+readable verdict when quality drops. No plugin registration needed - it's a plain function that
+raises ``AssertionError``, so it works in any test runner and any CI.
+
+Usage::
+
+    from agentx import AgentX
+    from agentx.testing import assert_evaluation
+
+    def test_support_agent_quality():
+        client = AgentX.from_env()
+        report = (
+            client.evaluations
+            .run(dataset_id=DATASET_ID, scorer_id=SCORER_ID, subject=SUBJECT)
+            .execute(my_agent)
+            .finalize()
+        )
+        assert_evaluation(report, min_rating=7.0, no_regression=True)
+
+The check rides the engine's CI gate, so every pytest verdict is also recorded in the
+dashboard's gate history (CI Gates tab) with ``caller="pytest"`` - a red test and the
+dashboard's gate row are the same event, not two systems drifting apart.
+"""
+
+from typing import Any, List, Optional
+
+
+class EvaluationAssertionError(AssertionError):
+    """Raised when an evaluation run fails its quality checks.
+
+    Subclasses ``AssertionError`` so pytest renders it as a plain test failure; carries the
+    ``gate`` result for programmatic inspection in test hooks.
+    """
+
+    def __init__(self, message: str, gate: Any = None):
+        super().__init__(message)
+        self.gate = gate
+
+
+def _format_failures(gate: Any) -> str:
+    lines: List[str] = []
+    checks = getattr(gate, "checks", None) or []
+    for check in checks:
+        get = check.get if isinstance(check, dict) else lambda k, d=None: getattr(check, k, d)
+        status = "PASS" if get("passed") else "FAIL"
+        lines.append(f"  [{status}] {get('name', 'check')}: {get('detail', '')}")
+    average = getattr(gate, "average_rating", None)
+    if average is not None:
+        lines.append(f"  average rating: {average}")
+    return "\n".join(lines) if lines else f"  gate: {gate!r}"
+
+
+def assert_evaluation(
+    report: Any,
+    *,
+    min_rating: Optional[float] = None,
+    no_regression: bool = False,
+    tolerance: Optional[float] = None,
+    caller: str = "pytest",
+) -> Any:
+    """Assert a finalized evaluation run meets its quality floor.
+
+    ``report`` is the finalized :class:`~agentx.evaluations.runner.EvaluationRunContext`
+    returned by ``.execute(...).finalize()`` (or any object exposing the same ``.gate()``).
+
+    - ``min_rating`` - fail when the run's average judge rating is below this floor (0-10).
+    - ``no_regression`` - fail when the average dropped more than ``tolerance`` (default 0.5;
+      judge scores are noisy) below the dataset's previous completed run.
+
+    At least one check is required. Returns the ``GateResult`` on success; raises
+    :class:`EvaluationAssertionError` with a per-check verdict on failure.
+    """
+    if min_rating is None and not no_regression:
+        raise ValueError("assert_evaluation needs at least one check: min_rating and/or no_regression=True")
+    gate = report.gate(
+        fail_under=min_rating,
+        no_regression=no_regression,
+        tolerance=tolerance,
+        caller=caller,
+    )
+    if getattr(gate, "passed", False):
+        return gate
+    run_id = getattr(report, "run_id", None) or getattr(getattr(report, "_run", None), "run_id", "?")
+    raise EvaluationAssertionError(
+        f"Evaluation run {run_id} failed its quality gate:\n{_format_failures(gate)}",
+        gate=gate,
+    )
+
+
+def _format_pairwise(comparison: Any) -> str:
+    summary = getattr(comparison, "summary", None)
+    lines: List[str] = []
+    if summary is not None:
+        flip = getattr(summary, "flip_rate", None)
+        lines.append(
+            f"  A won {getattr(summary, 'a_wins', 0)}, B won {getattr(summary, 'b_wins', 0)}, "
+            f"{getattr(summary, 'ties', 0)} tied, out of {getattr(summary, 'total', 0)}"
+            + (f" (flip rate {flip})" if flip is not None else "")
+        )
+    for case in getattr(comparison, "cases", None) or []:
+        if getattr(case, "winner", None) == "b":
+            query = (getattr(case, "query", None) or "").strip()
+            lines.append(f"  lost: {query[:80]} - {(getattr(case, 'justification', None) or '')[:120]}")
+    return "\n".join(lines) if lines else f"  comparison: {comparison!r}"
+
+
+def assert_pairwise(
+    comparison: Any,
+    *,
+    must_win: bool = False,
+    max_losses: Optional[int] = None,
+    max_flip_rate: Optional[float] = None,
+) -> Any:
+    """Assert a head-to-head comparison went the candidate's way.
+
+    ``comparison`` is what ``client.evaluations.compare_pairwise(a, b)`` returns; run A is the
+    candidate and run B is the baseline it has to beat.
+
+    - ``must_win`` - fail unless A won more cases than B. A tie fails: "no worse than before" is
+      not the same claim as "better", and a change that cannot win its own comparison has not
+      earned a green test.
+    - ``max_losses`` - fail when A lost more than this many individual cases, even if it won
+      overall. This is the check that catches a change that lifts the average by improving easy
+      cases while breaking hard ones.
+    - ``max_flip_rate`` - fail when too many verdicts reversed with the presentation order. That
+      is position bias rather than quality, and it means the comparison itself is inconclusive,
+      so treating it as a pass would be worse than a red test. Only meaningful for a comparison
+      run with ``both_orders=True``; a comparison without it has no flip rate and this check is
+      skipped rather than quietly passing.
+
+    At least one check is required. Returns the comparison on success; raises
+    :class:`EvaluationAssertionError` naming the cases that lost.
+    """
+    if not must_win and max_losses is None and max_flip_rate is None:
+        raise ValueError(
+            "assert_pairwise needs at least one check: must_win, max_losses, and/or max_flip_rate"
+        )
+    summary = getattr(comparison, "summary", None)
+    if summary is None:
+        raise ValueError("assert_pairwise expects the result of compare_pairwise()")
+
+    failures: List[str] = []
+    a_wins = getattr(summary, "a_wins", 0)
+    b_wins = getattr(summary, "b_wins", 0)
+    if must_win and a_wins <= b_wins:
+        failures.append(f"run A did not win ({a_wins} vs {b_wins})")
+    if max_losses is not None and b_wins > max_losses:
+        failures.append(f"run A lost {b_wins} cases, more than the {max_losses} allowed")
+    flip_rate = getattr(summary, "flip_rate", None)
+    if max_flip_rate is not None and flip_rate is not None and flip_rate > max_flip_rate:
+        failures.append(
+            f"verdicts flipped on {flip_rate:.0%} of cases with the presentation order, above the "
+            f"{max_flip_rate:.0%} allowed - this comparison is inconclusive, not a pass"
+        )
+
+    if not failures:
+        return comparison
+    batch_id = getattr(comparison, "batch_id", "?")
+    raise EvaluationAssertionError(
+        f"Head-to-head {batch_id} failed: {'; '.join(failures)}\n{_format_pairwise(comparison)}",
+        gate=comparison,
+    )

agentx_python-0.8.1/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.8.1"

{agentx_python-0.6.38 → agentx_python-0.8.1}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.38
+Version: 0.8.1
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.38 → agentx_python-0.8.1}/agentx_python.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ agentx/feedback.py
 agentx/outcomes.py
 agentx/projects.py
 agentx/py.typed
+agentx/testing.py
 agentx/traces.py
 agentx/util.py
 agentx/version.py
@@ -74,5 +75,7 @@ tests/test_docs_match_sdk.py
 tests/test_integration.py
 tests/test_integrations.py
 tests/test_judge_scorers.py
+tests/test_pairwise.py
 tests/test_selfhost_analysis_fallback.py
-tests/test_span_tree.py
+tests/test_span_tree.py
+tests/test_testing.py

{agentx_python-0.6.38 → agentx_python-0.8.1}/tests/test_integrations.py

@@ -640,22 +640,32 @@ def test_crewai_captures_real_per_task_timing_via_event_bus():
 
             return FakeCrewOutput(raw="final output", tasks_output=[output1, output2])
 
-    tracer = make_tracer()
+    # Boundary-mocked rather than make_tracer(): each task is a real child span now (9d45dd1
+    # replaced the synthetic performance_summary step list), and mocking tracer._send would
+    # bypass the very _send/_dispatch/child_span chain that builds them. Same idiom as
+    # tests/test_span_tree.py.
+    tracer = Tracer(ingest_client=MagicMock())
     observer = AgentXCrewObserver(tracer, name="my-crew")
 
     result = observer.kickoff(FakeCrew(), inputs={"topic": "AI"})
 
     assert result.raw == "final output"
-    tracer._send.assert_called_once()
-    _, kwargs = tracer._send.call_args
-    steps = kwargs["performance_summary"]["execution_steps"]
-    assert len(steps) == 2
-    assert steps[0]["name"] == "Research topic"
-    assert steps[1]["name"] == "Write summary"
-    # Real timing, not an even split — task 1 slept ~3x longer than task 2.
-    assert steps[0]["duration_ms"] > steps[1]["duration_ms"] * 1.5
-    assert steps[0]["output"] == "research done"
-    assert steps[1]["output"] == "summary done"
+    wires = [call.args[0] for call in tracer._client.enqueue.call_args_list]
+    assert len(wires) == 3
+    research, summary, root = wires
+    assert root["name"] == "my-crew"
+    assert root["output"] == "final output"
+
+    # Each task is its own child span under the crew's root, carrying its own real duration.
+    assert research["name"] == "Research topic"
+    assert summary["name"] == "Write summary"
+    assert research["parent_span_id"] == root["span_id"]
+    assert summary["parent_span_id"] == root["span_id"]
+    assert research["output"] == "research done"
+    assert summary["output"] == "summary done"
+    # The point of the test: real timing, not an even split — task 1 slept ~3x longer than
+    # task 2, and the old "divide latency evenly" approximation reported them identical.
+    assert research["latency_ms"] > summary["latency_ms"] * 1.5
 
 
 def test_crewai_falls_back_to_even_split_without_event_bus():
@@ -930,6 +940,10 @@ def test_llamaindex_llm_error_is_captured():
 def test_autogen_agent_run_traces_text_reply():
     """Drives a real AssistantAgent.run() via AutoGen's own ReplayChatCompletionClient (no network/API keys)."""
     pytest.importorskip("autogen_agentchat")
+    # autogen-ext is a separate distribution and is NOT part of the "autogen" extra -
+    # ReplayChatCompletionClient below is a test double that lives there, so guard it too
+    # or this fails with ModuleNotFoundError instead of skipping.
+    pytest.importorskip("autogen_ext")
     from autogen_agentchat.agents import AssistantAgent
     from autogen_ext.models.replay import ReplayChatCompletionClient
 
@@ -938,25 +952,33 @@ def test_autogen_agent_run_traces_text_reply():
     model_client = ReplayChatCompletionClient(["Hello from AutoGen!"])
     agent = AssistantAgent("assistant", model_client=model_client)
 
-    tracer = make_tracer()
+    # Boundary-mocked, not make_tracer(): the agent's turn is a real child span since
+    # 9d45dd1, and mocking tracer._send would bypass the chain that builds it.
+    tracer = Tracer(ingest_client=MagicMock())
     observer = AgentXAutoGenObserver(tracer, name="my-agent")
 
     result = asyncio.run(observer.run(agent, task="Say hello"))
 
     assert result.messages[-1].content == "Hello from AutoGen!"
-    tracer._send.assert_called_once()
-    _, kwargs = tracer._send.call_args
-    assert kwargs["input"] == "Say hello"
-    assert kwargs["output"] == "Hello from AutoGen!"
-    assert kwargs["input_tokens"] == 22
-    assert kwargs["output_tokens"] == 3
-    steps = kwargs["performance_summary"]["execution_steps"]
-    assert len(steps) == 1
-    assert steps[0]["output"] == "Hello from AutoGen!"
+    wires = [call.args[0] for call in tracer._client.enqueue.call_args_list]
+    assert len(wires) == 2
+    step, root = wires
+    assert root["input"] == "Say hello"
+    assert root["output"] == "Hello from AutoGen!"
+    assert root["input_tokens"] == 22
+    assert root["output_tokens"] == 3
+    # The turn itself, as its own child span rather than a performance_summary step.
+    assert step["name"] == "assistant"
+    assert step["output"] == "Hello from AutoGen!"
+    assert step["parent_span_id"] == root["span_id"]
 
 
 def test_autogen_agent_run_traces_tool_call():
     pytest.importorskip("autogen_agentchat")
+    # autogen-ext is a separate distribution and is NOT part of the "autogen" extra -
+    # ReplayChatCompletionClient below is a test double that lives there, so guard it too
+    # or this fails with ModuleNotFoundError instead of skipping.
+    pytest.importorskip("autogen_ext")
     import json
 
     from autogen_agentchat.agents import AssistantAgent
@@ -986,16 +1008,24 @@ def test_autogen_agent_run_traces_tool_call():
     )
     agent = AssistantAgent("assistant", model_client=model_client, tools=[tool])
 
-    tracer = make_tracer()
+    tracer = Tracer(ingest_client=MagicMock())
     observer = AgentXAutoGenObserver(tracer, name="my-agent")
 
     asyncio.run(observer.run(agent, task="What is the weather in NYC?"))
 
-    tracer._send.assert_called_once()
-    _, kwargs = tracer._send.call_args
-    perf = kwargs["performance_summary"]
-    assert len(perf["tool_calls"]) == 1
-    tool_call = perf["tool_calls"][0]
+    wires = [call.args[0] for call in tracer._client.enqueue.call_args_list]
+    assert len(wires) == 2
+    child, root = wires
+    # The tool call is a real child span...
+    assert child["name"] == "get_weather"
+    assert "NYC" in child["input"]
+    assert child["output"] == "sunny in NYC"
+    assert child["parent_span_id"] == root["span_id"]
+    # ...and is mirrored onto the ROOT's flat tool_calls, which is what the engine's built-in
+    # "Tool failure" check and the dashboard's Tool quality column read. Same deliberate
+    # dual-write as trace_tool_call() - see test_span_tree.py.
+    assert len(root["tool_calls"]) == 1
+    tool_call = root["tool_calls"][0]
     assert tool_call["name"] == "get_weather"
     assert "NYC" in tool_call["input"]
     assert tool_call["output"] == "sunny in NYC"

agentx_python-0.8.1/tests/test_pairwise.py

@@ -0,0 +1,119 @@
+"""Head-to-head judging: the client's request shape and the pytest assertion over the result."""
+
+import pytest
+
+from agentx.evaluations.models import PairwiseComparison
+from agentx.testing import EvaluationAssertionError, assert_pairwise
+
+
+def comparison(**over) -> PairwiseComparison:
+    payload = {
+        "batchId": "batch-1",
+        "runAId": "run-candidate",
+        "runBId": "run-baseline",
+        "bothOrders": False,
+        "judgeModel": "gpt-5.6-luna",
+        "summary": {"total": 3, "aWins": 2, "bWins": 1, "ties": 0, "winner": "a", "flipRate": None},
+        "cases": [
+            {
+                "questionIndex": 2,
+                "query": "Who pays return shipping?",
+                "winner": "b",
+                "presentedFirst": "a",
+                "justification": "Answer 2 names both cases explicitly.",
+            }
+        ],
+        "skipped": [],
+    }
+    payload.update(over)
+    return PairwiseComparison(**payload)
+
+
+class FakeClient:
+    """Captures the request instead of sending it - the wire shape is the contract with the
+    engine, and camelCase is the convention it has to keep."""
+
+    # The pairwise routes live on the /evaluate dialect, reached through this property.
+    _api_root = "https://engine.example/api/v1"
+
+    def __init__(self, response=None):
+        self.calls = []
+        self._response = response or {"comparison": comparison().model_dump(by_alias=True)}
+
+    def _request(self, method, path, **kwargs):
+        self.calls.append((method, path, kwargs))
+        return self._response
+
+
+def test_compare_pairwise_sends_camelcase_and_omits_unset_options():
+    from agentx.evaluations.client import EvaluationsClient
+
+    client = FakeClient()
+    result = EvaluationsClient.compare_pairwise(client, "run-candidate", "run-baseline")
+
+    method, path, kwargs = client.calls[0]
+    assert (method, path) == ("POST", "/evaluate/runs/pairwise")
+    # Defaults are the server's to choose; the SDK does not invent a criteria string or a
+    # judge model, and does not send bothOrders unless the caller asked for it.
+    assert kwargs["json"] == {"runAId": "run-candidate", "runBId": "run-baseline"}
+    assert result.summary.a_wins == 2
+    assert result.cases[0].presented_first == "a"
+
+
+def test_compare_pairwise_forwards_the_options_it_is_given():
+    from agentx.evaluations.client import EvaluationsClient
+
+    client = FakeClient()
+    EvaluationsClient.compare_pairwise(
+        client, "a", "b", criteria="Which is more concise?", judge_model="gpt-5.6-luna", both_orders=True
+    )
+    assert client.calls[0][2]["json"] == {
+        "runAId": "a",
+        "runBId": "b",
+        "criteria": "Which is more concise?",
+        "judgeModel": "gpt-5.6-luna",
+        "bothOrders": True,
+    }
+
+
+def test_assert_pairwise_passes_a_clear_win():
+    result = assert_pairwise(comparison(), must_win=True, max_losses=1)
+    assert result.summary.winner == "a"
+
+
+def test_a_tie_is_not_a_win():
+    tied = comparison(summary={"total": 2, "aWins": 1, "bWins": 1, "ties": 0, "winner": "tie", "flipRate": None})
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(tied, must_win=True)
+    assert "did not win" in str(excinfo.value)
+
+
+def test_max_losses_catches_a_win_that_broke_hard_cases():
+    # Wins overall, but lost more individual cases than the caller tolerates.
+    lossy = comparison(summary={"total": 10, "aWins": 5, "bWins": 4, "ties": 1, "winner": "a", "flipRate": None})
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(lossy, max_losses=2)
+    message = str(excinfo.value)
+    assert "lost 4 cases" in message
+    # The failure names the case that lost, so the test output is actionable on its own.
+    assert "Who pays return shipping?" in message
+
+
+def test_a_high_flip_rate_fails_instead_of_passing_on_position_bias():
+    biased = comparison(
+        bothOrders=True,
+        summary={"total": 4, "aWins": 3, "bWins": 1, "ties": 0, "winner": "a", "flipRate": 0.5},
+    )
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(biased, max_flip_rate=0.2)
+    assert "inconclusive" in str(excinfo.value)
+
+
+def test_flip_rate_check_is_skipped_when_both_orders_was_not_run():
+    # No flip rate exists to check, so this must not silently fail or silently pass a made-up 0.
+    assert_pairwise(comparison(), max_flip_rate=0.0)
+
+
+def test_requires_at_least_one_check():
+    with pytest.raises(ValueError):
+        assert_pairwise(comparison())

agentx_python-0.8.1/tests/test_testing.py

@@ -0,0 +1,49 @@
+"""agentx.testing.assert_evaluation - the pytest-native quality gate wrapper."""
+
+import pytest
+
+from agentx.evaluations.runner import GateResult
+from agentx.testing import EvaluationAssertionError, assert_evaluation
+
+
+class FakeReport:
+    def __init__(self, gate_payload):
+        self.run_id = "run-123"
+        self.gate_kwargs = None
+        self._payload = gate_payload
+
+    def gate(self, **kwargs):
+        self.gate_kwargs = kwargs
+        return GateResult(self._payload)
+
+
+def test_passing_gate_returns_result_and_forwards_checks():
+    report = FakeReport({"passed": True, "averageRating": 8.2, "checks": [{"name": "floor", "passed": True}]})
+    gate = assert_evaluation(report, min_rating=7.0, no_regression=True, tolerance=0.3)
+    assert gate.passed is True
+    assert report.gate_kwargs == {"fail_under": 7.0, "no_regression": True, "tolerance": 0.3, "caller": "pytest"}
+
+
+def test_failing_gate_raises_assertion_error_with_verdict():
+    report = FakeReport({
+        "passed": False,
+        "averageRating": 5.1,
+        "checks": [
+            {"name": "floor", "passed": False, "detail": "average 5.1 below fail_under 7"},
+            {"name": "regression", "passed": True, "detail": "no baseline"},
+        ],
+    })
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_evaluation(report, min_rating=7.0)
+    message = str(excinfo.value)
+    assert "run-123" in message
+    assert "[FAIL] floor" in message
+    assert "average 5.1 below fail_under 7" in message
+    # AssertionError subclass, so pytest treats it as a normal test failure.
+    assert isinstance(excinfo.value, AssertionError)
+    assert excinfo.value.gate.average_rating == 5.1
+
+
+def test_requires_at_least_one_check():
+    with pytest.raises(ValueError):
+        assert_evaluation(FakeReport({"passed": True}))

agentx_python-0.6.38/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.38"