agentx-python 0.6.39__tar.gz → 0.8.2__tar.gz

{agentx_python-0.6.39 → agentx_python-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.39
+Version: 0.8.2
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx/evaluations/client.py

@@ -17,6 +17,7 @@ from agentx.evaluations.models import (
     EvaluationSettings,
     EvaluationSubject,
     ModelInfo,
+    PairwiseComparison,
     Prompt,
     Report,
 )
@@ -541,6 +542,63 @@ class EvaluationsClient:
             "POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload
         )
 
+    # ------------------------------------------------------------------
+    # Head-to-head (pairwise) judging. Rides the /evaluate dialect via _api_root(), same
+    # precedent as get_report and the prompt loop above.
+    # ------------------------------------------------------------------
+
+    def compare_pairwise(
+        self,
+        run_a_id: str,
+        run_b_id: str,
+        *,
+        criteria: Optional[str] = None,
+        judge_model: Optional[str] = None,
+        both_orders: bool = False,
+    ) -> PairwiseComparison:
+        """Ask a judge which of two runs answered each question better.
+
+        Both runs must be of the same dataset. Absolute ratings answer "is this above the bar";
+        this answers "did the change help", which is the question a diff between two runs is
+        actually asking. The judge sees the two answers as "Answer 1"/"Answer 2" with the order
+        alternating case by case, so it never learns which run is the candidate.
+
+        ``both_orders=True`` judges every pair twice with the sides swapped. It doubles the judge
+        cost and is the only real defense against position bias: a pair whose winner reverses is
+        recorded as a tie, and the batch reports its ``flip_rate``.
+
+        ``criteria`` and ``judge_model`` default to the dataset's own evaluation criteria and
+        judge model, so a head-to-head grades on the same terms a normal run of it does.
+        """
+        payload: Dict[str, Any] = {"runAId": run_a_id, "runBId": run_b_id}
+        if criteria:
+            payload["criteria"] = criteria
+        if judge_model:
+            payload["judgeModel"] = judge_model
+        if both_orders:
+            payload["bothOrders"] = True
+        response = self._request("POST", "/evaluate/runs/pairwise", json=payload, base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def get_pairwise(self, batch_id: str) -> PairwiseComparison:
+        """Read back a stored head-to-head by its batch id."""
+        response = self._request("GET", f"/evaluate/runs/pairwise/{batch_id}", base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def list_pairwise(
+        self, *, run_a_id: Optional[str] = None, run_b_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        params: Dict[str, Any] = {}
+        if run_a_id:
+            params["runAId"] = run_a_id
+        if run_b_id:
+            params["runBId"] = run_b_id
+        response = self._request(
+            "GET", "/evaluate/runs/pairwise", params=params or None, base=self._api_root
+        )
+        return response.get("comparisons", [])
+
     # ------------------------------------------------------------------
     # Tool schema registry (same version-scoped propose/publish loop as prompts)
     # ------------------------------------------------------------------

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx/evaluations/models.py

@@ -742,3 +742,64 @@ class AnalysisStatus(BaseModel):
     @property
     def is_terminal(self) -> bool:
         return self.status in ("completed", "partially_failed", "failed")
+
+
+class PairwiseSummary(BaseModel):
+    """Batch-level verdict of a head-to-head comparison (``compare_pairwise``).
+
+    ``winner`` is "a", "b", or "tie" - a dead heat is reported as a tie rather than broken
+    arbitrarily. ``flip_rate`` is only populated for a ``both_orders=True`` comparison: it is the
+    share of cases whose winner reversed when the two answers were swapped, which is position
+    bias rather than quality. A high flip rate means the batch is inconclusive, so it is reported
+    instead of being folded away."""
+
+    total: int = 0
+    a_wins: int = Field(default=0, alias="aWins")
+    b_wins: int = Field(default=0, alias="bWins")
+    ties: int = 0
+    winner: str = "tie"
+    flip_rate: Optional[float] = Field(default=None, alias="flipRate")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseCase(BaseModel):
+    """One question's verdict. ``presented_first`` records which run's answer the judge read
+    first, because that is the confound pairwise judging exists to control for."""
+
+    id: Optional[str] = Field(default=None, alias="_id")
+    question_index: Optional[int] = Field(default=None, alias="questionIndex")
+    query: Optional[str] = None
+    winner: str = "tie"
+    presented_first: str = Field(default="a", alias="presentedFirst")
+    flipped: bool = False
+    justification: Optional[str] = None
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseComparison(BaseModel):
+    """A full head-to-head between two runs of the same dataset.
+
+    ``skipped`` names the cases that could not be judged (one side produced no answer, or the
+    batch hit the server's per-comparison cap) with the reason - a comparison that quietly
+    dropped half the dataset would read as a clean sweep."""
+
+    batch_id: str = Field(alias="batchId")
+    run_a_id: str = Field(alias="runAId")
+    run_b_id: str = Field(alias="runBId")
+    both_orders: bool = Field(default=False, alias="bothOrders")
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+    summary: PairwiseSummary = Field(default_factory=PairwiseSummary)
+    cases: List[PairwiseCase] = Field(default_factory=list)
+    skipped: List[Dict[str, Any]] = Field(default_factory=list)
+    created_at: Optional[str] = Field(default=None, alias="createdAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx/evaluations/runner.py

@@ -446,6 +446,20 @@ class EvaluationsRunner:
         EvaluationsClient.simulate_conversation for parameters."""
         return self._client.simulate_conversation(**kwargs)
 
+    def compare_pairwise(self, run_a_id: str, run_b_id: str, **kwargs):
+        """Head-to-head judging between two runs of the same dataset - "which answer is better"
+        rather than two absolute scores. See EvaluationsClient.compare_pairwise for the options,
+        and agentx.testing.assert_pairwise for the pytest-side check."""
+        return self._client.compare_pairwise(run_a_id, run_b_id, **kwargs)
+
+    def get_pairwise(self, batch_id: str):
+        """Read back a stored head-to-head by its batch id."""
+        return self._client.get_pairwise(batch_id)
+
+    def list_pairwise(self, **kwargs) -> list:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        return self._client.list_pairwise(**kwargs)
+
     def get_run(self, run_id: str) -> dict:
         """Run summary + per-result rows by id, without needing the EvaluationRunContext that
         created it (e.g. from a separate process)."""

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx/integrations/langchain.py

@@ -455,6 +455,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 input=node.get("input"),
                 output=node.get("output"),
                 error=node.get("error"),
+                span_kind="chain",
             )
 
         llm_count = 0
@@ -470,6 +471,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 model=step.get("model"),
                 input_tokens=step.get("inputTokenSize"),
                 output_tokens=step.get("outputTokenSize"),
+                span_kind="llm",
             )
         for tc in tool_calls:
             resolve_parent(tc.get("parent_run_id")).child_span(
@@ -480,6 +482,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 input=tc.get("input"),
                 output=tc.get("output"),
                 error=None if tc.get("success", True) else str(tc.get("output") or "Tool call failed"),
+                span_kind="tool",
             )
         for step in state.get("retrieval_steps", []):
             resolve_parent(step.get("parent_run_id")).child_span(
@@ -490,6 +493,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 input=step.get("query"),
                 output=step.get("output"),
                 metadata={"kind": "retrieval"},
+                span_kind="retrieval",
             )
 
     def on_chain_end(

agentx_python-0.8.2/agentx/testing.py

@@ -0,0 +1,165 @@
+"""pytest-friendly assertions over evaluation runs.
+
+The DeepEval-style dev loop (``assert_test`` inside a pytest suite) on top of AgentX's existing
+run + CI-gate primitives: run the evaluation however you like, then make the test fail with a
+readable verdict when quality drops. No plugin registration needed - it's a plain function that
+raises ``AssertionError``, so it works in any test runner and any CI.
+
+Usage::
+
+    from agentx import AgentX
+    from agentx.testing import assert_evaluation
+
+    def test_support_agent_quality():
+        client = AgentX.from_env()
+        report = (
+            client.evaluations
+            .run(dataset_id=DATASET_ID, scorer_id=SCORER_ID, subject=SUBJECT)
+            .execute(my_agent)
+            .finalize()
+        )
+        assert_evaluation(report, min_rating=7.0, no_regression=True)
+
+The check rides the engine's CI gate, so every pytest verdict is also recorded in the
+dashboard's gate history (CI Gates tab) with ``caller="pytest"`` - a red test and the
+dashboard's gate row are the same event, not two systems drifting apart.
+"""
+
+from typing import Any, List, Optional
+
+
+class EvaluationAssertionError(AssertionError):
+    """Raised when an evaluation run fails its quality checks.
+
+    Subclasses ``AssertionError`` so pytest renders it as a plain test failure; carries the
+    ``gate`` result for programmatic inspection in test hooks.
+    """
+
+    def __init__(self, message: str, gate: Any = None):
+        super().__init__(message)
+        self.gate = gate
+
+
+def _format_failures(gate: Any) -> str:
+    lines: List[str] = []
+    checks = getattr(gate, "checks", None) or []
+    for check in checks:
+        get = check.get if isinstance(check, dict) else lambda k, d=None: getattr(check, k, d)
+        status = "PASS" if get("passed") else "FAIL"
+        lines.append(f"  [{status}] {get('name', 'check')}: {get('detail', '')}")
+    average = getattr(gate, "average_rating", None)
+    if average is not None:
+        lines.append(f"  average rating: {average}")
+    return "\n".join(lines) if lines else f"  gate: {gate!r}"
+
+
+def assert_evaluation(
+    report: Any,
+    *,
+    min_rating: Optional[float] = None,
+    no_regression: bool = False,
+    tolerance: Optional[float] = None,
+    caller: str = "pytest",
+) -> Any:
+    """Assert a finalized evaluation run meets its quality floor.
+
+    ``report`` is the finalized :class:`~agentx.evaluations.runner.EvaluationRunContext`
+    returned by ``.execute(...).finalize()`` (or any object exposing the same ``.gate()``).
+
+    - ``min_rating`` - fail when the run's average judge rating is below this floor (0-10).
+    - ``no_regression`` - fail when the average dropped more than ``tolerance`` (default 0.5;
+      judge scores are noisy) below the dataset's previous completed run.
+
+    At least one check is required. Returns the ``GateResult`` on success; raises
+    :class:`EvaluationAssertionError` with a per-check verdict on failure.
+    """
+    if min_rating is None and not no_regression:
+        raise ValueError("assert_evaluation needs at least one check: min_rating and/or no_regression=True")
+    gate = report.gate(
+        fail_under=min_rating,
+        no_regression=no_regression,
+        tolerance=tolerance,
+        caller=caller,
+    )
+    if getattr(gate, "passed", False):
+        return gate
+    run_id = getattr(report, "run_id", None) or getattr(getattr(report, "_run", None), "run_id", "?")
+    raise EvaluationAssertionError(
+        f"Evaluation run {run_id} failed its quality gate:\n{_format_failures(gate)}",
+        gate=gate,
+    )
+
+
+def _format_pairwise(comparison: Any) -> str:
+    summary = getattr(comparison, "summary", None)
+    lines: List[str] = []
+    if summary is not None:
+        flip = getattr(summary, "flip_rate", None)
+        lines.append(
+            f"  A won {getattr(summary, 'a_wins', 0)}, B won {getattr(summary, 'b_wins', 0)}, "
+            f"{getattr(summary, 'ties', 0)} tied, out of {getattr(summary, 'total', 0)}"
+            + (f" (flip rate {flip})" if flip is not None else "")
+        )
+    for case in getattr(comparison, "cases", None) or []:
+        if getattr(case, "winner", None) == "b":
+            query = (getattr(case, "query", None) or "").strip()
+            lines.append(f"  lost: {query[:80]} - {(getattr(case, 'justification', None) or '')[:120]}")
+    return "\n".join(lines) if lines else f"  comparison: {comparison!r}"
+
+
+def assert_pairwise(
+    comparison: Any,
+    *,
+    must_win: bool = False,
+    max_losses: Optional[int] = None,
+    max_flip_rate: Optional[float] = None,
+) -> Any:
+    """Assert a head-to-head comparison went the candidate's way.
+
+    ``comparison`` is what ``client.evaluations.compare_pairwise(a, b)`` returns; run A is the
+    candidate and run B is the baseline it has to beat.
+
+    - ``must_win`` - fail unless A won more cases than B. A tie fails: "no worse than before" is
+      not the same claim as "better", and a change that cannot win its own comparison has not
+      earned a green test.
+    - ``max_losses`` - fail when A lost more than this many individual cases, even if it won
+      overall. This is the check that catches a change that lifts the average by improving easy
+      cases while breaking hard ones.
+    - ``max_flip_rate`` - fail when too many verdicts reversed with the presentation order. That
+      is position bias rather than quality, and it means the comparison itself is inconclusive,
+      so treating it as a pass would be worse than a red test. Only meaningful for a comparison
+      run with ``both_orders=True``; a comparison without it has no flip rate and this check is
+      skipped rather than quietly passing.
+
+    At least one check is required. Returns the comparison on success; raises
+    :class:`EvaluationAssertionError` naming the cases that lost.
+    """
+    if not must_win and max_losses is None and max_flip_rate is None:
+        raise ValueError(
+            "assert_pairwise needs at least one check: must_win, max_losses, and/or max_flip_rate"
+        )
+    summary = getattr(comparison, "summary", None)
+    if summary is None:
+        raise ValueError("assert_pairwise expects the result of compare_pairwise()")
+
+    failures: List[str] = []
+    a_wins = getattr(summary, "a_wins", 0)
+    b_wins = getattr(summary, "b_wins", 0)
+    if must_win and a_wins <= b_wins:
+        failures.append(f"run A did not win ({a_wins} vs {b_wins})")
+    if max_losses is not None and b_wins > max_losses:
+        failures.append(f"run A lost {b_wins} cases, more than the {max_losses} allowed")
+    flip_rate = getattr(summary, "flip_rate", None)
+    if max_flip_rate is not None and flip_rate is not None and flip_rate > max_flip_rate:
+        failures.append(
+            f"verdicts flipped on {flip_rate:.0%} of cases with the presentation order, above the "
+            f"{max_flip_rate:.0%} allowed - this comparison is inconclusive, not a pass"
+        )
+
+    if not failures:
+        return comparison
+    batch_id = getattr(comparison, "batch_id", "?")
+    raise EvaluationAssertionError(
+        f"Head-to-head {batch_id} failed: {'; '.join(failures)}\n{_format_pairwise(comparison)}",
+        gate=comparison,
+    )

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx/tracing/tracer.py

@@ -71,6 +71,7 @@ class _TraceSpan:
         monitor: bool = False,
         pattern_ids: Optional[List[str]] = None,
         agent_id: Optional[str] = None,
+        span_kind: Optional[str] = None,
     ) -> None:
         self._tracer = tracer
         self.name = name
@@ -84,6 +85,10 @@ class _TraceSpan:
         # from a prior GET /agents lookup) to pin this trace to that exact agent. None (the
         # default) resolves from `name` alone server-side, one stable agent per distinct name.
         self._agent_id = agent_id
+        # What kind of step this span is ("agent", "llm", ...), stated rather than left to the
+        # backend's fallback ladder. Optional: a root that says nothing still classifies the way
+        # it always did, which for the common flat trace (root carries the model) is "llm".
+        self._span_kind = span_kind
         # When True, __exit__ sends synchronously (blocking) instead of enqueueing, so trace_id
         # is populated by the time the `with` block exits - see Tracer.trace()'s sync param.
         self._sync = sync
@@ -183,6 +188,7 @@ class _TraceSpan:
             cache_write_tokens=self._cache_write_tokens or None,
             span_id=self._span_id,
             parent_span_id=self._parent_span_id,
+            span_kind=self._span_kind,
             started_at_unix_nano=str(int(self._start * 1_000_000_000)) if self._start else None,
         )
         return False  # never suppress exceptions
@@ -262,6 +268,7 @@ class _TraceSpan:
         error: Optional[str] = None,
         tool_calls: Optional[List[Dict[str, Any]]] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        span_kind: Optional[str] = None,
     ) -> "_TraceSpan":
         """
         Send one real child-span row parented to this span, with explicit timing (the caller's
@@ -318,6 +325,11 @@ class _TraceSpan:
             wire["tool_calls"] = child.tool_calls
         if metadata:
             wire["metadata"] = _safe_serialize(metadata)
+        # What kind of step this is, stated rather than left for the backend to guess from the
+        # span's name and which columns happen to be null. Same idea as LangSmith's run_type and
+        # Langfuse's observation type; the engine folds other vocabularies onto its own.
+        if span_kind:
+            wire["span_kind"] = span_kind
         if child._session_id:
             wire["session_id"] = child._session_id
         wire["span_id"] = child._span_id
@@ -383,6 +395,9 @@ class _TraceSpan:
                     output_tokens=step.get("outputTokenSize"),
                     cache_read_tokens=step.get("cacheReadTokenSize"),
                     cache_write_tokens=step.get("cacheWriteTokenSize"),
+                    # Stated, so a step named anything other than "LLM Call N" still classifies -
+                    # the backend's name regex was the only thing holding this together.
+                    span_kind="llm",
                 )
             for tc in tool_calls or []:
                 # Some callers' tool_calls dicts (e.g. langchain.py's, which sets these on the
@@ -399,6 +414,7 @@ class _TraceSpan:
                         input=tc.get("input"),
                         output=tc.get("output"),
                         error=None if tc.get("success", True) else str(tc.get("output") or "Tool call failed"),
+                        span_kind="tool",
                     )
                 # Also mirror onto this span's own flat tool_calls list, sent in this span's own
                 # wire payload on __exit__ (see tool_calls=self.tool_calls or None below). The
@@ -424,6 +440,7 @@ class _TraceSpan:
                     input=step.get("query"),
                     output=step.get("output"),
                     metadata={"kind": "retrieval"},
+                    span_kind="retrieval",
                 )
 
             if self.input is None and input is not None:
@@ -665,6 +682,7 @@ class Tracer:
                 input=input,
                 output=output,
                 error=error,
+                span_kind="tool",
             )
             # The child span above is only for the trace detail's span tree - the engine's
             # built-in "Tool failure" check and the dashboard's Tool quality column read the
@@ -781,6 +799,7 @@ class Tracer:
             input=query,
             output=output,
             metadata={"kind": "retrieval"},
+            span_kind="retrieval",
         )
 
     @contextmanager
@@ -823,6 +842,7 @@ class Tracer:
         monitor: Optional[bool] = None,
         pattern_ids: Optional[List[str]] = None,
         agent_id: Optional[str] = None,
+        span_kind: Optional[str] = None,
     ) -> _TraceSpan:
         """
         Return a :class:`_TraceSpan` that works as both a decorator and a
@@ -888,6 +908,7 @@ class Tracer:
             monitor=monitor,
             pattern_ids=pattern_ids,
             agent_id=agent_id,
+            span_kind=span_kind,
         )
 
     def flush(self, timeout: float = 5.0) -> bool:
@@ -1127,6 +1148,8 @@ class Tracer:
             wire["started_at_unix_nano"] = payload["started_at_unix_nano"]
         if "agent_id" in payload:
             wire["agent_id"] = payload["agent_id"]
+        if "span_kind" in payload:
+            wire["span_kind"] = payload["span_kind"]
 
         pending_tool_calls, self._pending_tool_calls = self._pending_tool_calls, []
         if pending_tool_calls:

agentx_python-0.8.2/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.8.2"

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.39
+Version: 0.8.2
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.39 → agentx_python-0.8.2}/agentx_python.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ agentx/feedback.py
 agentx/outcomes.py
 agentx/projects.py
 agentx/py.typed
+agentx/testing.py
 agentx/traces.py
 agentx/util.py
 agentx/version.py
@@ -74,5 +75,7 @@ tests/test_docs_match_sdk.py
 tests/test_integration.py
 tests/test_integrations.py
 tests/test_judge_scorers.py
+tests/test_pairwise.py
 tests/test_selfhost_analysis_fallback.py
-tests/test_span_tree.py
+tests/test_span_tree.py
+tests/test_testing.py

agentx_python-0.8.2/tests/test_pairwise.py

@@ -0,0 +1,119 @@
+"""Head-to-head judging: the client's request shape and the pytest assertion over the result."""
+
+import pytest
+
+from agentx.evaluations.models import PairwiseComparison
+from agentx.testing import EvaluationAssertionError, assert_pairwise
+
+
+def comparison(**over) -> PairwiseComparison:
+    payload = {
+        "batchId": "batch-1",
+        "runAId": "run-candidate",
+        "runBId": "run-baseline",
+        "bothOrders": False,
+        "judgeModel": "gpt-5.6-luna",
+        "summary": {"total": 3, "aWins": 2, "bWins": 1, "ties": 0, "winner": "a", "flipRate": None},
+        "cases": [
+            {
+                "questionIndex": 2,
+                "query": "Who pays return shipping?",
+                "winner": "b",
+                "presentedFirst": "a",
+                "justification": "Answer 2 names both cases explicitly.",
+            }
+        ],
+        "skipped": [],
+    }
+    payload.update(over)
+    return PairwiseComparison(**payload)
+
+
+class FakeClient:
+    """Captures the request instead of sending it - the wire shape is the contract with the
+    engine, and camelCase is the convention it has to keep."""
+
+    # The pairwise routes live on the /evaluate dialect, reached through this property.
+    _api_root = "https://engine.example/api/v1"
+
+    def __init__(self, response=None):
+        self.calls = []
+        self._response = response or {"comparison": comparison().model_dump(by_alias=True)}
+
+    def _request(self, method, path, **kwargs):
+        self.calls.append((method, path, kwargs))
+        return self._response
+
+
+def test_compare_pairwise_sends_camelcase_and_omits_unset_options():
+    from agentx.evaluations.client import EvaluationsClient
+
+    client = FakeClient()
+    result = EvaluationsClient.compare_pairwise(client, "run-candidate", "run-baseline")
+
+    method, path, kwargs = client.calls[0]
+    assert (method, path) == ("POST", "/evaluate/runs/pairwise")
+    # Defaults are the server's to choose; the SDK does not invent a criteria string or a
+    # judge model, and does not send bothOrders unless the caller asked for it.
+    assert kwargs["json"] == {"runAId": "run-candidate", "runBId": "run-baseline"}
+    assert result.summary.a_wins == 2
+    assert result.cases[0].presented_first == "a"
+
+
+def test_compare_pairwise_forwards_the_options_it_is_given():
+    from agentx.evaluations.client import EvaluationsClient
+
+    client = FakeClient()
+    EvaluationsClient.compare_pairwise(
+        client, "a", "b", criteria="Which is more concise?", judge_model="gpt-5.6-luna", both_orders=True
+    )
+    assert client.calls[0][2]["json"] == {
+        "runAId": "a",
+        "runBId": "b",
+        "criteria": "Which is more concise?",
+        "judgeModel": "gpt-5.6-luna",
+        "bothOrders": True,
+    }
+
+
+def test_assert_pairwise_passes_a_clear_win():
+    result = assert_pairwise(comparison(), must_win=True, max_losses=1)
+    assert result.summary.winner == "a"
+
+
+def test_a_tie_is_not_a_win():
+    tied = comparison(summary={"total": 2, "aWins": 1, "bWins": 1, "ties": 0, "winner": "tie", "flipRate": None})
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(tied, must_win=True)
+    assert "did not win" in str(excinfo.value)
+
+
+def test_max_losses_catches_a_win_that_broke_hard_cases():
+    # Wins overall, but lost more individual cases than the caller tolerates.
+    lossy = comparison(summary={"total": 10, "aWins": 5, "bWins": 4, "ties": 1, "winner": "a", "flipRate": None})
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(lossy, max_losses=2)
+    message = str(excinfo.value)
+    assert "lost 4 cases" in message
+    # The failure names the case that lost, so the test output is actionable on its own.
+    assert "Who pays return shipping?" in message
+
+
+def test_a_high_flip_rate_fails_instead_of_passing_on_position_bias():
+    biased = comparison(
+        bothOrders=True,
+        summary={"total": 4, "aWins": 3, "bWins": 1, "ties": 0, "winner": "a", "flipRate": 0.5},
+    )
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_pairwise(biased, max_flip_rate=0.2)
+    assert "inconclusive" in str(excinfo.value)
+
+
+def test_flip_rate_check_is_skipped_when_both_orders_was_not_run():
+    # No flip rate exists to check, so this must not silently fail or silently pass a made-up 0.
+    assert_pairwise(comparison(), max_flip_rate=0.0)
+
+
+def test_requires_at_least_one_check():
+    with pytest.raises(ValueError):
+        assert_pairwise(comparison())

agentx_python-0.8.2/tests/test_testing.py

@@ -0,0 +1,49 @@
+"""agentx.testing.assert_evaluation - the pytest-native quality gate wrapper."""
+
+import pytest
+
+from agentx.evaluations.runner import GateResult
+from agentx.testing import EvaluationAssertionError, assert_evaluation
+
+
+class FakeReport:
+    def __init__(self, gate_payload):
+        self.run_id = "run-123"
+        self.gate_kwargs = None
+        self._payload = gate_payload
+
+    def gate(self, **kwargs):
+        self.gate_kwargs = kwargs
+        return GateResult(self._payload)
+
+
+def test_passing_gate_returns_result_and_forwards_checks():
+    report = FakeReport({"passed": True, "averageRating": 8.2, "checks": [{"name": "floor", "passed": True}]})
+    gate = assert_evaluation(report, min_rating=7.0, no_regression=True, tolerance=0.3)
+    assert gate.passed is True
+    assert report.gate_kwargs == {"fail_under": 7.0, "no_regression": True, "tolerance": 0.3, "caller": "pytest"}
+
+
+def test_failing_gate_raises_assertion_error_with_verdict():
+    report = FakeReport({
+        "passed": False,
+        "averageRating": 5.1,
+        "checks": [
+            {"name": "floor", "passed": False, "detail": "average 5.1 below fail_under 7"},
+            {"name": "regression", "passed": True, "detail": "no baseline"},
+        ],
+    })
+    with pytest.raises(EvaluationAssertionError) as excinfo:
+        assert_evaluation(report, min_rating=7.0)
+    message = str(excinfo.value)
+    assert "run-123" in message
+    assert "[FAIL] floor" in message
+    assert "average 5.1 below fail_under 7" in message
+    # AssertionError subclass, so pytest treats it as a normal test failure.
+    assert isinstance(excinfo.value, AssertionError)
+    assert excinfo.value.gate.average_rating == 5.1
+
+
+def test_requires_at_least_one_check():
+    with pytest.raises(ValueError):
+        assert_evaluation(FakeReport({"passed": True}))

agentx_python-0.6.39/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.39"