agentx-python 0.4.9__py3-none-any.whl → 0.4.11__py3-none-any.whl

agentx/agentx.py

@@ -23,6 +23,7 @@ class AgentX:
         from agentx.evaluations.client import EvaluationsClient
         from agentx.evaluations.runner import EvaluationsRunner
         from agentx.version import VERSION
+
         _eval_client = EvaluationsClient(
             api_key=self.api_key,
             sdk_version=VERSION,

agentx/evaluations/_term.py

@@ -1,4 +1,5 @@
 """Minimal ANSI terminal helpers — no external dependencies."""
+
 from __future__ import annotations
 
 import itertools
@@ -13,23 +14,42 @@ def _c(code: str) -> str:
     return f"\033[{code}m" if _IS_TTY else ""
 
 
-RESET  = _c("0")
-BOLD   = _c("1")
-DIM    = _c("2")
-GREEN  = _c("32")
+RESET = _c("0")
+BOLD = _c("1")
+DIM = _c("2")
+GREEN = _c("32")
 YELLOW = _c("33")
-RED    = _c("31")
-CYAN   = _c("36")
+RED = _c("31")
+CYAN = _c("36")
 MAGENTA = _c("35")
 
 
-def green(s: str) -> str:   return f"{GREEN}{s}{RESET}"
-def yellow(s: str) -> str:  return f"{YELLOW}{s}{RESET}"
-def red(s: str) -> str:     return f"{RED}{s}{RESET}"
-def cyan(s: str) -> str:    return f"{CYAN}{s}{RESET}"
-def bold(s: str) -> str:    return f"{BOLD}{s}{RESET}"
-def dim(s: str) -> str:     return f"{DIM}{s}{RESET}"
-def magenta(s: str) -> str: return f"{MAGENTA}{s}{RESET}"
+def green(s: str) -> str:
+    return f"{GREEN}{s}{RESET}"
+
+
+def yellow(s: str) -> str:
+    return f"{YELLOW}{s}{RESET}"
+
+
+def red(s: str) -> str:
+    return f"{RED}{s}{RESET}"
+
+
+def cyan(s: str) -> str:
+    return f"{CYAN}{s}{RESET}"
+
+
+def bold(s: str) -> str:
+    return f"{BOLD}{s}{RESET}"
+
+
+def dim(s: str) -> str:
+    return f"{DIM}{s}{RESET}"
+
+
+def magenta(s: str) -> str:
+    return f"{MAGENTA}{s}{RESET}"
 
 
 class Spinner:
@@ -64,6 +84,8 @@ class Spinner:
         for frame in itertools.cycle(self._FRAMES):
             if self._stop.is_set():
                 break
-            sys.stdout.write(f"\r  {CYAN}{frame}{RESET}  {self._message}  {DIM}(this may take ~60s+){RESET}")
+            sys.stdout.write(
+                f"\r  {CYAN}{frame}{RESET}  {self._message}  {DIM}(this may take ~60s+){RESET}"
+            )
             sys.stdout.flush()
             time.sleep(0.08)

agentx/evaluations/adapters/precomputed.py

@@ -27,7 +27,9 @@ class PrecomputedAdapter:
             self._lookup = {str(k): v for k, v in outputs.items()}
 
     def run(self, case: EvaluationCase) -> EvaluationResult:
-        raw = self._lookup.get(case.case_id) or self._lookup.get(str(case.question_index))
+        raw = self._lookup.get(case.case_id) or self._lookup.get(
+            str(case.question_index)
+        )
         if raw is None:
             raw = ""
         return normalize_result(case, raw, latency_ms=0)

agentx/evaluations/adapters/raw.py

@@ -23,8 +23,11 @@ class RawCallableAdapter:
     def __init__(self, fn: Callable[[EvaluationCase], Any]):
         self._fn = fn
 
-    def run(self, case: EvaluationCase, latency_ms: int | None = None) -> EvaluationResult:
+    def run(
+        self, case: EvaluationCase, latency_ms: int | None = None
+    ) -> EvaluationResult:
         import time
+
         start = time.monotonic()
         try:
             raw = self._fn(case)

agentx/evaluations/client.py

@@ -14,6 +14,7 @@ from agentx.evaluations.models import (
     EvaluationResult,
     EvaluationRun,
     EvaluationSubject,
+    ModelInfo,
     Report,
 )
 
@@ -42,26 +43,33 @@ class AgentXValidationError(AgentXEvaluationsError):
 
 
 class EvaluationsClient:
-    def __init__(self, api_key: str, sdk_version: str = "unknown", base_url: str = None):
+    def __init__(
+        self, api_key: str, sdk_version: str = "unknown", base_url: str = None
+    ):
         if not api_key:
             raise AgentXAuthError("AGENTX_API_KEY is required")
         self._api_key = api_key
         self._sdk_version = sdk_version
         # Priority: constructor arg > env var > SDK default
         # Always append /custom-agent-evaluations so users only need to provide /api/v1
-        _api_base = (base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE)).rstrip("/")
+        _api_base = (
+            base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE)
+        ).rstrip("/")
         if not _api_base.endswith("/custom-agent-evaluations"):
             _api_base = f"{_api_base}/custom-agent-evaluations"
         self._base_url = _api_base
         self._session = requests.Session()
-        self._session.headers.update({
-            "x-api-key": self._api_key,
-            "Content-Type": "application/json",
-            "User-Agent": f"{SDK_NAME}/{self._sdk_version}",
-            "accept": "*/*",
-        })
+        self._session.headers.update(
+            {
+                "x-api-key": self._api_key,
+                "Content-Type": "application/json",
+                "User-Agent": f"{SDK_NAME}/{self._sdk_version}",
+                "accept": "*/*",
+            }
+        )
         # Expose dataset builder factory
         from agentx.evaluations.datasets import DatasetClient
+
         self.datasets = DatasetClient(self)
 
     # ------------------------------------------------------------------
@@ -86,7 +94,9 @@ class EvaluationsClient:
             if resp.status_code == 422:
                 raise AgentXValidationError(resp.text)
             if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
-                logger.debug("Retryable status %d (attempt %d)", resp.status_code, attempt + 1)
+                logger.debug(
+                    "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
+                )
                 last_exc = AgentXEvaluationsError(f"HTTP {resp.status_code}")
                 continue
             if not resp.ok:
@@ -97,6 +107,19 @@ class EvaluationsClient:
                 return resp.text
         raise AgentXEvaluationsError(f"Request failed after retries: {last_exc}")
 
+    # ------------------------------------------------------------------
+    # Model registry
+    # ------------------------------------------------------------------
+
+    def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
+        """List the LLM models AgentX supports — the same set selectable for
+        the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
+        to filter."""
+        params = {"provider": provider} if provider else None
+        data = self._request("GET", "/models", params=params)
+        items = data if isinstance(data, list) else data.get("models", [])
+        return [ModelInfo(**m) for m in items]
+
     # ------------------------------------------------------------------
     # Dataset endpoints
     # ------------------------------------------------------------------
@@ -107,7 +130,10 @@ class EvaluationsClient:
 
     def list_datasets(self) -> List[Dataset]:
         data = self._request("GET", "/datasets")
-        return [Dataset(**d) for d in (data if isinstance(data, list) else data.get("datasets", []))]
+        return [
+            Dataset(**d)
+            for d in (data if isinstance(data, list) else data.get("datasets", []))
+        ]
 
     def get_dataset(self, dataset_id: str) -> Dataset:
         data = self._request("GET", f"/datasets/{dataset_id}")
@@ -124,6 +150,7 @@ class EvaluationsClient:
         python_version: Optional[str] = None,
     ) -> EvaluationRun:
         from agentx.version import VERSION
+
         payload = {
             "datasetId": dataset_id,
             "evaluationSubject": subject.model_dump(by_alias=True, exclude_none=True),
@@ -138,7 +165,9 @@ class EvaluationsClient:
         data = self._request("POST", "/runs", json=payload)
         return EvaluationRun(**data)
 
-    def append_results(self, run_id: str, batch_id: str, results: List[EvaluationResult]) -> BatchAppendResponse:
+    def append_results(
+        self, run_id: str, batch_id: str, results: List[EvaluationResult]
+    ) -> BatchAppendResponse:
         payload = {
             "batchId": batch_id,
             "results": [_result_to_payload(r) for r in results],
@@ -147,7 +176,9 @@ class EvaluationsClient:
         return BatchAppendResponse(**data)
 
     def finalize_run(self, run_id: str) -> Dict[str, Any]:
-        return self._request("POST", f"/runs/{run_id}/finalize", json={"status": "completed"})
+        return self._request(
+            "POST", f"/runs/{run_id}/finalize", json={"status": "completed"}
+        )
 
     def analyze_run(self, run_id: str) -> Dict[str, Any]:
         return self._request("POST", f"/runs/{run_id}/analyze", json={}, timeout=300)
@@ -168,6 +199,7 @@ class EvaluationsClient:
 # Helpers
 # ---------------------------------------------------------------------------
 
+
 def _result_to_payload(r: EvaluationResult) -> dict:
     d = r.model_dump(by_alias=True, exclude_none=True)
     # Rename snake_case fields the model may have stored locally
@@ -180,4 +212,5 @@ def _result_to_payload(r: EvaluationResult) -> dict:
 
 def _python_version() -> str:
     import sys
+
     return f"{sys.version_info.major}.{sys.version_info.minor}"

agentx/evaluations/datasets.py

@@ -27,6 +27,10 @@ class DatasetBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ):
         self._client = client
         self._payload: Dict[str, Any] = {
@@ -38,6 +42,22 @@ class DatasetBuilder:
             "evaluationCriteria": evaluation_criteria,
             "questions": [],
         }
+        # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
+        # jaccard_similarity (computed against each case's expected_results).
+        if vector_similarity:
+            vs: Dict[str, Any] = {"enabled": True}
+            if similarity_model:
+                vs["model"] = similarity_model
+            self._payload["vectorSimilarity"] = vs
+        if jaccard_similarity:
+            self._payload["jaccardSimilarity"] = {"enabled": True}
+        # Sovereignty & Portability — the models to compare on this dataset (use
+        # client.evaluations.list_models() to discover valid ids).
+        if sovereignty_models:
+            self._payload["sovereigntyIndex"] = {
+                "enabled": True,
+                "models": list(sovereignty_models),
+            }
 
     def add_case(
         self,
@@ -57,16 +77,22 @@ class DatasetBuilder:
             main["expectedKnowledgeBase"] = expected_knowledge_base
         if expected_delegations:
             main["expectedDelegations"] = expected_delegations
-        self._payload["questions"].append({
-            "main_question": main,
-            "follow_up_questions": follow_up_questions or [],
-        })
+        self._payload["questions"].append(
+            {
+                "main_question": main,
+                "follow_up_questions": follow_up_questions or [],
+            }
+        )
         return self
 
     def publish(self) -> Dataset:
         if not self._payload["questions"]:
             raise ValueError("Dataset must have at least one case before publishing")
-        logger.info("Publishing dataset '%s' with %d case(s)", self._payload["name"], len(self._payload["questions"]))
+        logger.info(
+            "Publishing dataset '%s' with %d case(s)",
+            self._payload["name"],
+            len(self._payload["questions"]),
+        )
         return self._client.create_dataset(self._payload)
 
     # ------------------------------------------------------------------
@@ -109,9 +135,15 @@ class DatasetBuilder:
                 builder.add_case(
                     query=query,
                     expected_results=row.get("expected_results", "").strip() or None,
-                    expected_capabilities=_split_semi(row.get("expected_capabilities", "")),
-                    expected_knowledge_base=_split_semi(row.get("expected_knowledge_base", "")),
-                    expected_delegations=_split_semi(row.get("expected_delegations", "")),
+                    expected_capabilities=_split_semi(
+                        row.get("expected_capabilities", "")
+                    ),
+                    expected_knowledge_base=_split_semi(
+                        row.get("expected_knowledge_base", "")
+                    ),
+                    expected_delegations=_split_semi(
+                        row.get("expected_delegations", "")
+                    ),
                 )
 
         if invalid:
@@ -150,7 +182,9 @@ class DatasetBuilder:
                 query=query,
                 expected_results=_str_or_none(row.get("expected_results")),
                 expected_capabilities=_split_semi(row.get("expected_capabilities", "")),
-                expected_knowledge_base=_split_semi(row.get("expected_knowledge_base", "")),
+                expected_knowledge_base=_split_semi(
+                    row.get("expected_knowledge_base", "")
+                ),
                 expected_delegations=_split_semi(row.get("expected_delegations", "")),
             )
 
@@ -176,6 +210,10 @@ class DatasetClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ) -> DatasetBuilder:
         return DatasetBuilder(
             self._client,
@@ -185,6 +223,10 @@ class DatasetClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            vector_similarity=vector_similarity,
+            jaccard_similarity=jaccard_similarity,
+            similarity_model=similarity_model,
+            sovereignty_models=sovereignty_models,
         )
 
     def from_csv(self, path: str, name: str, **kwargs) -> DatasetBuilder:
@@ -204,6 +246,7 @@ class DatasetClient:
 # Helpers
 # ---------------------------------------------------------------------------
 
+
 def _split_semi(value: Any) -> Optional[List[str]]:
     if not value:
         return None