agentx-python 0.4.8__py3-none-any.whl → 0.4.10__py3-none-any.whl

agentx/agentx.py

@@ -3,37 +3,51 @@ import requests
 import os
 import logging
 
-from agentx.util import get_headers
+from agentx.util import get_headers, api_base
 from agentx.resources.agent import Agent
 from agentx.resources.workforce import Workforce
 
 
 class AgentX:
 
-    def __init__(self, api_key: str = None):
+    def __init__(self, api_key: str = None, base_url: str = None):
         self.api_key = api_key or os.getenv("AGENTX_API_KEY")
         if self.api_key and not os.getenv("AGENTX_API_KEY"):
             os.environ["AGENTX_API_KEY"] = self.api_key
 
+        # base_url overrides AGENTX_API_BASE_URL env var (and the SDK default)
+        self.base_url = base_url or os.getenv("AGENTX_API_BASE_URL")
+        if self.base_url:
+            os.environ["AGENTX_API_BASE_URL"] = self.base_url
+
+        from agentx.evaluations.client import EvaluationsClient
+        from agentx.evaluations.runner import EvaluationsRunner
+        from agentx.version import VERSION
+
+        _eval_client = EvaluationsClient(
+            api_key=self.api_key,
+            sdk_version=VERSION,
+            base_url=self.base_url,
+        )
+        self.evaluations = EvaluationsRunner(_eval_client)
+
+    @classmethod
+    def from_env(cls) -> "AgentX":
+        """Create an AgentX client using AGENTX_API_KEY (and optionally AGENTX_API_BASE_URL) from the environment."""
+        return cls()
+
     def get_agent(self, id: str) -> Agent:
-        url = f"https://api.agentx.so/api/v1/access/agents/{id}"
+        url = f"{api_base()}/access/agents/{id}"
         # Make a GET request to the AgentX API
         response = requests.get(url, headers=get_headers())
         # Check if response was successful
         if response.status_code == 200:
-            agent_res = response.json()
-            return Agent(
-                id=agent_res.get("_id"),
-                name=agent_res.get("name"),
-                avatar=agent_res.get("avatar"),
-                createdAt=agent_res.get("createdAt"),
-                updatedAt=agent_res.get("updatedAt"),
-            )
+            return Agent(**response.json())
         else:
             raise Exception(f"Failed to retrieve agent: {response.reason}")
 
     def list_agents(self) -> List[Agent]:
-        url = "https://api.agentx.so/api/v1/access/agents"
+        url = f"{api_base()}/access/agents"
         # Make a GET request to the AgentX API
         response = requests.get(url, headers=get_headers())
         # Check if response was successful
@@ -45,7 +59,7 @@ class AgentX:
     @staticmethod
     def list_workforces() -> List["Workforce"]:
         """List all workforces/teams."""
-        url = "https://api.agentx.so/api/v1/access/teams"
+        url = f"{api_base()}/access/teams"
         response = requests.get(url, headers=get_headers())
         if response.status_code == 200:
             return [Workforce(**workforce) for workforce in response.json()]
@@ -56,7 +70,7 @@ class AgentX:
 
     def get_profile(self):
         """Get the current user's profile information."""
-        url = "https://api.agentx.so/api/v1/access/getProfile"
+        url = f"{api_base()}/access/getProfile"
         response = requests.get(url, headers=get_headers())
         if response.status_code == 200:
             return response.json()

agentx/evaluations/__init__.py

@@ -0,0 +1,3 @@
+from agentx.evaluations.client import EvaluationsClient
+
+__all__ = ["EvaluationsClient"]

agentx/evaluations/_term.py

@@ -0,0 +1,91 @@
+"""Minimal ANSI terminal helpers — no external dependencies."""
+
+from __future__ import annotations
+
+import itertools
+import sys
+import threading
+import time
+
+_IS_TTY = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
+
+
+def _c(code: str) -> str:
+    return f"\033[{code}m" if _IS_TTY else ""
+
+
+RESET = _c("0")
+BOLD = _c("1")
+DIM = _c("2")
+GREEN = _c("32")
+YELLOW = _c("33")
+RED = _c("31")
+CYAN = _c("36")
+MAGENTA = _c("35")
+
+
+def green(s: str) -> str:
+    return f"{GREEN}{s}{RESET}"
+
+
+def yellow(s: str) -> str:
+    return f"{YELLOW}{s}{RESET}"
+
+
+def red(s: str) -> str:
+    return f"{RED}{s}{RESET}"
+
+
+def cyan(s: str) -> str:
+    return f"{CYAN}{s}{RESET}"
+
+
+def bold(s: str) -> str:
+    return f"{BOLD}{s}{RESET}"
+
+
+def dim(s: str) -> str:
+    return f"{DIM}{s}{RESET}"
+
+
+def magenta(s: str) -> str:
+    return f"{MAGENTA}{s}{RESET}"
+
+
+class Spinner:
+    """Inline spinner that overwrites the current line on a TTY."""
+
+    _FRAMES = ("⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏")
+
+    def __init__(self, message: str):
+        self._message = message
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def __enter__(self) -> "Spinner":
+        if not _IS_TTY:
+            print(f"  {self._message}...", flush=True)
+            return self
+        self._stop.clear()
+        self._thread = threading.Thread(target=self._spin, daemon=True)
+        self._thread.start()
+        return self
+
+    def __exit__(self, *_) -> None:
+        if not _IS_TTY:
+            return
+        self._stop.set()
+        if self._thread:
+            self._thread.join()
+        sys.stdout.write(f"\r{' ' * (len(self._message) + 12)}\r")
+        sys.stdout.flush()
+
+    def _spin(self) -> None:
+        for frame in itertools.cycle(self._FRAMES):
+            if self._stop.is_set():
+                break
+            sys.stdout.write(
+                f"\r  {CYAN}{frame}{RESET}  {self._message}  {DIM}(this may take ~60s+){RESET}"
+            )
+            sys.stdout.flush()
+            time.sleep(0.08)

agentx/evaluations/adapters/__init__.py

@@ -0,0 +1,5 @@
+from agentx.evaluations.adapters.raw import RawCallableAdapter
+from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
+from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
+
+__all__ = ["RawCallableAdapter", "PrecomputedAdapter", "HttpEndpointAdapter"]

agentx/evaluations/adapters/http_endpoint.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+import requests
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_error, normalize_result
+
+
+class HttpEndpointAdapter:
+    """
+    Calls a user-hosted HTTP endpoint for each evaluation case.
+    The SDK (running locally) makes the request — the AgentX API never
+    touches the customer's endpoint.
+
+    The endpoint receives a POST with::
+
+        {"query": "...", "case_id": "...", "metadata": {...}}
+
+    It should respond with a JSON body containing at least ``output``
+    (or ``text``) and optionally ``trace`` and ``metadata``.
+
+    Usage::
+
+        adapter = HttpEndpointAdapter(
+            url="http://localhost:8080/eval",
+            headers={"Authorization": "Bearer my-token"},
+            timeout=30,
+        )
+    """
+
+    def __init__(
+        self,
+        url: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: int = 30,
+        method: str = "POST",
+    ):
+        self._url = url
+        self._headers = headers or {}
+        self._timeout = timeout
+        self._method = method.upper()
+
+    def run(self, case: EvaluationCase) -> EvaluationResult:
+        payload = {
+            "query": case.query,
+            "case_id": case.case_id,
+            "question_index": case.question_index,
+            "run_number": case.run_number,
+        }
+        start = time.monotonic()
+        try:
+            resp = requests.request(
+                self._method,
+                self._url,
+                json=payload,
+                headers=self._headers,
+                timeout=self._timeout,
+            )
+            resp.raise_for_status()
+            elapsed = int((time.monotonic() - start) * 1000)
+            raw = resp.json()
+        except Exception as exc:
+            elapsed = int((time.monotonic() - start) * 1000)
+            return normalize_error(case, exc, latency_ms=elapsed)
+
+        return normalize_result(case, raw, latency_ms=elapsed)

agentx/evaluations/adapters/precomputed.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Union
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_result
+
+
+class PrecomputedAdapter:
+    """
+    Adapter for pre-computed outputs — useful when you already have agent
+    responses and just want AgentX to score them.
+
+    Accepts a list or dict keyed by case_id::
+
+        outputs = {
+            "case-0": "You can reset your password from account settings.",
+            "case-1": {"output": "Contact support@example.com", "metadata": {...}},
+        }
+        adapter = PrecomputedAdapter(outputs)
+    """
+
+    def __init__(self, outputs: Union[List[Any], Dict[str, Any]]):
+        if isinstance(outputs, list):
+            self._lookup: Dict[str, Any] = {str(i): v for i, v in enumerate(outputs)}
+        else:
+            self._lookup = {str(k): v for k, v in outputs.items()}
+
+    def run(self, case: EvaluationCase) -> EvaluationResult:
+        raw = self._lookup.get(case.case_id) or self._lookup.get(
+            str(case.question_index)
+        )
+        if raw is None:
+            raw = ""
+        return normalize_result(case, raw, latency_ms=0)

agentx/evaluations/adapters/raw.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_error, normalize_result
+
+
+class RawCallableAdapter:
+    """
+    Wraps any Python callable that accepts an EvaluationCase and returns
+    str | dict | EvaluationResult.
+
+    Usage::
+
+        def my_agent(case: EvaluationCase) -> str:
+            return my_llm.invoke(case.query)
+
+        adapter = RawCallableAdapter(my_agent)
+        result = adapter.run(case)
+    """
+
+    def __init__(self, fn: Callable[[EvaluationCase], Any]):
+        self._fn = fn
+
+    def run(
+        self, case: EvaluationCase, latency_ms: int | None = None
+    ) -> EvaluationResult:
+        import time
+
+        start = time.monotonic()
+        try:
+            raw = self._fn(case)
+        except Exception as exc:
+            elapsed = int((time.monotonic() - start) * 1000)
+            return normalize_error(case, exc, latency_ms=elapsed)
+        elapsed = int((time.monotonic() - start) * 1000)
+        return normalize_result(case, raw, latency_ms=elapsed)

agentx/evaluations/client.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import logging
+import os
+import time
+import uuid
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.evaluations.models import (
+    BatchAppendResponse,
+    Dataset,
+    EvaluationResult,
+    EvaluationRun,
+    EvaluationSubject,
+    Report,
+)
+
+logger = logging.getLogger(__name__)
+
+from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE
+
+_DEFAULT_BASE_URL = f"{_UTIL_API_BASE}/custom-agent-evaluations"
+SDK_NAME = "agentx-python"
+
+_RETRYABLE_STATUS = {429, 500, 502, 503, 504}
+_MAX_RETRIES = 3
+_RETRY_BACKOFF = [1.0, 2.0, 4.0]
+
+
+class AgentXEvaluationsError(Exception):
+    pass
+
+
+class AgentXAuthError(AgentXEvaluationsError):
+    pass
+
+
+class AgentXValidationError(AgentXEvaluationsError):
+    pass
+
+
+class EvaluationsClient:
+    def __init__(
+        self, api_key: str, sdk_version: str = "unknown", base_url: str = None
+    ):
+        if not api_key:
+            raise AgentXAuthError("AGENTX_API_KEY is required")
+        self._api_key = api_key
+        self._sdk_version = sdk_version
+        # Priority: constructor arg > env var > SDK default
+        # Always append /custom-agent-evaluations so users only need to provide /api/v1
+        _api_base = (
+            base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE)
+        ).rstrip("/")
+        if not _api_base.endswith("/custom-agent-evaluations"):
+            _api_base = f"{_api_base}/custom-agent-evaluations"
+        self._base_url = _api_base
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "x-api-key": self._api_key,
+                "Content-Type": "application/json",
+                "User-Agent": f"{SDK_NAME}/{self._sdk_version}",
+                "accept": "*/*",
+            }
+        )
+        # Expose dataset builder factory
+        from agentx.evaluations.datasets import DatasetClient
+
+        self.datasets = DatasetClient(self)
+
+    # ------------------------------------------------------------------
+    # Low-level HTTP
+    # ------------------------------------------------------------------
+
+    def _request(self, method: str, path: str, timeout: int = 30, **kwargs) -> Any:
+        url = f"{self._base_url}{path}"
+        last_exc: Optional[Exception] = None
+        for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF):
+            if wait:
+                time.sleep(wait)
+            try:
+                resp = self._session.request(method, url, timeout=timeout, **kwargs)
+            except requests.RequestException as e:
+                last_exc = e
+                logger.debug("Request error (attempt %d): %s", attempt + 1, e)
+                continue
+
+            if resp.status_code == 401:
+                raise AgentXAuthError("Invalid or missing API key")
+            if resp.status_code == 422:
+                raise AgentXValidationError(resp.text)
+            if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
+                logger.debug(
+                    "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
+                )
+                last_exc = AgentXEvaluationsError(f"HTTP {resp.status_code}")
+                continue
+            if not resp.ok:
+                raise AgentXEvaluationsError(f"HTTP {resp.status_code}: {resp.text}")
+            try:
+                return resp.json()
+            except Exception:
+                return resp.text
+        raise AgentXEvaluationsError(f"Request failed after retries: {last_exc}")
+
+    # ------------------------------------------------------------------
+    # Dataset endpoints
+    # ------------------------------------------------------------------
+
+    def create_dataset(self, payload: dict) -> Dataset:
+        data = self._request("POST", "/datasets", json=payload)
+        return Dataset(**data)
+
+    def list_datasets(self) -> List[Dataset]:
+        data = self._request("GET", "/datasets")
+        return [
+            Dataset(**d)
+            for d in (data if isinstance(data, list) else data.get("datasets", []))
+        ]
+
+    def get_dataset(self, dataset_id: str) -> Dataset:
+        data = self._request("GET", f"/datasets/{dataset_id}")
+        return Dataset(**data)
+
+    # ------------------------------------------------------------------
+    # Run endpoints
+    # ------------------------------------------------------------------
+
+    def init_run(
+        self,
+        dataset_id: str,
+        subject: EvaluationSubject,
+        python_version: Optional[str] = None,
+    ) -> EvaluationRun:
+        from agentx.version import VERSION
+
+        payload = {
+            "datasetId": dataset_id,
+            "evaluationSubject": subject.model_dump(by_alias=True, exclude_none=True),
+            "runSource": "sdk",
+            "sdk": {
+                "name": SDK_NAME,
+                "version": VERSION,
+                "runnerVersion": "1",
+                "pythonVersion": python_version or _python_version(),
+            },
+        }
+        data = self._request("POST", "/runs", json=payload)
+        return EvaluationRun(**data)
+
+    def append_results(
+        self, run_id: str, batch_id: str, results: List[EvaluationResult]
+    ) -> BatchAppendResponse:
+        payload = {
+            "batchId": batch_id,
+            "results": [_result_to_payload(r) for r in results],
+        }
+        data = self._request("POST", f"/runs/{run_id}/results", json=payload)
+        return BatchAppendResponse(**data)
+
+    def finalize_run(self, run_id: str) -> Dict[str, Any]:
+        return self._request(
+            "POST", f"/runs/{run_id}/finalize", json={"status": "completed"}
+        )
+
+    def analyze_run(self, run_id: str) -> Dict[str, Any]:
+        return self._request("POST", f"/runs/{run_id}/analyze", json={}, timeout=300)
+
+    def get_run(self, run_id: str) -> Dict[str, Any]:
+        return self._request("GET", f"/runs/{run_id}")
+
+    def get_report(self, run_id: str) -> Report:
+        data = self._request("GET", f"/runs/{run_id}/report")
+        return Report(**data)
+
+    def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
+        data = self._request("GET", f"/runs/{run_id}/missing-results")
+        return data if isinstance(data, list) else data.get("missing", [])
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _result_to_payload(r: EvaluationResult) -> dict:
+    d = r.model_dump(by_alias=True, exclude_none=True)
+    # Rename snake_case fields the model may have stored locally
+    d["caseId"] = d.pop("case_id", d.get("caseId"))
+    d["questionIndex"] = d.pop("question_index", d.get("questionIndex"))
+    d["runNumber"] = d.pop("run_number", d.get("runNumber"))
+    d["idempotencyKey"] = d.pop("idempotency_key", d.get("idempotencyKey"))
+    return {k: v for k, v in d.items() if v is not None}
+
+
+def _python_version() -> str:
+    import sys
+
+    return f"{sys.version_info.major}.{sys.version_info.minor}"