agentx-python 0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

agentx/__init__.py

@@ -0,0 +1,31 @@
+import logging
+
+from agentx.agentx import AgentX
+from agentx.version import VERSION
+from agentx.exceptions import (
+    AgentXError,
+    AgentXAuthError,
+    AgentXAPIError,
+    DatasetNotFound,
+    CINotEnabled,
+    CIRunExpired,
+    CIGateFailure,
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S %Z",
+)
+
+__all__ = [
+    "AgentX",
+    "AgentXError",
+    "AgentXAuthError",
+    "AgentXAPIError",
+    "DatasetNotFound",
+    "CINotEnabled",
+    "CIRunExpired",
+    "CIGateFailure",
+]
+__version__ = VERSION

agentx/agentx.py

@@ -0,0 +1,195 @@
+from typing import List, Optional
+import requests
+import os
+import logging
+
+from agentx.util import get_headers, api_base
+from agentx.resources.agent import Agent
+from agentx.resources.workforce import Workforce
+
+
+class AgentX:
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ):
+        self.api_key = api_key or os.getenv("AGENTX_API_KEY")
+        if self.api_key and not os.getenv("AGENTX_API_KEY"):
+            os.environ["AGENTX_API_KEY"] = self.api_key
+
+        # base_url overrides AGENTX_API_BASE_URL env var (and the SDK default). It is
+        # deliberately NOT written back into os.environ: the constructor used to do that, which
+        # made the last-constructed client silently re-point every other client in the process
+        # (deep-dive round 3, bug #1). Each sub-client below receives this value explicitly and
+        # captures it at construction instead.
+        self.base_url = base_url or os.getenv("AGENTX_API_BASE_URL")
+
+        self.workspace_id = workspace_id or os.getenv("AGENTX_WORKSPACE_ID")
+
+        from agentx.evaluations.client import EvaluationsClient
+        from agentx.evaluations.runner import EvaluationsRunner
+        from agentx.monitor.client import MonitorClient
+        from agentx.tracing.ingest_client import IngestClient
+        from agentx.tracing.tracer import Tracer
+        from agentx.version import VERSION
+
+        _eval_client = EvaluationsClient(
+            api_key=self.api_key,
+            sdk_version=VERSION,
+            base_url=self.base_url,
+            workspace_id=self.workspace_id,
+        )
+        self.evaluations = EvaluationsRunner(_eval_client)
+
+        # Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
+        # against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]), then read
+        # back the resulting alerts/findings with client.monitor.signals. Per-agent coverage and
+        # detection settings (sample rate, retention, threshold overrides like the built-in
+        # "Latency regression" pattern's threshold) are client.monitor.profile.
+        self.monitor = MonitorClient(
+            api_key=self.api_key,
+            sdk_version=VERSION,
+            base_url=self.base_url,
+            workspace_id=self.workspace_id,
+        )
+
+        from agentx.outcomes import OutcomesClient
+
+        # Report real, after-the-fact outcomes ("the ticket got reopened") against traces - the
+        # ground truth behind the dashboard's Judge Calibration card. Self-host only.
+        self.outcomes = OutcomesClient(api_key=self.api_key, base_url=self.base_url)
+
+        from agentx.projects import ProjectsClient
+
+        # Project CRUD (self-host): isolated tenants with their own API keys (P1.1).
+        self.projects = ProjectsClient(api_key=self.api_key, base_url=self.base_url)
+
+        from agentx.traces import TracesClient
+
+        # The read side of tracing: trace-by-id detail and paginated listing (P1.2).
+        self.traces = TracesClient(api_key=self.api_key, base_url=self.base_url)
+
+        from agentx.export import ExportClient
+
+        # Bulk NDJSON egress for backup/migration (P2.1): manifest, per-entity streaming, and
+        # directory dumps. Self-host only.
+        self.export = ExportClient(api_key=self.api_key, base_url=self.base_url)
+
+        from agentx.feedback import FeedbackClient
+
+        # Forward end-user votes ("up"/"down") on traced responses - a "down" raises a signal
+        # directly, and every vote feeds Judge Calibration alongside outcomes. Self-host only.
+        self.feedback = FeedbackClient(api_key=self.api_key, base_url=self.base_url)
+
+        _ingest_client = IngestClient(
+            api_key=self.api_key,
+            sdk_version=VERSION,
+            base_url=self.base_url,
+            workspace_id=self.workspace_id,
+        )
+        self.tracer = Tracer(_ingest_client)
+
+    @classmethod
+    def from_env(cls) -> "AgentX":
+        """Create an AgentX client from the environment: AGENTX_API_KEY plus, for the base URL,
+        the first of AGENTX_API_BASE_URL / AGENTX_SELFHOST_BASE_URL / BASE_URL that is set.
+        The fallbacks match the conventions the self-host samples and .env files already use,
+        so from_env works wherever an explicit AgentX(base_url=...) would."""
+        import os
+
+        base_url = (
+            os.getenv("AGENTX_API_BASE_URL")
+            or os.getenv("AGENTX_SELFHOST_BASE_URL")
+            or os.getenv("BASE_URL")
+        )
+        return cls(base_url=base_url) if base_url else cls()
+
+    def get_agent(self, id: str) -> Agent:
+        url = f"{self.base_url or api_base()}/access/agents/{id}"
+        # Make a GET request to the AgentX API
+        response = requests.get(url, headers=get_headers(self.api_key))
+        # Check if response was successful
+        if response.status_code == 200:
+            return Agent(**response.json())
+        else:
+            raise Exception(f"Failed to retrieve agent: {response.reason}")
+
+    def list_agents(self) -> List[Agent]:
+        url = f"{self.base_url or api_base()}/access/agents"
+        # Make a GET request to the AgentX API
+        response = requests.get(url, headers=get_headers(self.api_key))
+        # Check if response was successful
+        if response.status_code == 200:
+            return [Agent(**agent) for agent in response.json()]
+        else:
+            raise Exception(f"Failed to list agents: {response.reason}")
+
+    @staticmethod
+    def list_workforces() -> List["Workforce"]:
+        """List all workforces/teams."""
+        url = f"{api_base()}/access/teams"
+        response = requests.get(url, headers=get_headers())
+        if response.status_code == 200:
+            return [Workforce(**workforce) for workforce in response.json()]
+        else:
+            raise Exception(
+                f"Failed to list workforces: {response.status_code} - {response.reason}"
+            )
+
+    def ping(self) -> dict:
+        """Verify the client can actually reach AgentX and that the API key is accepted.
+
+        The constructor is deliberately lazy (no network call - standard SDK behavior, so
+        offline construction and tests work), and trace delivery is fire-and-forget, so a
+        wrong ``base_url`` or ``api_key`` otherwise surfaces only as a one-time warning in
+        logs while traces silently go nowhere. Call this once at startup of a long-running
+        service to fail fast instead::
+
+            client = AgentX.from_env()
+            client.ping()  # raises immediately on a bad URL or key
+
+        Raises :class:`agentx.exceptions.AgentXConnectionError` when the URL is unreachable,
+        :class:`agentx.exceptions.AgentXAuthError` when the key is rejected, and
+        :class:`agentx.exceptions.AgentXAPIError` on any other non-OK response. Returns
+        ``{"ok": True, "base_url": ...}`` on success.
+        """
+        from agentx.exceptions import AgentXAPIError, AgentXAuthError, AgentXConnectionError
+
+        base = (self.base_url or api_base()).rstrip("/")
+        # /monitor/patterns: the cheapest key-authenticated endpoint that exists on both the
+        # hosted API and the self-host engine's SDK-facing router.
+        url = f"{base}/monitor/patterns"
+        try:
+            response = requests.get(url, headers=get_headers(self.api_key), timeout=10)
+        except requests.RequestException as exc:
+            raise AgentXConnectionError(
+                f"Cannot reach AgentX at {base} ({exc.__class__.__name__}: {exc}). "
+                "Check base_url / AGENTX_API_BASE_URL - for self-host it should look like "
+                "http://localhost:4700/api/v1."
+            ) from exc
+        if response.status_code in (401, 403):
+            raise AgentXAuthError(
+                f"AgentX at {base} rejected the API key (HTTP {response.status_code}). "
+                "Check api_key / AGENTX_API_KEY - for self-host, copy the 'Default project "
+                "API key' from the engine's startup log."
+            )
+        if not response.ok:
+            raise AgentXAPIError(
+                f"AgentX at {base} responded HTTP {response.status_code} to the health probe.",
+                status_code=response.status_code,
+            )
+        return {"ok": True, "base_url": base}
+
+    def get_profile(self):
+        """Get the current user's profile information."""
+        url = f"{self.base_url or api_base()}/access/getProfile"
+        response = requests.get(url, headers=get_headers(self.api_key))
+        if response.status_code == 200:
+            return response.json()
+        else:
+            raise Exception(
+                f"Failed to get profile: {response.status_code} - {response.reason}"
+            )

agentx/cli.py

@@ -0,0 +1,158 @@
+"""
+`agentx-trace-eval` - thin launcher for AgentX's self-hostable governance engine (Trace,
+Evaluate, Monitor), published separately at github.com/AgentX-ai/AgentX-trace-eval (a Go CLI
+wrapping a Bun-compiled TypeScript engine, not Python). That compiled engine binary is tens of
+megabytes; most `pip install agentx-python` installs are just this SDK talking to the hosted
+AgentX SaaS and would never touch it, so it isn't bundled in this package. Instead, this command
+downloads the matching release into ~/.agentx/bin the first time it's needed (mirroring
+AgentX-trace-eval's own install.sh) and then hands off to the real `agentx-server` binary.
+
+Usage:
+    agentx-trace-eval --dev
+    agentx-trace-eval --port 5000 --db-url postgres://...
+"""
+
+import os
+import platform
+import shutil
+import stat
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+from typing import Tuple
+
+import requests
+
+REPO = "AgentX-ai/AgentX-trace-eval"
+INSTALL_DIR = Path(os.environ.get("AGENTX_INSTALL_DIR", str(Path.home() / ".agentx" / "bin")))
+_BIN_NAMES = ("agentx", "agentx-server", "agentx-engine")
+
+
+def _platform_tag() -> Tuple[str, str]:
+    system = platform.system()
+    if system == "Darwin":
+        os_name = "darwin"
+    elif system == "Linux":
+        os_name = "linux"
+    else:
+        raise SystemExit(
+            f"agentx-trace-eval: unsupported OS {system!r} (self-host currently supports macOS and Linux)"
+        )
+
+    machine = platform.machine().lower()
+    if machine in ("arm64", "aarch64"):
+        arch = "arm64"
+    elif machine in ("x86_64", "amd64"):
+        arch = "amd64"
+    else:
+        raise SystemExit(f"agentx-trace-eval: unsupported architecture {machine!r}")
+
+    return os_name, arch
+
+
+def _release_url(asset: str, version: str) -> str:
+    if version == "latest":
+        return f"https://github.com/{REPO}/releases/latest/download/{asset}"
+    return f"https://github.com/{REPO}/releases/download/{version}/{asset}"
+
+
+def _download(url: str, dest: Path) -> None:
+    response = requests.get(url, stream=True, timeout=60)
+    response.raise_for_status()
+    with open(dest, "wb") as f:
+        for chunk in response.iter_content(chunk_size=1 << 16):
+            f.write(chunk)
+
+
+def _extract_tar(archive: Path, dest_dir: Path) -> None:
+    with tarfile.open(archive) as tar:
+        try:
+            # filter="data" (PEP 706, Python 3.12+) rejects absolute paths/symlink escapes.
+            # Belt-and-suspenders here since these are trusted release assets built by our own CI
+            # (see REPO above), not arbitrary user-supplied archives.
+            tar.extractall(dest_dir, filter="data")
+        except TypeError:
+            tar.extractall(dest_dir)  # Python < 3.12: filter kwarg doesn't exist yet
+
+
+def _install(version: str = "latest") -> None:
+    os_name, arch = _platform_tag()
+    INSTALL_DIR.mkdir(parents=True, exist_ok=True)
+
+    print(f"agentx-trace-eval: downloading agentx ({os_name}/{arch})...", file=sys.stderr)
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_dir = Path(tmp)
+        archive = tmp_dir / "agentx.tar.gz"
+        url = _release_url(f"agentx_{os_name}_{arch}.tar.gz", version)
+        try:
+            _download(url, archive)
+        except requests.HTTPError as exc:
+            raise SystemExit(
+                f"agentx-trace-eval: failed to download {url} ({exc}).\n"
+                "No published release found; see AgentX-trace-eval's README for building from source."
+            )
+
+        _extract_tar(archive, tmp_dir)
+
+        found_any = False
+        for name in _BIN_NAMES:
+            src = tmp_dir / name
+            if not src.exists():
+                continue
+            found_any = True
+            dest = INSTALL_DIR / name
+            shutil.move(str(src), str(dest))
+            dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+        if not found_any:
+            raise SystemExit(f"agentx-trace-eval: downloaded archive from {url} didn't contain any of {_BIN_NAMES}")
+
+    if os.environ.get("AGENTX_TRACE_EVAL_SKIP_WEB"):
+        return
+
+    # Best-effort: the dashboard is a separate, platform-independent asset (see
+    # AgentX-trace-eval's README's "Dashboard release"). Missing it shouldn't block getting the
+    # engine running headless, so a failure here warns and continues rather than raising.
+    print("agentx-trace-eval: downloading dashboard...", file=sys.stderr)
+    web_dir = INSTALL_DIR / "web"
+    with tempfile.TemporaryDirectory() as tmp:
+        web_archive = Path(tmp) / "agentx-web.tar.gz"
+        web_url = _release_url("agentx-web.tar.gz", version)
+        try:
+            _download(web_url, web_archive)
+        except requests.HTTPError:
+            print(
+                f"agentx-trace-eval: no dashboard bundle found at {web_url}, continuing without one",
+                file=sys.stderr,
+            )
+            return
+        if web_dir.exists():
+            shutil.rmtree(web_dir)
+        web_dir.mkdir(parents=True)
+        _extract_tar(web_archive, web_dir)
+
+
+def ensure_installed(version: str = "latest") -> Path:
+    """Downloads agentx-server (+ its engine) into ~/.agentx/bin if not already present there.
+    Returns the path to the agentx-server executable. Set AGENTX_INSTALL_DIR to change where
+    this looks/installs; set AGENTX_TRACE_EVAL_VERSION to pin a release tag instead of latest."""
+    server_path = INSTALL_DIR / "agentx-server"
+    if not server_path.exists():
+        _install(version=version)
+    return server_path
+
+
+def main() -> None:
+    version = os.environ.get("AGENTX_TRACE_EVAL_VERSION", "latest")
+    server_path = ensure_installed(version=version)
+    if not server_path.exists():
+        raise SystemExit(f"agentx-trace-eval: {server_path} still missing after install, giving up")
+
+    # os.execv replaces this process rather than spawning a subprocess: signals, stdio, and the
+    # exit code all pass straight through to agentx-server, same as invoking it directly.
+    os.execv(str(server_path), [str(server_path)] + sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

agentx/evaluations/__init__.py

@@ -0,0 +1,7 @@
+from agentx.evaluations.client import EvaluationsClient
+from agentx.evaluations.evaluation_settings import (
+    EvaluationSettingsBuilder,
+    EvaluationSettingsClient,
+)
+
+__all__ = ["EvaluationsClient", "EvaluationSettingsBuilder", "EvaluationSettingsClient"]

agentx/evaluations/_term.py

@@ -0,0 +1,102 @@
+"""Minimal ANSI terminal helpers - no external dependencies."""
+
+from __future__ import annotations
+
+import itertools
+import sys
+import os
+import threading
+import time
+
+_IS_TTY = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
+
+
+def _c(code: str) -> str:
+    return f"\033[{code}m" if _IS_TTY else ""
+
+
+RESET = _c("0")
+BOLD = _c("1")
+DIM = _c("2")
+GREEN = _c("32")
+YELLOW = _c("33")
+RED = _c("31")
+CYAN = _c("36")
+MAGENTA = _c("35")
+
+
+def green(s: str) -> str:
+    return f"{GREEN}{s}{RESET}"
+
+
+def yellow(s: str) -> str:
+    return f"{YELLOW}{s}{RESET}"
+
+
+def red(s: str) -> str:
+    return f"{RED}{s}{RESET}"
+
+
+def cyan(s: str) -> str:
+    return f"{CYAN}{s}{RESET}"
+
+
+def bold(s: str) -> str:
+    return f"{BOLD}{s}{RESET}"
+
+
+def dim(s: str) -> str:
+    return f"{DIM}{s}{RESET}"
+
+
+def magenta(s: str) -> str:
+    return f"{MAGENTA}{s}{RESET}"
+
+
+class Spinner:
+    """Inline spinner that overwrites the current line on a TTY."""
+
+    _FRAMES = ("⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏")
+
+    def __init__(self, message: str):
+        self._message = message
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        # AGENTX_EVAL_QUIET=1: no spinner thread at all - CI logs stay clean.
+        self._quiet = os.getenv("AGENTX_EVAL_QUIET", "").lower() in ("1", "true", "yes")
+
+    def __enter__(self) -> "Spinner":
+        if self._quiet:
+            return self
+        if not _IS_TTY:
+            print(f"  {self._message}...", flush=True)
+            return self
+        self._stop.clear()
+        self._thread = threading.Thread(target=self._spin, daemon=True)
+        self._thread.start()
+        return self
+
+    def update(self, message: str) -> None:
+        """Change the displayed message while the spinner keeps running."""
+        self._message = message
+        if not _IS_TTY:
+            print(f"  {message}...", flush=True)
+
+    def __exit__(self, *_) -> None:
+        if self._quiet or not _IS_TTY:
+            return
+        self._stop.set()
+        if self._thread:
+            self._thread.join()
+        sys.stdout.write(f"\r{' ' * (len(self._message) + 12)}\r")
+        sys.stdout.flush()
+
+    def _spin(self) -> None:
+        for frame in itertools.cycle(self._FRAMES):
+            if self._stop.is_set():
+                break
+            sys.stdout.write(
+                f"\r  {CYAN}{frame}{RESET}  {self._message}  {DIM}(this may take ~60s+){RESET}"
+            )
+            sys.stdout.flush()
+            time.sleep(0.08)

agentx/evaluations/adapters/__init__.py

@@ -0,0 +1,5 @@
+from agentx.evaluations.adapters.raw import RawCallableAdapter
+from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
+from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
+
+__all__ = ["RawCallableAdapter", "PrecomputedAdapter", "HttpEndpointAdapter"]

agentx/evaluations/adapters/http_endpoint.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+import requests
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_error, normalize_result
+
+
+class HttpEndpointAdapter:
+    """
+    Calls a user-hosted HTTP endpoint for each evaluation case.
+    The SDK (running locally) makes the request - the AgentX API never
+    touches the customer's endpoint.
+
+    The endpoint receives a POST with::
+
+        {"query": "...", "case_id": "...", "metadata": {...}}
+
+    It should respond with a JSON body containing at least ``output``
+    (or ``text``) and optionally ``trace`` and ``metadata``.
+
+    Usage::
+
+        adapter = HttpEndpointAdapter(
+            url="http://localhost:8080/eval",
+            headers={"Authorization": "Bearer my-token"},
+            timeout=30,
+        )
+    """
+
+    def __init__(
+        self,
+        url: str,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: int = 30,
+        method: str = "POST",
+    ):
+        self._url = url
+        self._headers = headers or {}
+        self._timeout = timeout
+        self._method = method.upper()
+
+    def run(self, case: EvaluationCase) -> EvaluationResult:
+        payload = {
+            "query": case.query,
+            "case_id": case.case_id,
+            "question_index": case.question_index,
+            "run_number": case.run_number,
+        }
+        start = time.monotonic()
+        try:
+            resp = requests.request(
+                self._method,
+                self._url,
+                json=payload,
+                headers=self._headers,
+                timeout=self._timeout,
+            )
+            resp.raise_for_status()
+            elapsed = int((time.monotonic() - start) * 1000)
+            raw = resp.json()
+        except Exception as exc:
+            elapsed = int((time.monotonic() - start) * 1000)
+            return normalize_error(case, exc, latency_ms=elapsed)
+
+        return normalize_result(case, raw, latency_ms=elapsed)

agentx/evaluations/adapters/precomputed.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Union
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_result
+
+
+class PrecomputedAdapter:
+    """
+    Adapter for pre-computed outputs - useful when you already have agent
+    responses and just want AgentX to score them.
+
+    Accepts a list or dict keyed by case_id::
+
+        outputs = {
+            "case-0": "You can reset your password from account settings.",
+            "case-1": {"output": "Contact support@example.com", "metadata": {...}},
+        }
+        adapter = PrecomputedAdapter(outputs)
+    """
+
+    def __init__(self, outputs: Union[List[Any], Dict[str, Any]]):
+        if isinstance(outputs, list):
+            self._lookup: Dict[str, Any] = {str(i): v for i, v in enumerate(outputs)}
+        else:
+            self._lookup = {str(k): v for k, v in outputs.items()}
+
+    def run(self, case: EvaluationCase) -> EvaluationResult:
+        raw = self._lookup.get(case.case_id) or self._lookup.get(
+            str(case.question_index)
+        )
+        if raw is None:
+            raw = ""
+        return normalize_result(case, raw, latency_ms=0)

agentx/evaluations/adapters/raw.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from agentx.evaluations.models import EvaluationCase, EvaluationResult
+from agentx.evaluations.results import normalize_error, normalize_result
+
+
+class RawCallableAdapter:
+    """
+    Wraps any Python callable that accepts an EvaluationCase and returns
+    str | dict | EvaluationResult.
+
+    Usage::
+
+        def my_agent(case: EvaluationCase) -> str:
+            return my_llm.invoke(case.query)
+
+        adapter = RawCallableAdapter(my_agent)
+        result = adapter.run(case)
+    """
+
+    def __init__(self, fn: Callable[[EvaluationCase], Any]):
+        self._fn = fn
+
+    def run(
+        self, case: EvaluationCase, latency_ms: int | None = None
+    ) -> EvaluationResult:
+        import time
+
+        start = time.monotonic()
+        try:
+            raw = self._fn(case)
+        except Exception as exc:
+            elapsed = int((time.monotonic() - start) * 1000)
+            return normalize_error(case, exc, latency_ms=elapsed)
+        elapsed = int((time.monotonic() - start) * 1000)
+        return normalize_result(case, raw, latency_ms=elapsed)