agentx-python 0.4.10__py3-none-any.whl → 0.4.12__py3-none-any.whl

agentx/__init__.py

@@ -2,6 +2,15 @@ import logging
 
 from agentx.agentx import AgentX
 from agentx.version import VERSION
+from agentx.exceptions import (
+    AgentXError,
+    AgentXAuthError,
+    AgentXAPIError,
+    DatasetNotFound,
+    CINotEnabled,
+    CIRunExpired,
+    CIGateFailure,
+)
 
 logging.basicConfig(
     level=logging.INFO,
@@ -9,5 +18,14 @@ logging.basicConfig(
     datefmt="%Y-%m-%d %H:%M:%S %Z",
 )
 
-__all__ = ["AgentX"]
+__all__ = [
+    "AgentX",
+    "AgentXError",
+    "AgentXAuthError",
+    "AgentXAPIError",
+    "DatasetNotFound",
+    "CINotEnabled",
+    "CIRunExpired",
+    "CIGateFailure",
+]
 __version__ = VERSION

agentx/agentx.py

@@ -22,6 +22,8 @@ class AgentX:
 
         from agentx.evaluations.client import EvaluationsClient
         from agentx.evaluations.runner import EvaluationsRunner
+        from agentx.tracing.ingest_client import IngestClient
+        from agentx.tracing.tracer import Tracer
         from agentx.version import VERSION
 
         _eval_client = EvaluationsClient(
@@ -31,6 +33,13 @@ class AgentX:
         )
         self.evaluations = EvaluationsRunner(_eval_client)
 
+        _ingest_client = IngestClient(
+            api_key=self.api_key,
+            sdk_version=VERSION,
+            base_url=self.base_url,
+        )
+        self.tracer = Tracer(_ingest_client)
+
     @classmethod
     def from_env(cls) -> "AgentX":
         """Create an AgentX client using AGENTX_API_KEY (and optionally AGENTX_API_BASE_URL) from the environment."""

agentx/evaluations/client.py

@@ -14,6 +14,7 @@ from agentx.evaluations.models import (
     EvaluationResult,
     EvaluationRun,
     EvaluationSubject,
+    ModelInfo,
     Report,
 )
 
@@ -106,6 +107,19 @@ class EvaluationsClient:
                 return resp.text
         raise AgentXEvaluationsError(f"Request failed after retries: {last_exc}")
 
+    # ------------------------------------------------------------------
+    # Model registry
+    # ------------------------------------------------------------------
+
+    def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
+        """List the LLM models AgentX supports — the same set selectable for
+        the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
+        to filter."""
+        params = {"provider": provider} if provider else None
+        data = self._request("GET", "/models", params=params)
+        items = data if isinstance(data, list) else data.get("models", [])
+        return [ModelInfo(**m) for m in items]
+
     # ------------------------------------------------------------------
     # Dataset endpoints
     # ------------------------------------------------------------------

agentx/evaluations/datasets.py

@@ -27,6 +27,10 @@ class DatasetBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ):
         self._client = client
         self._payload: Dict[str, Any] = {
@@ -38,6 +42,22 @@ class DatasetBuilder:
             "evaluationCriteria": evaluation_criteria,
             "questions": [],
         }
+        # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
+        # jaccard_similarity (computed against each case's expected_results).
+        if vector_similarity:
+            vs: Dict[str, Any] = {"enabled": True}
+            if similarity_model:
+                vs["model"] = similarity_model
+            self._payload["vectorSimilarity"] = vs
+        if jaccard_similarity:
+            self._payload["jaccardSimilarity"] = {"enabled": True}
+        # Sovereignty & Portability — the models to compare on this dataset (use
+        # client.evaluations.list_models() to discover valid ids).
+        if sovereignty_models:
+            self._payload["sovereigntyIndex"] = {
+                "enabled": True,
+                "models": list(sovereignty_models),
+            }
 
     def add_case(
         self,
@@ -190,6 +210,10 @@ class DatasetClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ) -> DatasetBuilder:
         return DatasetBuilder(
             self._client,
@@ -199,6 +223,10 @@ class DatasetClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            vector_similarity=vector_similarity,
+            jaccard_similarity=jaccard_similarity,
+            similarity_model=similarity_model,
+            sovereignty_models=sovereignty_models,
         )
 
     def from_csv(self, path: str, name: str, **kwargs) -> DatasetBuilder:

agentx/evaluations/models.py

@@ -58,6 +58,18 @@ class Dataset(BaseModel):
     questions: List[DatasetQuestion] = Field(default_factory=list)
     status: str = "published"
     version_id: Optional[str] = Field(default=None, alias="versionId")
+    # Sovereignty & Portability — models selected to compare on this dataset.
+    # Hoisted from the nested ``sovereigntyIndex`` object when enabled.
+    sovereignty_models: List[str] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _extract_sovereignty_models(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            sov = data.get("sovereigntyIndex") or data.get("sovereignty_index") or {}
+            if isinstance(sov, dict) and sov.get("enabled") and sov.get("models"):
+                data = {**data, "sovereignty_models": list(sov.get("models") or [])}
+        return data
 
     class Config:
         populate_by_name = True
@@ -99,6 +111,30 @@ class EvaluationSubject(BaseModel):
         extra = "ignore"
 
 
+# ---------------------------------------------------------------------------
+# Supported models (the AgentX model registry / portability set)
+# ---------------------------------------------------------------------------
+
+
+class ModelInfo(BaseModel):
+    """One entry from the AgentX supported-model registry — the set selectable
+    for the Sovereignty & Portability Index. Returned by ``list_models()``."""
+
+    name: str
+    display: Optional[str] = None
+    provider: Optional[str] = None
+    context_window: Optional[int] = Field(default=None, alias="contextWindow")
+    max_output_tokens: Optional[int] = Field(default=None, alias="maxOutputTokens")
+    input_cost: Optional[float] = Field(default=None, alias="inputCost")
+    output_cost: Optional[float] = Field(default=None, alias="outputCost")
+    knowledge_cutoff: Optional[str] = Field(default=None, alias="knowledgeCutOff")
+    legacy: Optional[bool] = None
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 # ---------------------------------------------------------------------------
 # Run
 # ---------------------------------------------------------------------------
@@ -144,6 +180,10 @@ class EvaluationCase(BaseModel):
     expected_capabilities: Optional[List[str]] = None
     expected_knowledge_base: Optional[List[str]] = None
     expected_delegations: Optional[List[str]] = None
+    # Sovereignty & Portability: the model this case should run on. Set when the
+    # dataset selects comparison models; your callable can read it to pick the
+    # model. The SDK also tags the submitted result with it.
+    model: Optional[str] = None
 
     class Config:
         extra = "ignore"
@@ -299,6 +339,43 @@ class ReportRecommendation(BaseModel):
         extra = "ignore"
 
 
+class SovereigntyModelMetrics(BaseModel):
+    """Per-model row of the Sovereignty & Portability matrix."""
+
+    model: str
+    provider: Optional[str] = None
+    is_baseline: bool = Field(default=False, alias="isBaseline")
+    run_count: int = Field(default=0, alias="runCount")
+    average_rating: Optional[float] = Field(default=None, alias="averageRating")
+    min_rating: Optional[float] = Field(default=None, alias="minRating")
+    max_rating: Optional[float] = Field(default=None, alias="maxRating")
+    rating_variance: Optional[float] = Field(default=None, alias="ratingVariance")
+    average_vector_similarity: Optional[float] = Field(
+        default=None, alias="averageVectorSimilarity"
+    )
+    average_jaccard_similarity: Optional[float] = Field(
+        default=None, alias="averageJaccardSimilarity"
+    )
+    average_latency_ms: Optional[float] = Field(default=None, alias="averageLatencyMs")
+    total_input_tokens: Optional[int] = Field(default=None, alias="totalInputTokens")
+    total_output_tokens: Optional[int] = Field(default=None, alias="totalOutputTokens")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class SovereigntyIndex(BaseModel):
+    """Sovereignty & Portability matrix — side-by-side per-model performance,
+    grouped from the model-tagged results of a run."""
+
+    enabled: bool = False
+    models: List[SovereigntyModelMetrics] = Field(default_factory=list)
+
+    class Config:
+        extra = "ignore"
+
+
 class Report(BaseModel):
     run_id: str = Field(alias="runId")
     dataset_id: str = Field(alias="datasetId")
@@ -325,6 +402,9 @@ class Report(BaseModel):
     low_scoring_cases: List[Dict[str, Any]] = Field(
         default_factory=list, alias="lowScoringCases"
     )
+    sovereignty_index: Optional[SovereigntyIndex] = Field(
+        default=None, alias="sovereigntyIndex"
+    )
     dashboard_url: Optional[str] = Field(default=None, alias="dashboardUrl")
 
     class Config:

agentx/evaluations/runner.py

@@ -14,6 +14,7 @@ from agentx.evaluations.models import (
     EvaluationResult,
     EvaluationRun,
     EvaluationSubject,
+    ModelInfo,
     Report,
 )
 from agentx.evaluations.redaction import redact_dict
@@ -108,6 +109,12 @@ class EvaluationRunContext:
 
             result = normalized(case)
             result.idempotency_key = idem_key
+            # Tag the result with the case's model so the server can group it into
+            # the Sovereignty & Portability matrix (the callable may also set it).
+            if case.model:
+                meta = dict(result.metadata or {})
+                meta.setdefault("model", case.model)
+                result.metadata = meta
             result = EvaluationResult(
                 **{**result.model_dump(), "idempotencyKey": idem_key}
             )
@@ -217,6 +224,13 @@ class EvaluationsRunner:
         self._client = client
         self.datasets = client.datasets
 
+    def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
+        """List the LLM models AgentX supports — the same set selectable for
+        the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
+        to filter. Useful for discovering valid model identifiers to compare
+        against."""
+        return self._client.list_models(provider)
+
     def run(
         self,
         dataset_id: str,
@@ -257,21 +271,29 @@ def _wrap_adapter(adapter: AdapterLike) -> Callable[[EvaluationCase], Evaluation
 def _build_cases(dataset: Dataset) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
     n_runs = max(dataset.number_of_requests, 1)
+    # Sovereignty & Portability: when the dataset selects comparison models, run
+    # every question/run once per model in this single run so the report groups
+    # results into a per-model portability matrix (mirrors the native route).
+    # ``[None]`` keeps legacy single-model behavior (case.model stays unset).
+    models: List[Optional[str]] = list(dataset.sovereignty_models) or [None]
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
         for run_num in range(1, n_runs + 1):
-            cases.append(
-                EvaluationCase(
-                    case_id=f"case-{q_idx}",
-                    question_index=q_idx,
-                    run_number=run_num,
-                    query=mq.query,
-                    expected_results=mq.expected_results,
-                    expected_capabilities=mq.expected_capabilities,
-                    expected_knowledge_base=mq.expected_knowledge_base,
-                    expected_delegations=mq.expected_delegations,
+            for model in models:
+                suffix = f"::{model}" if model else ""
+                cases.append(
+                    EvaluationCase(
+                        case_id=f"case-{q_idx}{suffix}",
+                        question_index=q_idx,
+                        run_number=run_num,
+                        query=mq.query,
+                        expected_results=mq.expected_results,
+                        expected_capabilities=mq.expected_capabilities,
+                        expected_knowledge_base=mq.expected_knowledge_base,
+                        expected_delegations=mq.expected_delegations,
+                        model=model,
+                    )
                 )
-            )
     return cases
 
 

agentx/exceptions.py

@@ -0,0 +1,46 @@
+"""AgentX SDK exceptions."""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentx.tracing.ci_types import CIRunResult
+
+
+class AgentXError(Exception):
+    """Base class for all AgentX SDK errors."""
+
+
+class AgentXAuthError(AgentXError):
+    """Invalid or missing API key."""
+
+
+class AgentXAPIError(AgentXError):
+    """Unexpected API error."""
+
+    def __init__(self, message: str, status_code: int | None = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class DatasetNotFound(AgentXError):
+    """Dataset ID does not exist or is not accessible from this API key."""
+
+
+class CINotEnabled(AgentXError):
+    """Dataset exists but ci.enabled is false. Enable CI in the dataset settings."""
+
+
+class CIRunExpired(AgentXError):
+    """CI run was not finalized within the 2-hour window."""
+
+
+class CIGateFailure(AgentXError):
+    """Gate result is 'fail'. Raised when fail_on_gate=True."""
+
+    def __init__(self, result: "CIRunResult") -> None:
+        super().__init__(
+            f"CI gate failed: {result.pass_rate:.0%} passed "
+            f"({result.passed_questions}/{result.total_questions} questions)"
+        )
+        self.result = result

agentx/integrations/__init__.py

@@ -0,0 +1,7 @@
+# Framework-specific integrations for production tracing.
+# Each sub-module is lazily importable so the core SDK has no extra dependencies.
+#
+#   from agentx.integrations.langchain import AgentXCallbackHandler
+#   from agentx.integrations.crewai import AgentXCrewObserver
+#   from agentx.integrations.openai_agents import AgentXTracingProcessor
+#   from agentx.integrations.anthropic import patch_anthropic_client

agentx/integrations/anthropic.py

@@ -0,0 +1,152 @@
+"""
+Anthropic SDK integration for AgentX production tracing.
+
+Usage::
+
+    from agentx.integrations.anthropic import patch_anthropic_client
+    import anthropic
+
+    client = anthropic.Anthropic()
+    patch_anthropic_client(client, agentx.tracer, name="claude-agent")
+
+    # All subsequent client.messages.create() calls are now traced automatically.
+
+Requires: ``pip install agentx[anthropic]``
+"""
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+from agentx.tracing.tracer import Tracer, _safe_serialize
+
+
+def patch_anthropic_client(
+    client: Any,
+    tracer: Tracer,
+    name: str = "anthropic-agent",
+    metadata: Optional[Dict[str, Any]] = None,
+    session_id: Optional[str] = None,
+) -> None:
+    """
+    Monkey-patch ``client.messages.create`` and ``client.messages.stream``
+    (if present) to automatically send a trace for every call.
+
+    The original method is still called and its return value is passed through
+    unchanged so nothing in the caller needs to change.
+    """
+    messages = getattr(client, "messages", None)
+    if messages is None:
+        raise ValueError("Provided client does not have a .messages attribute")
+
+    _patch_create(messages, tracer, name, metadata, session_id)
+
+    # stream is optional (not present in all versions)
+    if hasattr(messages, "stream"):
+        _patch_stream(messages, tracer, name, metadata, session_id)
+
+
+def _patch_create(
+    messages_resource: Any,
+    tracer: Tracer,
+    name: str,
+    metadata: Optional[Dict[str, Any]],
+    session_id: Optional[str],
+) -> None:
+    original = messages_resource.create
+    if getattr(original, "_agentx_patched", False):
+        return  # already patched
+
+    def patched_create(*args, **kwargs):
+        start = time.time()
+        error: Optional[str] = None
+        response = None
+        try:
+            response = original(*args, **kwargs)
+            return response
+        except Exception as exc:
+            error = str(exc)
+            raise
+        finally:
+            latency_ms = int((time.time() - start) * 1000)
+            input_messages = kwargs.get("messages") or (args[0] if args else None)
+            model = kwargs.get("model")
+            output = None
+            if response is not None:
+                try:
+                    output = response.content[0].text if response.content else None
+                except Exception:
+                    output = str(response)[:500]
+            tracer._send(
+                name=name,
+                input=_safe_serialize(input_messages),
+                output=output,
+                latency_ms=latency_ms,
+                error=error,
+                framework="anthropic",
+                model=model,
+                metadata=metadata,
+                session_id=session_id,
+            )
+
+    patched_create._agentx_patched = True
+    messages_resource.create = patched_create
+
+
+def _patch_stream(
+    messages_resource: Any,
+    tracer: Tracer,
+    name: str,
+    metadata: Optional[Dict[str, Any]],
+    session_id: Optional[str],
+) -> None:
+    original_stream = messages_resource.stream
+    if getattr(original_stream, "_agentx_patched", False):
+        return
+
+    def patched_stream(*args, **kwargs):
+        start = time.time()
+        ctx = original_stream(*args, **kwargs)
+
+        class _TracedStream:
+            """Thin wrapper that records timing when the stream context exits."""
+
+            def __enter__(self_inner):
+                return ctx.__enter__()
+
+            def __exit__(self_inner, exc_type, exc_val, tb):
+                result = ctx.__exit__(exc_type, exc_val, tb)
+                latency_ms = int((time.time() - start) * 1000)
+                error = str(exc_val) if exc_val else None
+                output = None
+                try:
+                    final = ctx.get_final_message()
+                    output = final.content[0].text if final.content else None
+                except Exception:
+                    pass
+                tracer._send(
+                    name=name,
+                    input=_safe_serialize(kwargs.get("messages")),
+                    output=output,
+                    latency_ms=latency_ms,
+                    error=error,
+                    framework="anthropic",
+                    model=kwargs.get("model"),
+                    metadata=metadata,
+                    session_id=session_id,
+                )
+                return result
+
+            def __iter__(self_inner):
+                return iter(ctx)
+
+            def __aiter__(self_inner):
+                return aiter(ctx)
+
+            def __getattr__(self_inner, item):
+                return getattr(ctx, item)
+
+        return _TracedStream()
+
+    patched_stream._agentx_patched = True
+    messages_resource.stream = patched_stream

agentx/integrations/crewai.py

@@ -0,0 +1,106 @@
+"""
+CrewAI integration for AgentX production tracing.
+
+Usage::
+
+    from agentx.integrations.crewai import AgentXCrewObserver
+
+    observer = AgentXCrewObserver(agentx.tracer, name="my-crew")
+    result = observer.kickoff(crew, inputs={"topic": "AI"})
+
+Or as a context manager around your own kickoff::
+
+    with observer.observe(name="my-crew", input={"topic": "AI"}) as span:
+        result = crew.kickoff(inputs={"topic": "AI"})
+        span.output = result.raw
+
+Requires: ``pip install agentx[crewai]``
+"""
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+from agentx.tracing.tracer import Tracer, _safe_serialize
+
+
+class AgentXCrewObserver:
+    """
+    Wraps a CrewAI ``Crew.kickoff()`` call and sends one trace per execution.
+    Does not require any CrewAI version-specific event system.
+    """
+
+    def __init__(
+        self,
+        tracer: Tracer,
+        name: str = "crewai-crew",
+        metadata: Optional[Dict[str, Any]] = None,
+        session_id: Optional[str] = None,
+    ) -> None:
+        self._tracer = tracer
+        self._name = name
+        self._metadata = metadata
+        self._session_id = session_id
+
+    def kickoff(self, crew: Any, inputs: Optional[Dict[str, Any]] = None) -> Any:
+        """
+        Call ``crew.kickoff(inputs=inputs)``, capture result, and send a trace.
+        Returns the raw CrewAI ``CrewOutput`` object unchanged.
+        """
+        start = time.time()
+        error: Optional[str] = None
+        result = None
+        try:
+            result = crew.kickoff(inputs=inputs or {})
+            return result
+        except Exception as exc:
+            error = str(exc)
+            raise
+        finally:
+            latency_ms = int((time.time() - start) * 1000)
+            output = None
+            if result is not None:
+                output = getattr(result, "raw", None) or _safe_serialize(result)
+
+            # Collect task outputs as tool_calls for observability
+            tool_calls = []
+            if result is not None:
+                task_outputs = getattr(result, "tasks_output", []) or []
+                for task_out in task_outputs:
+                    tool_calls.append(
+                        {
+                            "name": getattr(task_out, "description", "task")[:100],
+                            "output": str(getattr(task_out, "raw", ""))[:500],
+                        }
+                    )
+
+            self._tracer._send(
+                name=self._name,
+                input=_safe_serialize(inputs) if inputs else None,
+                output=output,
+                latency_ms=latency_ms,
+                error=error,
+                framework="crewai",
+                tool_calls=tool_calls or None,
+                metadata=self._metadata,
+                session_id=self._session_id,
+            )
+
+    def observe(
+        self,
+        name: Optional[str] = None,
+        input: Optional[Any] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        session_id: Optional[str] = None,
+    ):
+        """Return a context-manager span for manual kickoff wrapping."""
+        from agentx.tracing.tracer import _TraceSpan
+
+        return _TraceSpan(
+            tracer=self._tracer,
+            name=name or self._name,
+            input=_safe_serialize(input) if input is not None else None,
+            metadata=metadata or self._metadata,
+            framework="crewai",
+            session_id=session_id or self._session_id,
+        )