agentx-python 0.6.4__tar.gz → 0.6.5__tar.gz

{agentx_python-0.6.4/agentx_python.egg-info → agentx_python-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.4
+Version: 0.6.5
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/agentx.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 import requests
 import os
 import logging
@@ -10,7 +10,12 @@ from agentx.resources.workforce import Workforce
 
 class AgentX:
 
-    def __init__(self, api_key: str = None, base_url: str = None, workspace_id: str = None):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ):
         self.api_key = api_key or os.getenv("AGENTX_API_KEY")
         if self.api_key and not os.getenv("AGENTX_API_KEY"):
             os.environ["AGENTX_API_KEY"] = self.api_key
@@ -32,6 +37,7 @@ class AgentX:
             api_key=self.api_key,
             sdk_version=VERSION,
             base_url=self.base_url,
+            workspace_id=self.workspace_id,
         )
         self.evaluations = EvaluationsRunner(_eval_client)
 

agentx_python-0.6.5/agentx/evaluations/__init__.py

@@ -0,0 +1,7 @@
+from agentx.evaluations.client import EvaluationsClient
+from agentx.evaluations.evaluation_settings import (
+    EvaluationSettingsBuilder,
+    EvaluationSettingsClient,
+)
+
+__all__ = ["EvaluationsClient", "EvaluationSettingsBuilder", "EvaluationSettingsClient"]

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/client.py

@@ -13,6 +13,7 @@ from agentx.evaluations.models import (
     Dataset,
     EvaluationResult,
     EvaluationRun,
+    EvaluationSettings,
     EvaluationSubject,
     ModelInfo,
     Report,
@@ -44,12 +45,20 @@ class AgentXValidationError(AgentXEvaluationsError):
 
 class EvaluationsClient:
     def __init__(
-        self, api_key: str, sdk_version: str = "unknown", base_url: str = None
+        self,
+        api_key: str,
+        sdk_version: str = "unknown",
+        base_url: Optional[str] = None,
+        workspace_id: Optional[str] = None,
     ):
         if not api_key:
             raise AgentXAuthError("AGENTX_API_KEY is required")
         self._api_key = api_key
         self._sdk_version = sdk_version
+        # Falls back to the caller's default workspace server-side when unset — see
+        # _with_workspace(). Without this, dataset/settings/run creation silently land in
+        # whatever workspace the API key's user defaults to, not the one the caller intended.
+        self._workspace_id = workspace_id
         # Priority: constructor arg > env var > SDK default
         # Always append /custom-agent-evaluations so users only need to provide /api/v1
         _api_base = (
@@ -67,15 +76,29 @@ class EvaluationsClient:
                 "accept": "*/*",
             }
         )
-        # Expose dataset builder factory
+        # Expose dataset / evaluation-settings builder factories
         from agentx.evaluations.datasets import DatasetClient
+        from agentx.evaluations.evaluation_settings import EvaluationSettingsClient
 
         self.datasets = DatasetClient(self)
+        self.settings = EvaluationSettingsClient(self)
 
     # ------------------------------------------------------------------
     # Low-level HTTP
     # ------------------------------------------------------------------
 
+    def _with_workspace(self, payload: dict) -> dict:
+        """Injects the client's workspace_id into a request payload, unless the caller already
+        set one explicitly. Without this, requests silently fall back to the API key user's
+        default workspace server-side, which may not be the workspace the caller intended."""
+        if self._workspace_id and not payload.get("workspaceId"):
+            return {**payload, "workspaceId": self._workspace_id}
+        return payload
+
+    def _workspace_params(self) -> Optional[dict]:
+        """Same as _with_workspace(), for GET requests that take workspaceId as a query param."""
+        return {"workspaceId": self._workspace_id} if self._workspace_id else None
+
     def _request(self, method: str, path: str, timeout: int = 30, **kwargs) -> Any:
         url = f"{self._base_url}{path}"
         last_exc: Optional[Exception] = None
@@ -125,20 +148,52 @@ class EvaluationsClient:
     # ------------------------------------------------------------------
 
     def create_dataset(self, payload: dict) -> Dataset:
-        data = self._request("POST", "/datasets", json=payload)
+        data = self._request("POST", "/datasets", json=self._with_workspace(payload))
         return Dataset(**data)
 
     def list_datasets(self) -> List[Dataset]:
-        data = self._request("GET", "/datasets")
+        data = self._request("GET", "/datasets", params=self._workspace_params())
         return [
             Dataset(**d)
             for d in (data if isinstance(data, list) else data.get("datasets", []))
         ]
 
     def get_dataset(self, dataset_id: str) -> Dataset:
-        data = self._request("GET", f"/datasets/{dataset_id}")
+        data = self._request(
+            "GET", f"/datasets/{dataset_id}", params=self._workspace_params()
+        )
         return Dataset(**data)
 
+    # ------------------------------------------------------------------
+    # Evaluation Settings endpoints — standalone grading config, reusable
+    # across datasets.
+    # ------------------------------------------------------------------
+
+    def create_evaluation_settings(self, payload: dict) -> EvaluationSettings:
+        data = self._request(
+            "POST", "/evaluation-settings", json=self._with_workspace(payload)
+        )
+        return EvaluationSettings(**data)
+
+    def list_evaluation_settings(self) -> List[EvaluationSettings]:
+        data = self._request(
+            "GET", "/evaluation-settings", params=self._workspace_params()
+        )
+        return [
+            EvaluationSettings(**e)
+            for e in (
+                data if isinstance(data, list) else data.get("evaluationSettings", [])
+            )
+        ]
+
+    def get_evaluation_settings(self, evaluation_settings_id: str) -> EvaluationSettings:
+        data = self._request(
+            "GET",
+            f"/evaluation-settings/{evaluation_settings_id}",
+            params=self._workspace_params(),
+        )
+        return EvaluationSettings(**data)
+
     # ------------------------------------------------------------------
     # Run endpoints
     # ------------------------------------------------------------------
@@ -148,6 +203,7 @@ class EvaluationsClient:
         dataset_id: str,
         subject: EvaluationSubject,
         python_version: Optional[str] = None,
+        evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRun:
         from agentx.version import VERSION
 
@@ -162,7 +218,9 @@ class EvaluationsClient:
                 "pythonVersion": python_version or _python_version(),
             },
         }
-        data = self._request("POST", "/runs", json=payload)
+        if evaluation_settings_id:
+            payload["evaluationSettingsId"] = evaluation_settings_id
+        data = self._request("POST", "/runs", json=self._with_workspace(payload))
         return EvaluationRun(**data)
 
     def append_results(
@@ -207,6 +265,7 @@ def _result_to_payload(r: EvaluationResult) -> dict:
     d["questionIndex"] = d.pop("question_index", d.get("questionIndex"))
     d["runNumber"] = d.pop("run_number", d.get("runNumber"))
     d["idempotencyKey"] = d.pop("idempotency_key", d.get("idempotencyKey"))
+    d["traceId"] = d.pop("trace_id", d.get("traceId"))
     return {k: v for k, v in d.items() if v is not None}
 
 

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/datasets.py

@@ -29,6 +29,8 @@ class DatasetBuilder:
         evaluation_criteria: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
     ):
@@ -43,7 +45,8 @@ class DatasetBuilder:
             "questions": [],
         }
         # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
-        # jaccard_similarity (computed against each case's expected_results).
+        # jaccard_similarity / bleu_score / rouge_score (computed against each
+        # case's expected_results).
         if vector_similarity:
             vs: Dict[str, Any] = {"enabled": True}
             if similarity_model:
@@ -51,6 +54,10 @@ class DatasetBuilder:
             self._payload["vectorSimilarity"] = vs
         if jaccard_similarity:
             self._payload["jaccardSimilarity"] = {"enabled": True}
+        if bleu_score:
+            self._payload["bleuScore"] = {"enabled": True}
+        if rouge_score:
+            self._payload["rougeScore"] = {"enabled": True}
         # Sovereignty & Portability — the models to compare on this dataset (use
         # client.evaluations.list_models() to discover valid ids).
         if sovereignty_models:
@@ -212,6 +219,8 @@ class DatasetClient:
         evaluation_criteria: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
     ) -> DatasetBuilder:
@@ -225,6 +234,8 @@ class DatasetClient:
             evaluation_criteria=evaluation_criteria,
             vector_similarity=vector_similarity,
             jaccard_similarity=jaccard_similarity,
+            bleu_score=bleu_score,
+            rouge_score=rouge_score,
             similarity_model=similarity_model,
             sovereignty_models=sovereignty_models,
         )

agentx_python-0.6.5/agentx/evaluations/evaluation_settings.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from agentx.evaluations.models import EvaluationSettings
+
+if TYPE_CHECKING:
+    from agentx.evaluations.client import EvaluationsClient
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationSettingsBuilder:
+    """Fluent builder for creating a standalone, reusable grading config (no
+    dataset/questions attached)."""
+
+    def __init__(
+        self,
+        client: "EvaluationsClient",
+        name: str,
+        description: Optional[str] = None,
+        number_of_requests: int = 1,
+        acceptance_criteria: Optional[str] = None,
+        rejection_criteria: Optional[str] = None,
+        evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
+    ):
+        self._client = client
+        self._payload: Dict[str, Any] = {
+            "name": name,
+            "description": description,
+            "numberOfRequests": number_of_requests,
+            "acceptanceCriteria": acceptance_criteria,
+            "rejectionCriteria": rejection_criteria,
+            "evaluationCriteria": evaluation_criteria,
+        }
+        # Opt-in similarity metrics, mirrors DatasetBuilder's config kwargs.
+        if vector_similarity:
+            vs: Dict[str, Any] = {"enabled": True}
+            if similarity_model:
+                vs["model"] = similarity_model
+            self._payload["vectorSimilarity"] = vs
+        if jaccard_similarity:
+            self._payload["jaccardSimilarity"] = {"enabled": True}
+        if bleu_score:
+            self._payload["bleuScore"] = {"enabled": True}
+        if rouge_score:
+            self._payload["rougeScore"] = {"enabled": True}
+        # Sovereignty & Portability — the models to compare when this config runs
+        # (use client.evaluations.list_models() to discover valid ids).
+        if sovereignty_models:
+            self._payload["sovereigntyIndex"] = {
+                "enabled": True,
+                "models": list(sovereignty_models),
+            }
+
+    def publish(self) -> EvaluationSettings:
+        logger.info("Publishing evaluation settings '%s'", self._payload["name"])
+        return self._client.create_evaluation_settings(self._payload)
+
+
+class EvaluationSettingsClient:
+    """Thin wrapper surfaced as client.evaluations.settings."""
+
+    def __init__(self, client: "EvaluationsClient"):
+        self._client = client
+
+    def builder(
+        self,
+        name: str,
+        description: Optional[str] = None,
+        number_of_requests: int = 1,
+        acceptance_criteria: Optional[str] = None,
+        rejection_criteria: Optional[str] = None,
+        evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
+    ) -> EvaluationSettingsBuilder:
+        return EvaluationSettingsBuilder(
+            self._client,
+            name=name,
+            description=description,
+            number_of_requests=number_of_requests,
+            acceptance_criteria=acceptance_criteria,
+            rejection_criteria=rejection_criteria,
+            evaluation_criteria=evaluation_criteria,
+            vector_similarity=vector_similarity,
+            jaccard_similarity=jaccard_similarity,
+            bleu_score=bleu_score,
+            rouge_score=rouge_score,
+            similarity_model=similarity_model,
+            sovereignty_models=sovereignty_models,
+        )
+
+    def get(self, evaluation_settings_id: str) -> EvaluationSettings:
+        return self._client.get_evaluation_settings(evaluation_settings_id)
+
+    def list(self) -> List[EvaluationSettings]:
+        return self._client.list_evaluation_settings()

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/models.py

@@ -76,6 +76,38 @@ class Dataset(BaseModel):
         extra = "ignore"
 
 
+class EvaluationSettings(BaseModel):
+    """A standalone, reusable grading config — no dataset/questions attached.
+    Created via ``client.evaluations.settings.builder(...).publish()`` and run
+    against any dataset by passing its id as ``evaluation_settings_id`` to
+    ``client.evaluations.run(...)``."""
+
+    id: str = Field(alias="_id")
+    name: str
+    description: Optional[str] = None
+    number_of_requests: int = Field(default=1, alias="numberOfRequests")
+    acceptance_criteria: Optional[str] = Field(default=None, alias="acceptanceCriteria")
+    rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria")
+    evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
+    status: str = "published"
+    # Sovereignty & Portability — models selected to compare when this config runs.
+    # Hoisted from the nested ``sovereigntyIndex`` object when enabled.
+    sovereignty_models: List[str] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _extract_sovereignty_models(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            sov = data.get("sovereigntyIndex") or data.get("sovereignty_index") or {}
+            if isinstance(sov, dict) and sov.get("enabled") and sov.get("models"):
+                data = {**data, "sovereignty_models": list(sov.get("models") or [])}
+        return data
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 # ---------------------------------------------------------------------------
 # Evaluation subject
 # ---------------------------------------------------------------------------
@@ -154,6 +186,22 @@ class ServerLimits(BaseModel):
         extra = "ignore"
 
 
+class LiveStatistics(BaseModel):
+    """Rating aggregate computed server-side from submitted results — available
+    as soon as results are scored, independent of the `.analyze()` step (which
+    only adds the LLM-driven qualitative report). Returned on the run resource
+    (``GET /runs/:runId``) as ``liveStatistics``."""
+
+    average_rating: Optional[float] = Field(default=None, alias="averageRating")
+    min_rating: Optional[float] = Field(default=None, alias="minRating")
+    max_rating: Optional[float] = Field(default=None, alias="maxRating")
+    rated_count: int = Field(default=0, alias="ratedCount")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class EvaluationRun(BaseModel):
     run_id: str = Field(alias="runId")
     dataset_id: str = Field(alias="datasetId")
@@ -226,6 +274,10 @@ class EvaluationResult(BaseModel):
     timings: Optional[ResultTimings] = None
     metadata: Optional[Dict[str, Any]] = None
     idempotency_key: Optional[str] = Field(default=None, alias="idempotencyKey")
+    # Links this result to a PromptTrace ingested via client.tracer.trace(..., sync=True) — lets
+    # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
+    # trace for this case, not just the lightweight observable_trace events above.
+    trace_id: Optional[str] = Field(default=None, alias="traceId")
 
     class Config:
         populate_by_name = True
@@ -262,6 +314,8 @@ class BatchAppendResponse(BaseModel):
     scored_results: List[ScoredResult] = Field(
         default_factory=list, alias="scoredResults"
     )
+    # Server-computed rating aggregate, refreshed after this batch — see LiveStatistics.
+    live_statistics: Optional[LiveStatistics] = Field(default=None, alias="liveStatistics")
 
     class Config:
         populate_by_name = True
@@ -280,6 +334,8 @@ class ReportStatistics(BaseModel):
     max_rating: float = Field(default=0.0, alias="maxRating")
     cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
     jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
+    bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
+    rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
 
     class Config:
         populate_by_name = True
@@ -356,6 +412,12 @@ class SovereigntyModelMetrics(BaseModel):
     average_jaccard_similarity: Optional[float] = Field(
         default=None, alias="averageJaccardSimilarity"
     )
+    average_bleu_score: Optional[float] = Field(
+        default=None, alias="averageBleuScore"
+    )
+    average_rouge_score: Optional[float] = Field(
+        default=None, alias="averageRougeScore"
+    )
     average_latency_ms: Optional[float] = Field(default=None, alias="averageLatencyMs")
     total_input_tokens: Optional[int] = Field(default=None, alias="totalInputTokens")
     total_output_tokens: Optional[int] = Field(default=None, alias="totalOutputTokens")
@@ -421,11 +483,15 @@ class Report(BaseModel):
             return data
         stats = data.get("statistics")
         stats = dict(stats) if isinstance(stats, dict) else {}
-        for top_key, nested_key in (
-            ("cosineSimilarity", "cosineSimilarity"),
-            ("cosine_similarity", "cosine_similarity"),
-            ("jaccardSimilarity", "jaccardSimilarity"),
-            ("jaccard_similarity", "jaccard_similarity"),
+        for top_key, nested_key, marker in (
+            ("cosineSimilarity", "cosineSimilarity", "cosine"),
+            ("cosine_similarity", "cosine_similarity", "cosine"),
+            ("jaccardSimilarity", "jaccardSimilarity", "jaccard"),
+            ("jaccard_similarity", "jaccard_similarity", "jaccard"),
+            ("bleuScore", "bleuScore", "bleu"),
+            ("bleu_score", "bleu_score", "bleu"),
+            ("rougeScore", "rougeScore", "rouge"),
+            ("rouge_score", "rouge_score", "rouge"),
         ):
             top_val = data.get(top_key)
             if top_val is None:
@@ -433,15 +499,19 @@ class Report(BaseModel):
             if (
                 stats.get("cosineSimilarity") is None
                 and stats.get("cosine_similarity") is None
-                and "cosine" in nested_key.lower()
+                and marker == "cosine"
             ):
                 stats[nested_key] = top_val
             if (
                 stats.get("jaccardSimilarity") is None
                 and stats.get("jaccard_similarity") is None
-                and "jaccard" in nested_key.lower()
+                and marker == "jaccard"
             ):
                 stats[nested_key] = top_val
+            if stats.get("bleuScore") is None and stats.get("bleu_score") is None and marker == "bleu":
+                stats[nested_key] = top_val
+            if stats.get("rougeScore") is None and stats.get("rouge_score") is None and marker == "rouge":
+                stats[nested_key] = top_val
         if stats:
             data["statistics"] = stats
         return data
@@ -462,6 +532,18 @@ class Report(BaseModel):
             self.statistics.jaccard_similarity if self.statistics is not None else None
         )
 
+    @property
+    def bleu_score(self) -> Optional[float]:
+        """Average BLEU score across scored results, or ``None`` if the metric
+        was not enabled for the dataset or no result has a value yet."""
+        return self.statistics.bleu_score if self.statistics is not None else None
+
+    @property
+    def rouge_score(self) -> Optional[float]:
+        """Average ROUGE-L (F1) score across scored results, or ``None`` if the
+        metric was not enabled for the dataset or no result has a value yet."""
+        return self.statistics.rouge_score if self.statistics is not None else None
+
     @property
     def average_rating(self) -> Optional[float]:
         """Convenience accessor matching cosine_similarity / jaccard_similarity."""

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/reporting.py

@@ -67,6 +67,20 @@ def print_report(report: Report) -> None:
             f"  {dim('Jaccard :')} {jac_color(f'{jac * 100:.1f}%')}  {dim('(token-set overlap)')}"
         )
 
+    bleu = report.bleu_score
+    if bleu is not None:
+        bleu_color = green if bleu >= 0.6 else (yellow if bleu >= 0.3 else red)
+        print(
+            f"  {dim('BLEU    :')} {bleu_color(f'{bleu * 100:.1f}%')}  {dim('(n-gram precision)')}"
+        )
+
+    rouge = report.rouge_score
+    if rouge is not None:
+        rouge_color = green if rouge >= 0.6 else (yellow if rouge >= 0.3 else red)
+        print(
+            f"  {dim('ROUGE-L :')} {rouge_color(f'{rouge * 100:.1f}%')}  {dim('(longest common subsequence)')}"
+        )
+
     if report.consistency_score is not None:
         cs = report.consistency_score
         cs_color = green if cs >= 7 else (yellow if cs >= 4 else red)

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/results.py

@@ -34,6 +34,7 @@ def normalize_result(
 
     output: Optional[dict] = None
     trace = None
+    trace_id: Optional[str] = None
     metadata: Optional[dict] = None
     error: Optional[ResultError] = None
     input_tokens: Optional[int] = None
@@ -50,6 +51,8 @@ def normalize_result(
             output = {"text": str(text)} if text else None
 
         trace = build_trace(raw.get("trace") or raw.get("observable_trace"))
+        trace_id_raw = raw.get("trace_id") or raw.get("traceId")
+        trace_id = str(trace_id_raw) if trace_id_raw else None
         meta_raw = raw.get("metadata")
         if isinstance(meta_raw, dict):
             metadata = redact_dict(meta_raw)
@@ -102,6 +105,7 @@ def normalize_result(
             else None
         ),
         metadata=metadata,
+        traceId=trace_id,
     )
 
 

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/evaluations/runner.py

@@ -13,7 +13,9 @@ from agentx.evaluations.models import (
     EvaluationCase,
     EvaluationResult,
     EvaluationRun,
+    EvaluationSettings,
     EvaluationSubject,
+    LiveStatistics,
     ModelInfo,
     Report,
 )
@@ -54,14 +56,24 @@ class EvaluationRunContext:
         dataset: Dataset,
         run: EvaluationRun,
         subject: EvaluationSubject,
+        evaluation_settings: Optional[EvaluationSettings] = None,
     ):
         self._client = client
         self._dataset = dataset
         self._run = run
         self._subject = subject
+        # When set, this run was started with an independently chosen grading
+        # config (evaluation_settings_id) — its fields take precedence over the
+        # dataset's own for anything execution-time reads (see _build_cases).
+        self._evaluation_settings = evaluation_settings
         self._results: List[EvaluationResult] = []
         self._submitted_keys: Set[str] = set()
         self._report: Optional[Report] = None
+        # Server-computed rating aggregate (Evaluate.liveStatistics) — refreshed
+        # from the response of each append_results()/finalize_run() call. The
+        # API is the single source of truth for this number (same value the
+        # dashboard UI reads), so the SDK does not average results itself.
+        self._live_stats: Optional[LiveStatistics] = None
 
     # ------------------------------------------------------------------
     # Step 1: execute
@@ -70,7 +82,7 @@ class EvaluationRunContext:
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
         """Run all cases locally and submit batches to AgentX."""
         normalized = _wrap_adapter(adapter)
-        cases = _build_cases(self._dataset)
+        cases = _build_cases(self._dataset, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size
 
         # Banner
@@ -80,7 +92,11 @@ class EvaluationRunContext:
         runtime = self._subject.runtime or "local"
         display = self._subject.display_name or ""
         n_q = len(self._dataset.questions)
-        n_r = self._dataset.number_of_requests
+        n_r = (
+            self._evaluation_settings.number_of_requests
+            if self._evaluation_settings
+            else self._dataset.number_of_requests
+        )
 
         print(cyan(sep))
         print(f"  {bold('AgentX Evaluation')}  {dim('—')}  {name}")
@@ -137,6 +153,8 @@ class EvaluationRunContext:
         with Spinner(f"Scoring — AI is rating {n} result{'s' if n != 1 else ''}"):
             try:
                 resp = self._client.append_results(self._run.run_id, batch_id, batch)
+                if resp.live_statistics is not None:
+                    self._live_stats = resp.live_statistics
                 print(
                     f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
                 )
@@ -168,7 +186,9 @@ class EvaluationRunContext:
         print()
         with Spinner("Finalizing — submitting results"):
             try:
-                self._client.finalize_run(self._run.run_id)
+                data = self._client.finalize_run(self._run.run_id)
+                if isinstance(data, dict) and data.get("liveStatistics") is not None:
+                    self._live_stats = LiveStatistics(**data["liveStatistics"])
                 print(f"  {green('✓')}  Finalized")
                 logger.info("Run %s finalized", self._run.run_id)
             except Exception as exc:
@@ -176,6 +196,34 @@ class EvaluationRunContext:
                 logger.error("Finalize failed: %s", exc)
         return self
 
+    # ------------------------------------------------------------------
+    # Live rating stats — server-computed (Evaluate.liveStatistics), refreshed
+    # from the response of each append_results()/finalize_run() call. Available
+    # as soon as .execute() submits batches, no .analyze() required. The SDK
+    # does not average ratings itself — this mirrors exactly what the dashboard
+    # UI reads, computed once in the API.
+    # ------------------------------------------------------------------
+
+    @property
+    def rated_count(self) -> int:
+        """Number of submitted results that have received a rating so far."""
+        return self._live_stats.rated_count if self._live_stats else 0
+
+    @property
+    def average_rating(self) -> Optional[float]:
+        """Live average rating across all results scored so far. Populated as
+        soon as .execute() submits batches — unlike Report.average_rating,
+        does not require .analyze()."""
+        return self._live_stats.average_rating if self._live_stats else None
+
+    @property
+    def min_rating(self) -> Optional[float]:
+        return self._live_stats.min_rating if self._live_stats else None
+
+    @property
+    def max_rating(self) -> Optional[float]:
+        return self._live_stats.max_rating if self._live_stats else None
+
     # ------------------------------------------------------------------
     # Step 3: analyze + report
     # ------------------------------------------------------------------
@@ -223,6 +271,7 @@ class EvaluationsRunner:
     def __init__(self, client: EvaluationsClient):
         self._client = client
         self.datasets = client.datasets
+        self.settings = client.settings
 
     def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
         """List the LLM models AgentX supports — the same set selectable for
@@ -235,20 +284,36 @@ class EvaluationsRunner:
         self,
         dataset_id: str,
         subject: Union[Dict[str, Any], EvaluationSubject],
+        evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRunContext:
+        """Start a run of ``dataset_id`` against ``subject``. Pass
+        ``evaluation_settings_id`` to grade against a standalone, reusable
+        config (created via ``client.evaluations.settings.builder(...)``)
+        instead of the dataset's own default config."""
         if isinstance(subject, dict):
             subject = EvaluationSubject(**subject)
 
         dataset = self._client.get_dataset(dataset_id)
-        run = self._client.init_run(dataset_id, subject)
+        evaluation_settings = (
+            self._client.get_evaluation_settings(evaluation_settings_id)
+            if evaluation_settings_id
+            else None
+        )
+        run = self._client.init_run(
+            dataset_id, subject, evaluation_settings_id=evaluation_settings_id
+        )
         logger.info(
             "Started evaluation run %s on dataset %s (%d case(s), %d repetition(s))",
             run.run_id,
             dataset_id,
             len(dataset.questions),
-            dataset.number_of_requests,
+            evaluation_settings.number_of_requests
+            if evaluation_settings
+            else dataset.number_of_requests,
+        )
+        return EvaluationRunContext(
+            self._client, dataset, run, subject, evaluation_settings=evaluation_settings
         )
-        return EvaluationRunContext(self._client, dataset, run, subject)
 
 
 # ---------------------------------------------------------------------------
@@ -268,14 +333,26 @@ def _wrap_adapter(adapter: AdapterLike) -> Callable[[EvaluationCase], Evaluation
     )
 
 
-def _build_cases(dataset: Dataset) -> List[EvaluationCase]:
+def _build_cases(
+    dataset: Dataset, evaluation_settings: Optional[EvaluationSettings] = None
+) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
-    n_runs = max(dataset.number_of_requests, 1)
-    # Sovereignty & Portability: when the dataset selects comparison models, run
+    # When an independent evaluation_settings was chosen (evaluation_settings_id
+    # passed to .run()), its numberOfRequests/sovereigntyIndex take precedence
+    # over the dataset's own — that's the whole point of decoupling them. With
+    # no evaluation_settings, this reproduces today's exact behavior.
+    n_runs = max(
+        (evaluation_settings.number_of_requests if evaluation_settings else dataset.number_of_requests),
+        1,
+    )
+    # Sovereignty & Portability: when the config selects comparison models, run
     # every question/run once per model in this single run so the report groups
     # results into a per-model portability matrix (mirrors the native route).
     # ``[None]`` keeps legacy single-model behavior (case.model stays unset).
-    models: List[Optional[str]] = list(dataset.sovereignty_models) or [None]
+    sovereignty_models = (
+        evaluation_settings.sovereignty_models if evaluation_settings else dataset.sovereignty_models
+    )
+    models: List[Optional[str]] = list(sovereignty_models) or [None]
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
         for run_num in range(1, n_runs + 1):

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/resources/conversation.py

@@ -82,7 +82,7 @@ class Conversation(BaseModel):
                 f"Failed to retrieve agent details: {response.status_code} - {response.reason}"
             )
 
-    def chat(self, message: str, context: int = None):
+    def chat(self, message: str, context: Optional[int] = None):
         url = f"{api_base()}/access/conversations/{self.id}/message"
         response = requests.post(
             url,
@@ -91,7 +91,7 @@ class Conversation(BaseModel):
         )
         return response.json()
 
-    def chat_stream(self, message: str, context: int = None) -> Iterator[ChatResponse]:
+    def chat_stream(self, message: str, context: Optional[int] = None) -> Iterator[ChatResponse]:
         url = f"{api_base()}/access/conversations/{self.id}/jsonmessagesse"
         response = requests.post(
             url, headers=get_headers(), json={"message": message, "context": context}

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/tracing/ingest_client.py

@@ -90,6 +90,30 @@ class IngestClient:
         """Block until all queued traces have been sent (or timeout elapses)."""
         self._queue.join()
 
+    def send_trace_sync(self, payload: Dict[str, Any]) -> Optional[str]:
+        """
+        Send a trace payload synchronously and return the ingested trace's id, or ``None`` on
+        failure. Used by ``Tracer.trace(..., sync=True)`` when the caller needs the trace_id back
+        immediately (e.g. to attach it to an evaluation result) — unlike ``enqueue()``, this blocks
+        and does not retry, trading the tracer's usual fire-and-forget guarantee for a same-call
+        result. Never raises; a failed send just means no trace_id (never blocks the caller's eval
+        run over a tracing hiccup).
+        """
+        if self._workspace_id:
+            payload = {**payload, "workspaceId": self._workspace_id}
+        try:
+            resp = self._session.post(self._endpoint, json=payload, timeout=10)
+        except requests.RequestException as exc:
+            logger.debug("agentx ingest sync send error: %s", exc)
+            return None
+        if not resp.ok:
+            logger.debug("agentx ingest sync HTTP %d: %s", resp.status_code, resp.text[:200])
+            return None
+        try:
+            return resp.json().get("trace_id")
+        except Exception:
+            return None
+
     def evaluate_trace(
         self,
         trace_id: str,

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/tracing/tracer.py

@@ -77,6 +77,7 @@ class _TraceSpan:
         framework: Optional[str] = None,
         model: Optional[str] = None,
         session_id: Optional[str] = None,
+        sync: bool = False,
     ) -> None:
         self._tracer = tracer
         self.name = name
@@ -86,6 +87,10 @@ class _TraceSpan:
         self._framework = framework
         self._model = model
         self._session_id = session_id
+        # When True, __exit__ sends synchronously (blocking) instead of enqueueing, so trace_id
+        # is populated by the time the `with` block exits — see Tracer.trace()'s sync param.
+        self._sync = sync
+        self._trace_id: Optional[str] = None
 
         # Fields the caller can set while inside the context manager
         self.output: Any = None
@@ -123,9 +128,32 @@ class _TraceSpan:
         if exc_val is not None and self._error is None:
             self._error = str(exc_val)
 
+        # Auto-instrumented integrations (patched Anthropic client, AgentXCallbackHandler, ...)
+        # populate _execution_steps via _record_llm_call while this span is active. Wrapping a raw
+        # API call with no such integration (e.g. a bare `openai` call) never populates it — without
+        # this fallback the Execution Timeline would be empty despite the span having real
+        # input/output, since nothing else here describes what the wrapped code actually did.
+        # Mirrors what the @tracer.trace(...) decorator form has always synthesized for exactly
+        # this case (see _wrap_sync/_wrap_async below).
+        execution_steps = self._execution_steps or (
+            [
+                {
+                    "name": "LLM Call 1",
+                    "duration_ms": latency_ms or 0,
+                    "start_time": self._start,
+                    "end_time": time.time(),
+                    "model": self._model or self._captured_model,
+                    "input": _safe_serialize(self.input) if self.input is not None else None,
+                    "output": _safe_serialize(self.output) if self.output is not None else None,
+                }
+            ]
+            if self.input is not None or self.output is not None
+            else []
+        )
+
         perf = build_performance_summary(
             total_duration_ms=latency_ms or 0,
-            execution_steps=self._execution_steps,
+            execution_steps=execution_steps,
             tool_call_steps=[
                 {
                     "name": tc.get("name"),
@@ -141,7 +169,8 @@ class _TraceSpan:
             has_errors=self._error is not None,
         )
 
-        self._tracer._send(
+        self._trace_id = self._tracer._send(
+            sync=self._sync,
             name=self.name,
             input=_safe_serialize(self.input) if self.input is not None else None,
             output=_safe_serialize(self.output) if self.output is not None else None,
@@ -158,6 +187,13 @@ class _TraceSpan:
         )
         return False  # never suppress exceptions
 
+    @property
+    def trace_id(self) -> Optional[str]:
+        """The ingested trace's id — only populated once this span has exited AND it was opened
+        with ``tracer.trace(..., sync=True)``. ``None`` for the default (async/enqueued) mode,
+        since there's nothing to wait on for a same-call id."""
+        return self._trace_id
+
     # ------------------------------------------------------------------
     # Called by auto-instrumented integrations (e.g. patch_anthropic_client)
     # when this span is the tracer's active span, instead of them sending
@@ -578,10 +614,21 @@ class Tracer:
         framework: Optional[str] = None,
         model: Optional[str] = None,
         session_id: Optional[str] = None,
+        sync: bool = False,
     ) -> _TraceSpan:
         """
         Return a :class:`_TraceSpan` that works as both a decorator and a
         context manager.
+
+        By default the trace is queued and sent on a background thread — fire-and-forget, never
+        blocks the caller, but there's no way to learn the resulting trace_id. Pass ``sync=True``
+        to send it synchronously instead (blocks until ingested) so ``span.trace_id`` is populated
+        once the ``with`` block exits — e.g. to attach the trace to an evaluation result::
+
+            with client.tracer.trace("support_agent_call", framework="openai", sync=True) as span:
+                resp = call_llm(...)
+                span.output = resp
+            return {"output": resp, "trace_id": span.trace_id}
         """
         return _TraceSpan(
             tracer=self,
@@ -591,6 +638,7 @@ class Tracer:
             framework=framework,
             model=model,
             session_id=session_id,
+            sync=sync,
         )
 
     def flush(self, timeout: float = 5.0) -> None:
@@ -767,8 +815,8 @@ class Tracer:
         as-is — the agent is NOT re-run.
 
         Args:
-            trace_id:       ID returned by a previous ``trace()`` call
-                            (available as ``span._trace_id`` after flush).
+            trace_id:       ID of a trace ingested via ``trace(..., sync=True)``
+                            (available as ``span.trace_id`` once that `with` block exits).
             dataset_id:     EvaluationSettings ID to score against.
             question_index: Optional index into the dataset's questions array.
                             When supplied, that question's ``expectedResults``
@@ -784,7 +832,7 @@ class Tracer:
     # Internal
     # ------------------------------------------------------------------
 
-    def _send(self, **kwargs) -> None:
+    def _send(self, sync: bool = False, **kwargs) -> Optional[str]:
         payload = {k: v for k, v in kwargs.items() if v is not None}
         # Remap to snake_case wire format expected by the backend
         wire: Dict[str, Any] = {}
@@ -844,4 +892,7 @@ class Tracer:
                     wire["performance_summary"], pending_tool_calls
                 )
 
+        if sync:
+            return self._client.send_trace_sync(wire)
         self._client.enqueue(wire)
+        return None

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx/util.py

@@ -1,4 +1,5 @@
 import os
+from typing import Optional
 
 _DEFAULT_API_BASE = "https://api.agentx.so/api/v1"
 
@@ -15,5 +16,5 @@ def api_base() -> str:
     return _DEFAULT_API_BASE
 
 
-def get_headers(api_key: str = None):
+def get_headers(api_key: Optional[str] = None):
     return {"accept": "*/*", "x-api-key": api_key or os.getenv("AGENTX_API_KEY")}

agentx_python-0.6.5/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.5"

{agentx_python-0.6.4 → agentx_python-0.6.5/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.4
+Version: 0.6.5
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.4 → agentx_python-0.6.5}/agentx_python.egg-info/SOURCES.txt

@@ -4,12 +4,14 @@ setup.py
 agentx/__init__.py
 agentx/agentx.py
 agentx/exceptions.py
+agentx/py.typed
 agentx/util.py
 agentx/version.py
 agentx/evaluations/__init__.py
 agentx/evaluations/_term.py
 agentx/evaluations/client.py
 agentx/evaluations/datasets.py
+agentx/evaluations/evaluation_settings.py
 agentx/evaluations/models.py
 agentx/evaluations/redaction.py
 agentx/evaluations/reporting.py
@@ -39,6 +41,7 @@ agentx/tracing/tracer.py
 agentx_python.egg-info/PKG-INFO
 agentx_python.egg-info/SOURCES.txt
 agentx_python.egg-info/dependency_links.txt
+agentx_python.egg-info/not-zip-safe
 agentx_python.egg-info/requires.txt
 agentx_python.egg-info/top_level.txt
 tests/test_integration.py

agentx_python-0.6.5/agentx_python.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

{agentx_python-0.6.4 → agentx_python-0.6.5}/setup.py

@@ -21,6 +21,11 @@ setup(
     name="agentx-python",
     version=get_version(),
     packages=find_packages(),
+    # PEP 561: ships type hints for consumers' type checkers (mypy/pyright). Without this marker,
+    # every symbol imported from `agentx` is treated as untyped `Any`, silently disabling type
+    # checking for anything that touches the SDK.
+    package_data={"agentx": ["py.typed"]},
+    zip_safe=False,
     install_requires=[
         "urllib3>=1.26.11",
         "certifi",

agentx_python-0.6.4/agentx/evaluations/__init__.py

@@ -1,3 +0,0 @@
-from agentx.evaluations.client import EvaluationsClient
-
-__all__ = ["EvaluationsClient"]

agentx_python-0.6.4/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.4"