agentx-python 0.4.10__tar.gz → 0.4.11__tar.gz

{agentx_python-0.4.10/agentx_python.egg-info → agentx_python-0.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.4.10
+Version: 0.4.11
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.4.10 → agentx_python-0.4.11}/agentx/evaluations/client.py

@@ -14,6 +14,7 @@ from agentx.evaluations.models import (
     EvaluationResult,
     EvaluationRun,
     EvaluationSubject,
+    ModelInfo,
     Report,
 )
 
@@ -106,6 +107,19 @@ class EvaluationsClient:
                 return resp.text
         raise AgentXEvaluationsError(f"Request failed after retries: {last_exc}")
 
+    # ------------------------------------------------------------------
+    # Model registry
+    # ------------------------------------------------------------------
+
+    def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
+        """List the LLM models AgentX supports — the same set selectable for
+        the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
+        to filter."""
+        params = {"provider": provider} if provider else None
+        data = self._request("GET", "/models", params=params)
+        items = data if isinstance(data, list) else data.get("models", [])
+        return [ModelInfo(**m) for m in items]
+
     # ------------------------------------------------------------------
     # Dataset endpoints
     # ------------------------------------------------------------------

{agentx_python-0.4.10 → agentx_python-0.4.11}/agentx/evaluations/datasets.py

@@ -27,6 +27,10 @@ class DatasetBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ):
         self._client = client
         self._payload: Dict[str, Any] = {
@@ -38,6 +42,22 @@ class DatasetBuilder:
             "evaluationCriteria": evaluation_criteria,
             "questions": [],
         }
+        # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
+        # jaccard_similarity (computed against each case's expected_results).
+        if vector_similarity:
+            vs: Dict[str, Any] = {"enabled": True}
+            if similarity_model:
+                vs["model"] = similarity_model
+            self._payload["vectorSimilarity"] = vs
+        if jaccard_similarity:
+            self._payload["jaccardSimilarity"] = {"enabled": True}
+        # Sovereignty & Portability — the models to compare on this dataset (use
+        # client.evaluations.list_models() to discover valid ids).
+        if sovereignty_models:
+            self._payload["sovereigntyIndex"] = {
+                "enabled": True,
+                "models": list(sovereignty_models),
+            }
 
     def add_case(
         self,
@@ -190,6 +210,10 @@ class DatasetClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
     ) -> DatasetBuilder:
         return DatasetBuilder(
             self._client,
@@ -199,6 +223,10 @@ class DatasetClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            vector_similarity=vector_similarity,
+            jaccard_similarity=jaccard_similarity,
+            similarity_model=similarity_model,
+            sovereignty_models=sovereignty_models,
         )
 
     def from_csv(self, path: str, name: str, **kwargs) -> DatasetBuilder:

{agentx_python-0.4.10 → agentx_python-0.4.11}/agentx/evaluations/models.py

@@ -58,6 +58,18 @@ class Dataset(BaseModel):
     questions: List[DatasetQuestion] = Field(default_factory=list)
     status: str = "published"
     version_id: Optional[str] = Field(default=None, alias="versionId")
+    # Sovereignty & Portability — models selected to compare on this dataset.
+    # Hoisted from the nested ``sovereigntyIndex`` object when enabled.
+    sovereignty_models: List[str] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _extract_sovereignty_models(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            sov = data.get("sovereigntyIndex") or data.get("sovereignty_index") or {}
+            if isinstance(sov, dict) and sov.get("enabled") and sov.get("models"):
+                data = {**data, "sovereignty_models": list(sov.get("models") or [])}
+        return data
 
     class Config:
         populate_by_name = True
@@ -99,6 +111,30 @@ class EvaluationSubject(BaseModel):
         extra = "ignore"
 
 
+# ---------------------------------------------------------------------------
+# Supported models (the AgentX model registry / portability set)
+# ---------------------------------------------------------------------------
+
+
+class ModelInfo(BaseModel):
+    """One entry from the AgentX supported-model registry — the set selectable
+    for the Sovereignty & Portability Index. Returned by ``list_models()``."""
+
+    name: str
+    display: Optional[str] = None
+    provider: Optional[str] = None
+    context_window: Optional[int] = Field(default=None, alias="contextWindow")
+    max_output_tokens: Optional[int] = Field(default=None, alias="maxOutputTokens")
+    input_cost: Optional[float] = Field(default=None, alias="inputCost")
+    output_cost: Optional[float] = Field(default=None, alias="outputCost")
+    knowledge_cutoff: Optional[str] = Field(default=None, alias="knowledgeCutOff")
+    legacy: Optional[bool] = None
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 # ---------------------------------------------------------------------------
 # Run
 # ---------------------------------------------------------------------------
@@ -144,6 +180,10 @@ class EvaluationCase(BaseModel):
     expected_capabilities: Optional[List[str]] = None
     expected_knowledge_base: Optional[List[str]] = None
     expected_delegations: Optional[List[str]] = None
+    # Sovereignty & Portability: the model this case should run on. Set when the
+    # dataset selects comparison models; your callable can read it to pick the
+    # model. The SDK also tags the submitted result with it.
+    model: Optional[str] = None
 
     class Config:
         extra = "ignore"
@@ -299,6 +339,43 @@ class ReportRecommendation(BaseModel):
         extra = "ignore"
 
 
+class SovereigntyModelMetrics(BaseModel):
+    """Per-model row of the Sovereignty & Portability matrix."""
+
+    model: str
+    provider: Optional[str] = None
+    is_baseline: bool = Field(default=False, alias="isBaseline")
+    run_count: int = Field(default=0, alias="runCount")
+    average_rating: Optional[float] = Field(default=None, alias="averageRating")
+    min_rating: Optional[float] = Field(default=None, alias="minRating")
+    max_rating: Optional[float] = Field(default=None, alias="maxRating")
+    rating_variance: Optional[float] = Field(default=None, alias="ratingVariance")
+    average_vector_similarity: Optional[float] = Field(
+        default=None, alias="averageVectorSimilarity"
+    )
+    average_jaccard_similarity: Optional[float] = Field(
+        default=None, alias="averageJaccardSimilarity"
+    )
+    average_latency_ms: Optional[float] = Field(default=None, alias="averageLatencyMs")
+    total_input_tokens: Optional[int] = Field(default=None, alias="totalInputTokens")
+    total_output_tokens: Optional[int] = Field(default=None, alias="totalOutputTokens")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class SovereigntyIndex(BaseModel):
+    """Sovereignty & Portability matrix — side-by-side per-model performance,
+    grouped from the model-tagged results of a run."""
+
+    enabled: bool = False
+    models: List[SovereigntyModelMetrics] = Field(default_factory=list)
+
+    class Config:
+        extra = "ignore"
+
+
 class Report(BaseModel):
     run_id: str = Field(alias="runId")
     dataset_id: str = Field(alias="datasetId")
@@ -325,6 +402,9 @@ class Report(BaseModel):
     low_scoring_cases: List[Dict[str, Any]] = Field(
         default_factory=list, alias="lowScoringCases"
     )
+    sovereignty_index: Optional[SovereigntyIndex] = Field(
+        default=None, alias="sovereigntyIndex"
+    )
     dashboard_url: Optional[str] = Field(default=None, alias="dashboardUrl")
 
     class Config:

{agentx_python-0.4.10 → agentx_python-0.4.11}/agentx/evaluations/runner.py

@@ -14,6 +14,7 @@ from agentx.evaluations.models import (
     EvaluationResult,
     EvaluationRun,
     EvaluationSubject,
+    ModelInfo,
     Report,
 )
 from agentx.evaluations.redaction import redact_dict
@@ -108,6 +109,12 @@ class EvaluationRunContext:
 
             result = normalized(case)
             result.idempotency_key = idem_key
+            # Tag the result with the case's model so the server can group it into
+            # the Sovereignty & Portability matrix (the callable may also set it).
+            if case.model:
+                meta = dict(result.metadata or {})
+                meta.setdefault("model", case.model)
+                result.metadata = meta
             result = EvaluationResult(
                 **{**result.model_dump(), "idempotencyKey": idem_key}
             )
@@ -217,6 +224,13 @@ class EvaluationsRunner:
         self._client = client
         self.datasets = client.datasets
 
+    def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
+        """List the LLM models AgentX supports — the same set selectable for
+        the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
+        to filter. Useful for discovering valid model identifiers to compare
+        against."""
+        return self._client.list_models(provider)
+
     def run(
         self,
         dataset_id: str,
@@ -257,21 +271,29 @@ def _wrap_adapter(adapter: AdapterLike) -> Callable[[EvaluationCase], Evaluation
 def _build_cases(dataset: Dataset) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
     n_runs = max(dataset.number_of_requests, 1)
+    # Sovereignty & Portability: when the dataset selects comparison models, run
+    # every question/run once per model in this single run so the report groups
+    # results into a per-model portability matrix (mirrors the native route).
+    # ``[None]`` keeps legacy single-model behavior (case.model stays unset).
+    models: List[Optional[str]] = list(dataset.sovereignty_models) or [None]
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
         for run_num in range(1, n_runs + 1):
-            cases.append(
-                EvaluationCase(
-                    case_id=f"case-{q_idx}",
-                    question_index=q_idx,
-                    run_number=run_num,
-                    query=mq.query,
-                    expected_results=mq.expected_results,
-                    expected_capabilities=mq.expected_capabilities,
-                    expected_knowledge_base=mq.expected_knowledge_base,
-                    expected_delegations=mq.expected_delegations,
+            for model in models:
+                suffix = f"::{model}" if model else ""
+                cases.append(
+                    EvaluationCase(
+                        case_id=f"case-{q_idx}{suffix}",
+                        question_index=q_idx,
+                        run_number=run_num,
+                        query=mq.query,
+                        expected_results=mq.expected_results,
+                        expected_capabilities=mq.expected_capabilities,
+                        expected_knowledge_base=mq.expected_knowledge_base,
+                        expected_delegations=mq.expected_delegations,
+                        model=model,
+                    )
                 )
-            )
     return cases
 
 

agentx_python-0.4.11/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.4.11"

{agentx_python-0.4.10 → agentx_python-0.4.11/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.4.10
+Version: 0.4.11
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

agentx_python-0.4.10/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.4.10"