agentx-python 0.6.6__tar.gz → 0.6.7__tar.gz

{agentx_python-0.6.6 → agentx_python-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.6
+Version: 0.6.7
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -286,8 +286,13 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
 
 ---

{agentx_python-0.6.6 → agentx_python-0.6.7}/README.md

@@ -238,8 +238,13 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
 
 ---

{agentx_python-0.6.6 → agentx_python-0.6.7}/agentx/evaluations/_term.py

@@ -71,6 +71,12 @@ class Spinner:
         self._thread.start()
         return self
 
+    def update(self, message: str) -> None:
+        """Change the displayed message while the spinner keeps running."""
+        self._message = message
+        if not _IS_TTY:
+            print(f"  {message}...", flush=True)
+
     def __exit__(self, *_) -> None:
         if not _IS_TTY:
             return

{agentx_python-0.6.6 → agentx_python-0.6.7}/agentx/evaluations/client.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional
 import requests
 
 from agentx.evaluations.models import (
+    AnalysisStatus,
     BatchAppendResponse,
     Dataset,
     EvaluationResult,
@@ -238,8 +239,28 @@ class EvaluationsClient:
             "POST", f"/runs/{run_id}/finalize", json={"status": "completed"}
         )
 
-    def analyze_run(self, run_id: str) -> Dict[str, Any]:
-        return self._request("POST", f"/runs/{run_id}/analyze", json={}, timeout=300)
+    def analyze_run(
+        self,
+        run_id: str,
+        mode: Optional[str] = None,
+        quality_mode: Optional[str] = None,
+        judges: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        # Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
+        # "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
+        # get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
+        payload: Dict[str, Any] = {}
+        if mode is not None:
+            payload["mode"] = mode
+        if quality_mode is not None:
+            payload["qualityMode"] = quality_mode
+        if judges is not None:
+            payload["judges"] = [{"model": m} for m in judges]
+        return self._request("POST", f"/runs/{run_id}/analyze", json=payload, timeout=30)
+
+    def get_analysis_status(self, run_id: str) -> AnalysisStatus:
+        data = self._request("GET", f"/runs/{run_id}/analyze-status")
+        return AnalysisStatus(**data)
 
     def get_run(self, run_id: str) -> Dict[str, Any]:
         return self._request("GET", f"/runs/{run_id}")

{agentx_python-0.6.6 → agentx_python-0.6.7}/agentx/evaluations/models.py

@@ -438,11 +438,9 @@ class SovereigntyIndex(BaseModel):
         extra = "ignore"
 
 
-class Report(BaseModel):
-    run_id: str = Field(alias="runId")
-    dataset_id: str = Field(alias="datasetId")
-    status: str = "completed"
-    statistics: Optional[ReportStatistics] = None
+class AnalysisResult(BaseModel):
+    """Shared qualitative-report fields, produced by ``client.evaluations.run(...).analyze()``."""
+
     summary: Optional[str] = None
     consistency_score: Optional[float] = Field(default=None, alias="consistencyScore")
     instruction_adherence: Optional[ReportInstructionAdherence] = Field(
@@ -461,6 +459,17 @@ class Report(BaseModel):
     weaknesses: List[str] = Field(default_factory=list)
     overall_rating: Optional[str] = Field(default=None, alias="overallRating")
     recommendations: List[ReportRecommendation] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class Report(AnalysisResult):
+    run_id: str = Field(alias="runId")
+    dataset_id: str = Field(alias="datasetId")
+    status: str = "completed"
+    statistics: Optional[ReportStatistics] = None
     low_scoring_cases: List[Dict[str, Any]] = Field(
         default_factory=list, alias="lowScoringCases"
     )
@@ -548,3 +557,52 @@ class Report(BaseModel):
     def average_rating(self) -> Optional[float]:
         """Convenience accessor matching cosine_similarity / jaccard_similarity."""
         return self.statistics.average_rating if self.statistics is not None else None
+
+
+class AnalysisLevelProgress(BaseModel):
+    total: int = 0
+    completed: int = 0
+    failed: int = 0
+    percentage: int = 0
+
+    class Config:
+        extra = "ignore"
+
+
+class AnalysisProgress(BaseModel):
+    overall_percentage: int = Field(default=0, alias="overallPercentage")
+    current_level: Optional[str] = Field(default=None, alias="currentLevel")
+    levels: Dict[str, AnalysisLevelProgress] = Field(default_factory=dict)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class AnalysisFailureReason(BaseModel):
+    code: str
+    message: str
+    retryable: bool = False
+
+    class Config:
+        extra = "ignore"
+
+
+class AnalysisStatus(BaseModel):
+    """Returned by ``client.evaluations.run(...).analyze()``'s polling loop
+    (``EvaluationsClient.get_analysis_status``). ``status`` is terminal once it's one of
+    "completed", "partially_failed", or "failed"."""
+
+    job_id: Optional[str] = Field(default=None, alias="jobId")
+    status: str = "not_started"
+    progress: AnalysisProgress = Field(default_factory=AnalysisProgress)
+    failure_reason: Optional[AnalysisFailureReason] = Field(default=None, alias="failureReason")
+    warnings: List[Dict[str, Any]] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+    @property
+    def is_terminal(self) -> bool:
+        return self.status in ("completed", "partially_failed", "failed")

{agentx_python-0.6.6 → agentx_python-0.6.7}/agentx/evaluations/runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import time
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
@@ -9,6 +10,7 @@ from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
 from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
 from agentx.evaluations.client import EvaluationsClient
 from agentx.evaluations.models import (
+    AnalysisStatus,
     Dataset,
     EvaluationCase,
     EvaluationResult,
@@ -43,6 +45,15 @@ AdapterLike = Union[
     HttpEndpointAdapter,
 ]
 
+_ANALYSIS_LEVEL_LABELS = {
+    "l1_score": "scoring responses",
+    "l2_question_reduce": "reducing questions",
+    "l3_cluster_reduce": "reducing clusters",
+    "l4_final_reduce": "writing final report",
+}
+
+_DEFAULT_JUDGE_MODEL = "gpt-5.5"
+
 
 class EvaluationRunContext:
     """
@@ -228,12 +239,61 @@ class EvaluationRunContext:
     # Step 3: analyze + report
     # ------------------------------------------------------------------
 
-    def analyze(self) -> Report:
+    def analyze(
+        self,
+        mode: Optional[str] = None,
+        quality_mode: Optional[str] = None,
+        judges: Optional[List[str]] = None,
+        poll_interval: float = 5.0,
+        timeout: float = 1800.0,
+    ) -> Report:
+        """Generate the qualitative AI analysis report.
+
+        Runs the same durable, multi-stage pipeline as the dashboard's "Analyze" button: each
+        response is scored by 1-3 LLM judges (``judges``), then reduced into the final report.
+        This starts the job and polls until it finishes, which can take noticeably longer than a
+        single LLM call for larger runs.
+
+        Args:
+            mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
+            quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
+            judges: 1-3 model ids, e.g. ``["gpt-5.5", "claude-opus-4-8"]``. Defaults to a single
+                judge, ``["gpt-5.5"]``, rather than the dashboard's 3-judge default - SDK runs are
+                typically lighter-weight, quick-start evaluations.
+            poll_interval: seconds between status checks while waiting.
+            timeout: give up waiting after this many seconds (the job keeps running server-side;
+                call ``get_report()`` later to check on it).
+        """
+        if judges is not None and not (1 <= len(judges) <= 3):
+            raise ValueError("judges must contain 1-3 model ids")
+        resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
+
         print()
-        with Spinner("Analyzing — AI is reviewing your results"):
+        with Spinner("Analyzing — AI is reviewing your results") as spinner:
             try:
-                self._client.analyze_run(self._run.run_id)
-                print(f"  {green('✓')}  Analysis complete")
+                self._client.analyze_run(
+                    self._run.run_id,
+                    mode=mode,
+                    quality_mode=quality_mode,
+                    judges=resolved_judges,
+                )
+                deadline = time.monotonic() + timeout
+                status = self._client.get_analysis_status(self._run.run_id)
+                while not status.is_terminal and time.monotonic() < deadline:
+                    level = _ANALYSIS_LEVEL_LABELS.get(status.progress.current_level, "")
+                    spinner.update(
+                        f"Analyzing, {level + ' ' if level else ''}{status.progress.overall_percentage}%"
+                    )
+                    time.sleep(poll_interval)
+                    status = self._client.get_analysis_status(self._run.run_id)
+
+                if not status.is_terminal:
+                    print(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
+                elif status.status == "failed":
+                    reason = status.failure_reason.message if status.failure_reason else "unknown error"
+                    print(f"  {red('✗')}  Analyze failed: {dim(reason)}")
+                else:
+                    print(f"  {green('✓')}  Analysis complete")
             except Exception as exc:
                 print(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
                 logger.warning("Analyze request failed: %s", exc)
@@ -280,6 +340,12 @@ class EvaluationsRunner:
         against."""
         return self._client.list_models(provider)
 
+    def get_analysis_status(self, run_id: str) -> AnalysisStatus:
+        """Check on an in-progress ``.analyze()`` job by run id, without needing
+        the ``EvaluationRunContext`` that started it (e.g. from a separate
+        script execution)."""
+        return self._client.get_analysis_status(run_id)
+
     def run(
         self,
         dataset_id: str,

agentx_python-0.6.7/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.7"

{agentx_python-0.6.6 → agentx_python-0.6.7}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.6
+Version: 0.6.7
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -286,8 +286,13 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
 
 ---

agentx_python-0.6.6/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.6"